From b4d610feb2242c9d43387e02c7529b7c14c2b683 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Thu, 24 Aug 2017 22:03:39 +0100 Subject: [PATCH] sched! Fixes: 3fed382b46ba ("sched/numa: Implement NUMA node level wake_affine()") Cc: Rik van Riel Cc: Mel Gorman Cc: Mike Galbraith Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Ingo Molnar --- kernel/sched/fair.c | 59 +++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 53 insertions(+), 6 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index c95880e216f6..912ff81a0f3f 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -123,6 +123,11 @@ int __weak arch_asym_cpu_priority(int cpu) unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL; #endif +static unsigned long weighted_cpuload(const int cpu); +static unsigned long source_load(int cpu, int type); +static unsigned long target_load(int cpu, int type); +static unsigned long capacity_of(int cpu); + /* * The margin used when comparing utilization with CPU capacity: * util * margin < capacity * 1024 @@ -1378,11 +1383,6 @@ bool should_numa_migrate_memory(struct task_struct *p, struct page * page, group_faults_cpu(ng, src_nid) * group_faults(p, dst_nid) * 4; } -static unsigned long weighted_cpuload(const int cpu); -static unsigned long source_load(int cpu, int type); -static unsigned long target_load(int cpu, int type); -static unsigned long capacity_of(int cpu); - /* Cached statistics for all CPUs within a node */ struct numa_stats { unsigned long nr_running; @@ -2657,7 +2657,54 @@ static inline bool numa_wake_affine(struct sched_domain *sd, struct task_struct *p, int this_cpu, int prev_cpu, int sync) { - return true; + s64 this_eff_load, prev_eff_load; + s64 this_load, prev_load; + unsigned long task_load; + int idx; + + idx = sd->wake_idx; + prev_load = source_load(prev_cpu, idx); + this_load = target_load(this_cpu, idx); + task_load = task_h_load(p); + + /* + * If sync wakeup then subtract the (maximum possible) + * effect of the currently running task from the load + * of the current CPU: + */ + if (sync) { + unsigned long current_load = task_h_load(current); + + if (this_load > current_load) + this_load -= current_load; + else + this_load = 0; + } + + /* + * In low-load situations, where prev_cpu is idle and this_cpu is idle + * due to the sync cause above having dropped this_load to 0, we'll + * always have an imbalance, but there's really nothing you can do + * about that, so that's good too. + * + * Otherwise check if either cpus are near enough in load to allow this + * task to be woken on this_cpu. + */ + if (prev_load > task_load) { + prev_eff_load = 100 + (sd->imbalance_pct - 100) / 2; + prev_eff_load *= capacity_of(this_cpu); + prev_eff_load *= prev_load - task_load; + } else + return false; + + if (this_load > 0) { + this_eff_load = 100; + this_eff_load *= capacity_of(prev_cpu); + this_eff_load *= this_load + task_load; + } else + return true; + + return this_eff_load <= prev_eff_load; } #endif /* !SMP */ #endif /* CONFIG_NUMA_BALANCING */ -- 2.14.1