diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/sched/fair.c | 97 |
1 files changed, 72 insertions, 25 deletions
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 0c9533a2854a..8bdcf7485962 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -2801,10 +2801,34 @@ static inline u64 cpu_load(int cpu) return scale_load_to_cpu(rq->cumulative_runnable_avg, cpu); } +static inline u64 cpu_load_sync(int cpu, int sync) +{ + struct rq *rq = cpu_rq(cpu); + u64 load; + + load = rq->cumulative_runnable_avg; + + /* + * If load is being checked in a sync wakeup environment, + * we may want to discount the load of the currently running + * task. + */ + if (sync && cpu == smp_processor_id()) { + if (load > rq->curr->ravg.demand) + load -= rq->curr->ravg.demand; + else + load = 0; + } + + return scale_load_to_cpu(load, cpu); +} + static int -spill_threshold_crossed(struct task_struct *p, struct rq *rq, int cpu) +spill_threshold_crossed(struct task_struct *p, struct rq *rq, int cpu, + int sync) { - u64 total_load = cpu_load(cpu) + scale_load_to_cpu(task_load(p), cpu); + u64 total_load = cpu_load_sync(cpu, sync) + + scale_load_to_cpu(task_load(p), cpu); if (total_load > sched_spill_load || (rq->nr_running + 1) > sysctl_sched_spill_nr_run) @@ -2822,6 +2846,26 @@ int mostly_idle_cpu(int cpu) && !sched_cpu_high_irqload(cpu); } +static int mostly_idle_cpu_sync(int cpu, int sync) +{ + struct rq *rq = cpu_rq(cpu); + u64 load = cpu_load_sync(cpu, sync); + int nr_running; + + nr_running = rq->nr_running; + + /* + * Sync wakeups mean that the waker task will go to sleep + * soon so we should discount its load from this test. + */ + if (sync && cpu == smp_processor_id()) + nr_running--; + + return load <= rq->mostly_idle_load && + nr_running <= rq->mostly_idle_nr_run && + !sched_cpu_high_irqload(cpu); +} + static int boost_refcount; static DEFINE_SPINLOCK(boost_lock); static DEFINE_MUTEX(boost_mutex); @@ -2936,15 +2980,15 @@ static int task_will_fit(struct task_struct *p, int cpu) return 0; } -static int eligible_cpu(struct task_struct *p, int cpu) +static int eligible_cpu(struct task_struct *p, int cpu, int sync) { struct rq *rq = cpu_rq(cpu); - if (mostly_idle_cpu(cpu)) + if (mostly_idle_cpu_sync(cpu, sync)) return 1; if (rq->capacity != max_capacity) - return !spill_threshold_crossed(p, rq, cpu); + return !spill_threshold_crossed(p, rq, cpu, sync); return 0; } @@ -3020,7 +3064,7 @@ static unsigned int power_cost(struct task_struct *p, int cpu) return power_cost_at_freq(cpu, task_freq); } -static int best_small_task_cpu(struct task_struct *p) +static int best_small_task_cpu(struct task_struct *p, int sync) { int best_busy_cpu = -1, best_fallback_cpu = -1; int min_cost_cpu = -1, min_cstate_cpu = -1; @@ -3042,8 +3086,8 @@ static int best_small_task_cpu(struct task_struct *p) for_each_cpu(i, &search_cpus) { trace_sched_cpu_load(cpu_rq(i), idle_cpu(i), - mostly_idle_cpu(i), sched_irqload(i), - power_cost(p, i)); + mostly_idle_cpu_sync(i, sync), + sched_irqload(i), power_cost(p, i)); cpu_cost = power_cost(p, i); if (cpu_cost < min_cost) { @@ -3057,7 +3101,8 @@ static int best_small_task_cpu(struct task_struct *p) /* Optimization to steer task towards the minimum power cost CPU. The tradeoff is that we may have to check the same information again in pass 2 */ - if (!cpu_rq(min_cost_cpu)->cstate && mostly_idle_cpu(min_cost_cpu)) + if (!cpu_rq(min_cost_cpu)->cstate && + mostly_idle_cpu_sync(min_cost_cpu, sync)) return min_cost_cpu; for_each_cpu(i, &search_cpus) { @@ -3080,11 +3125,11 @@ static int best_small_task_cpu(struct task_struct *p) continue; } - if (mostly_idle_cpu(i)) + if (mostly_idle_cpu_sync(i, sync)) return i; - load = cpu_load(i); - if (!spill_threshold_crossed(p, rq, i)) { + load = cpu_load_sync(i, sync); + if (!spill_threshold_crossed(p, rq, i, sync)) { if (load < min_busy_load) { min_busy_load = load; best_busy_cpu = i; @@ -3171,7 +3216,8 @@ static int select_packing_target(struct task_struct *p, int best_cpu) /* return cheapest cpu that can fit this task */ -static int select_best_cpu(struct task_struct *p, int target, int reason) +static int select_best_cpu(struct task_struct *p, int target, int reason, + int sync) { int i, best_cpu = -1, fallback_idle_cpu = -1, min_cstate_cpu = -1; int prev_cpu = task_cpu(p); @@ -3186,7 +3232,7 @@ static int select_best_cpu(struct task_struct *p, int target, int reason) trace_sched_task_load(p, small_task, boost, reason); if (small_task && !boost) { - best_cpu = best_small_task_cpu(p); + best_cpu = best_small_task_cpu(p, sync); goto done; } @@ -3194,8 +3240,8 @@ static int select_best_cpu(struct task_struct *p, int target, int reason) for_each_cpu_and(i, tsk_cpus_allowed(p), cpu_online_mask) { trace_sched_cpu_load(cpu_rq(i), idle_cpu(i), - mostly_idle_cpu(i), sched_irqload(i), - power_cost(p, i)); + mostly_idle_cpu_sync(i, sync), + sched_irqload(i), power_cost(p, i)); if (skip_cpu(p, i, reason)) continue; @@ -3206,8 +3252,8 @@ static int select_best_cpu(struct task_struct *p, int target, int reason) * where the task will fit. */ if (!task_will_fit(p, i)) { - if (mostly_idle_cpu(i)) { - load = cpu_load(i); + if (mostly_idle_cpu_sync(i, sync)) { + load = cpu_load_sync(i, sync); if (load < min_fallback_load) { min_fallback_load = load; fallback_idle_cpu = i; @@ -3216,7 +3262,7 @@ static int select_best_cpu(struct task_struct *p, int target, int reason) continue; } - if (!eligible_cpu(p, i)) + if (!eligible_cpu(p, i, sync)) continue; /* @@ -3225,7 +3271,7 @@ static int select_best_cpu(struct task_struct *p, int target, int reason) * spill. */ - load = cpu_load(i); + load = cpu_load_sync(i, sync); cpu_cost = power_cost(p, i); cstate = cpu_rq(i)->cstate; @@ -3293,7 +3339,7 @@ static int select_best_cpu(struct task_struct *p, int target, int reason) } if (min_cstate_cpu >= 0 && (prefer_idle || - !(best_cpu >= 0 && mostly_idle_cpu(best_cpu)))) + !(best_cpu >= 0 && mostly_idle_cpu_sync(best_cpu, sync)))) best_cpu = min_cstate_cpu; done: if (best_cpu < 0) { @@ -3640,7 +3686,7 @@ void check_for_migration(struct rq *rq, struct task_struct *p) return; raw_spin_lock(&migration_lock); - new_cpu = select_best_cpu(p, cpu, reason); + new_cpu = select_best_cpu(p, cpu, reason, 0); if (new_cpu != cpu) { active_balance = kick_active_balance(rq, p, new_cpu); @@ -3669,7 +3715,8 @@ static inline int task_will_fit(struct task_struct *p, int cpu) return 1; } -static inline int select_best_cpu(struct task_struct *p, int target, int reason) +static inline int select_best_cpu(struct task_struct *p, int target, + int reason, int sync) { return 0; } @@ -3685,7 +3732,7 @@ static inline int power_cost(struct task_struct *p, int cpu) } static inline int -spill_threshold_crossed(struct task_struct *p, struct rq *rq, int cpu) +spill_threshold_crossed(struct task_struct *p, struct rq *rq, int cpu, int sync) { return 0; } @@ -6244,7 +6291,7 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_f int sync = wake_flags & WF_SYNC; if (sched_enable_hmp) - return select_best_cpu(p, prev_cpu, 0); + return select_best_cpu(p, prev_cpu, 0, sync); if (sd_flag & SD_BALANCE_WAKE) want_affine = !wake_wide(p) && cpumask_test_cpu(cpu, tsk_cpus_allowed(p)); |