diff options
| -rw-r--r-- | Documentation/scheduler/sched-hmp.txt | 2 | ||||
| -rw-r--r-- | drivers/base/cpu.c | 39 | ||||
| -rw-r--r-- | include/linux/sched.h | 2 | ||||
| -rw-r--r-- | include/linux/sched/sysctl.h | 1 | ||||
| -rw-r--r-- | kernel/sched/core.c | 1 | ||||
| -rw-r--r-- | kernel/sched/fair.c | 48 | ||||
| -rw-r--r-- | kernel/sched/sched.h | 1 | ||||
| -rw-r--r-- | kernel/sysctl.c | 7 |
8 files changed, 81 insertions, 20 deletions
diff --git a/Documentation/scheduler/sched-hmp.txt b/Documentation/scheduler/sched-hmp.txt index 581a9f4d5159..354905acb103 100644 --- a/Documentation/scheduler/sched-hmp.txt +++ b/Documentation/scheduler/sched-hmp.txt @@ -1247,7 +1247,7 @@ the CPU. ** 7.23 sched_prefer_idle -Appears at: /proc/sys/kernel/sched_prefer_idle +Appears at: /sys/devices/system/cpu/cpuX/sched_prefer_idle Default value: 1 diff --git a/drivers/base/cpu.c b/drivers/base/cpu.c index a59fa57ef42e..d84a4d646975 100644 --- a/drivers/base/cpu.c +++ b/drivers/base/cpu.c @@ -289,17 +289,56 @@ static ssize_t __ref store_sched_mostly_idle_nr_run(struct device *dev, return err; } +static ssize_t show_sched_prefer_idle(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cpu *cpu = container_of(dev, struct cpu, dev); + ssize_t rc; + int cpunum; + int prefer_idle; + + cpunum = cpu->dev.id; + + prefer_idle = sched_get_cpu_prefer_idle(cpunum); + + rc = snprintf(buf, PAGE_SIZE-2, "%d\n", prefer_idle); + + return rc; +} + +static ssize_t __ref store_sched_prefer_idle(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct cpu *cpu = container_of(dev, struct cpu, dev); + int cpuid = cpu->dev.id; + int prefer_idle, err; + + err = kstrtoint(strstrip((char *)buf), 0, &prefer_idle); + if (err) + return err; + + err = sched_set_cpu_prefer_idle(cpuid, prefer_idle); + if (err >= 0) + err = count; + + return err; +} + static DEVICE_ATTR(sched_mostly_idle_freq, 0664, show_sched_mostly_idle_freq, store_sched_mostly_idle_freq); static DEVICE_ATTR(sched_mostly_idle_load, 0664, show_sched_mostly_idle_load, store_sched_mostly_idle_load); static DEVICE_ATTR(sched_mostly_idle_nr_run, 0664, show_sched_mostly_idle_nr_run, store_sched_mostly_idle_nr_run); +static DEVICE_ATTR(sched_prefer_idle, 0664, + show_sched_prefer_idle, store_sched_prefer_idle); static struct attribute *hmp_sched_cpu_attrs[] = { &dev_attr_sched_mostly_idle_load.attr, &dev_attr_sched_mostly_idle_nr_run.attr, &dev_attr_sched_mostly_idle_freq.attr, + &dev_attr_sched_prefer_idle.attr, NULL }; diff --git a/include/linux/sched.h b/include/linux/sched.h index 0876b298c76e..d9879da4b777 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -2332,6 +2332,8 @@ sched_set_cpu_cstate(int cpu, int cstate, int wakeup_energy, int wakeup_latency) extern int sched_set_boost(int enable); extern int sched_set_init_task_load(struct task_struct *p, int init_load_pct); extern u32 sched_get_init_task_load(struct task_struct *p); +extern int sched_set_cpu_prefer_idle(int cpu, int prefer_idle); +extern int sched_get_cpu_prefer_idle(int cpu); extern int sched_set_cpu_mostly_idle_load(int cpu, int mostly_idle_pct); extern int sched_get_cpu_mostly_idle_load(int cpu); extern int sched_set_cpu_mostly_idle_nr_run(int cpu, int nr_run); diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h index 5437c599fd54..ce23dcd70b98 100644 --- a/include/linux/sched/sysctl.h +++ b/include/linux/sched/sysctl.h @@ -67,7 +67,6 @@ extern unsigned int sysctl_sched_small_task_pct; extern unsigned int sysctl_sched_upmigrate_pct; extern unsigned int sysctl_sched_downmigrate_pct; extern int sysctl_sched_upmigrate_min_nice; -extern unsigned int sysctl_sched_prefer_idle; extern unsigned int sysctl_sched_powerband_limit_pct; extern unsigned int sysctl_sched_boost; diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 332b920af31e..0ae0d5f71a51 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -9179,6 +9179,7 @@ void __init sched_init(void) rq->cur_irqload = 0; rq->avg_irqload = 0; rq->irqload_ts = 0; + rq->prefer_idle = 1; #ifdef CONFIG_SCHED_FREQ_INPUT rq->curr_runnable_sum = rq->prev_runnable_sum = 0; rq->old_busy_time = 0; diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 4260cb552d9f..7a8850cc15bf 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -2643,13 +2643,6 @@ static int __read_mostly sched_upmigrate_min_nice = 15; int __read_mostly sysctl_sched_upmigrate_min_nice = 15; /* - * Tunable to govern scheduler wakeup placement CPU selection - * preference. If set, the scheduler chooses to wake up a task - * on an idle CPU. - */ -unsigned int __read_mostly sysctl_sched_prefer_idle = 1; - -/* * Scheduler boost is a mechanism to temporarily place tasks on CPUs * with higher capacity than those where a task would have normally * ended up with their load characteristics. Any entity enabling @@ -2709,6 +2702,22 @@ int sched_set_init_task_load(struct task_struct *p, int init_load_pct) return 0; } +int sched_set_cpu_prefer_idle(int cpu, int prefer_idle) +{ + struct rq *rq = cpu_rq(cpu); + + rq->prefer_idle = !!prefer_idle; + + return 0; +} + +int sched_get_cpu_prefer_idle(int cpu) +{ + struct rq *rq = cpu_rq(cpu); + + return rq->prefer_idle; +} + int sched_set_cpu_mostly_idle_load(int cpu, int mostly_idle_pct) { struct rq *rq = cpu_rq(cpu); @@ -3257,20 +3266,24 @@ static int select_best_cpu(struct task_struct *p, int target, int reason, int small_task = is_small_task(p); int boost = sched_boost(); int cstate, min_cstate = INT_MAX; - int prefer_idle = reason ? 1 : sysctl_sched_prefer_idle; + int prefer_idle = -1; int curr_cpu = smp_processor_id(); int prefer_idle_override = 0; + if (reason) { + prefer_idle = 1; + prefer_idle_override = 1; + } + if (wake_to_idle(p)) { prefer_idle = 1; prefer_idle_override = 1; small_task = 0; } - trace_sched_task_load(p, small_task, boost, reason, sync, prefer_idle); - if (small_task && !boost) { best_cpu = best_small_task_cpu(p, sync); + prefer_idle = 0; /* For sched_task_load tracepoint */ goto done; } @@ -3301,6 +3314,10 @@ static int select_best_cpu(struct task_struct *p, int target, int reason, continue; } + /* Set prefer_idle based on the cpu where task will first fit */ + if (prefer_idle == -1) + prefer_idle = cpu_rq(i)->prefer_idle; + if (!eligible_cpu(p, i, sync)) continue; @@ -3327,6 +3344,8 @@ static int select_best_cpu(struct task_struct *p, int target, int reason, min_cstate = INT_MAX; min_cstate_cpu = -1; best_cpu = -1; + if (!prefer_idle_override) + prefer_idle = cpu_rq(i)->prefer_idle; } /* @@ -3382,7 +3401,7 @@ static int select_best_cpu(struct task_struct *p, int target, int reason, } } - if (min_cstate_cpu >= 0 && (prefer_idle || + if (min_cstate_cpu >= 0 && (prefer_idle > 0 || !(best_cpu >= 0 && mostly_idle_cpu_sync(best_cpu, sync)))) best_cpu = min_cstate_cpu; done: @@ -3401,6 +3420,13 @@ done: if (cpu_rq(best_cpu)->mostly_idle_freq && !prefer_idle_override) best_cpu = select_packing_target(p, best_cpu); + /* + * prefer_idle is initialized towards middle of function. Leave this + * tracepoint towards end to capture prefer_idle flag used for this + * instance of wakeup. + */ + trace_sched_task_load(p, small_task, boost, reason, sync, prefer_idle); + return best_cpu; } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 89b2f4dee627..baa75b4af635 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -658,6 +658,7 @@ struct rq { int capacity; int max_possible_capacity; u64 window_start; + int prefer_idle; u32 mostly_idle_load; int mostly_idle_nr_run; int mostly_idle_freq; diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 5e0354d6317c..16e465103020 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -415,13 +415,6 @@ static struct ctl_table kern_table[] = { .proc_handler = sched_hmp_proc_update_handler, }, { - .procname = "sched_prefer_idle", - .data = &sysctl_sched_prefer_idle, - .maxlen = sizeof(unsigned int), - .mode = 0644, - .proc_handler = proc_dointvec, - }, - { .procname = "sched_init_task_load", .data = &sysctl_sched_init_task_load_pct, .maxlen = sizeof(unsigned int), |