sched: Provide knob to prefer mostly_idle over idle cpus

sysctl_sched_prefer_idle lets the scheduler bias selection of
idle cpus over mostly idle cpus for tasks. This knob could be
useful to control balance between power and performance.

Change-Id: Ide6eef684ef94ac8b9927f53c220ccf94976fe67
Signed-off-by: Srivatsa Vaddagiri <vatsa@codeaurora.org>

2 files changed, 61 insertions, 21 deletions

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index bd4f9fc66950..0c9533a2854a 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -2634,6 +2634,13 @@ unsigned int __read_mostly sysctl_sched_downmigrate_pct = 60;
 int __read_mostly sysctl_sched_upmigrate_min_nice = 15;
 
 /*
+ * Tunable to govern scheduler wakeup placement CPU selection
+ * preference. If set, the scheduler chooses to wake up a task
+ * on an idle CPU.
+ */
+unsigned int __read_mostly sysctl_sched_prefer_idle = 1;
+
+/*
  * Scheduler boost is a mechanism to temporarily place tasks on CPUs
  * with higher capacity than those where a task would have normally
  * ended up with their load characteristics. Any entity enabling
@@ -3166,13 +3173,15 @@ static int select_packing_target(struct task_struct *p, int best_cpu)
 /* return cheapest cpu that can fit this task */
 static int select_best_cpu(struct task_struct *p, int target, int reason)
 {
-	int i, best_cpu = -1, fallback_idle_cpu = -1;
+	int i, best_cpu = -1, fallback_idle_cpu = -1, min_cstate_cpu = -1;
 	int prev_cpu = task_cpu(p);
 	int cpu_cost, min_cost = INT_MAX;
+	int min_idle_cost = INT_MAX, min_busy_cost = INT_MAX;
 	u64 load, min_load = ULLONG_MAX, min_fallback_load = ULLONG_MAX;
 	int small_task = is_small_task(p);
 	int boost = sched_boost();
 	int cstate, min_cstate = INT_MAX;
+	int prefer_idle = reason ? 1 : sysctl_sched_prefer_idle;
 
 	trace_sched_task_load(p, small_task, boost, reason);
 
@@ -3225,43 +3234,67 @@ static int select_best_cpu(struct task_struct *p, int target, int reason)
 		 * overrides load and C-state.
 		 */
 		if (power_delta_exceeded(cpu_cost, min_cost)) {
-			if (cpu_cost < min_cost) {
-				min_load = load;
-				min_cost = cpu_cost;
+			if (cpu_cost > min_cost)
+				continue;
+
+			min_cost = cpu_cost;
+			min_load = ULLONG_MAX;
+			min_cstate = INT_MAX;
+			min_cstate_cpu = -1;
+			best_cpu = -1;
+		}
+
+		/*
+		 * Partition CPUs based on whether they are completely idle
+		 * or not. For completely idle CPUs we choose the one in
+		 * the lowest C-state and then break ties with power cost
+		 */
+		if (idle_cpu(i)) {
+			if (cstate > min_cstate)
+				continue;
+
+			if (cstate < min_cstate) {
+				min_idle_cost = cpu_cost;
 				min_cstate = cstate;
-				best_cpu = i;
+				min_cstate_cpu = i;
+				continue;
+			}
+
+			if (cpu_cost < min_idle_cost) {
+				min_idle_cost = cpu_cost;
+				min_cstate_cpu = i;
 			}
 			continue;
 		}
 
-		/* After power band, load is prioritized next. */
+		/*
+		 * For CPUs that are not completely idle, pick one with the
+		 * lowest load and break ties with power cost
+		 */
+		if (load > min_load)
+			continue;
+
 		if (load < min_load) {
 			min_load = load;
-			min_cost = cpu_cost;
-			min_cstate = cstate;
+			min_busy_cost = cpu_cost;
 			best_cpu = i;
 			continue;
 		}
-		if (load > min_load)
-			continue;
 
 		/*
 		 * The load is equal to the previous selected CPU.
-		 * This will most often occur when deciding between
-		 * idle CPUs. Power cost is prioritized after load,
-		 * followed by cstate.
+		 * This is rare but when it does happen opt for the
+		 * more power efficient CPU option.
 		 */
-		if (cpu_cost < min_cost) {
-			min_cost = cpu_cost;
-			min_cstate = cstate;
-			best_cpu = i;
-			continue;
-		}
-		if (cpu_cost == min_cost && cstate < min_cstate) {
-			min_cstate = cstate;
+		if (cpu_cost < min_busy_cost) {
+			min_busy_cost = cpu_cost;
 			best_cpu = i;
 		}
 	}
+
+	if (min_cstate_cpu >= 0 && (prefer_idle ||
+			!(best_cpu >= 0 && mostly_idle_cpu(best_cpu))))
+		best_cpu = min_cstate_cpu;
 done:
 	if (best_cpu < 0) {
 		if (unlikely(fallback_idle_cpu < 0))
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index 738b154269ea..1465fb869657 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -408,6 +408,13 @@ static struct ctl_table kern_table[] = {
 		.proc_handler	= proc_dointvec,
 	},
 	{
+		.procname	= "sched_prefer_idle",
+		.data		= &sysctl_sched_prefer_idle,
+		.maxlen		= sizeof(unsigned int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec,
+	},
+	{
 		.procname	= "sched_init_task_load",
 		.data		= &sysctl_sched_init_task_load_pct,
 		.maxlen		= sizeof(unsigned int),