sched/fair: Introduce C-state aware task placement for small tasks

Small tasks execute for small durations. This means that the power
cost of taking CPUs out of a low power mode outweigh any performance
advantage of using an idle core or power advantage of using the most
power efficient CPU. Introduce C-state aware task placement for small
tasks. This requires a two pass approach where we first determine the
most power effecient CPU and establish a band of CPUs offering a
similar power cost for the task. The order of preference then is as
follows:

1) Any mostly idle CPU in active C-state in the same power band.
2) A CPU with the shallowest C-state in the same power band.
3) A CPU with the least load in the same power band.
4) Lowest power CPU in a higher power band.

The patch also modifies the definition of a small task. Small tasks
are now determined relative to minimum capacity CPUs in the system
and not the task CPU.

Change-Id: Ia09840a5972881cad7ba7bea8fe34c45f909725e
Signed-off-by: Syed Rameez Mustafa <rameezmustafa@codeaurora.org>

3 files changed, 88 insertions, 11 deletions

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 411e0050a6ae..e5c0c7da2a36 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -1118,6 +1118,7 @@ unsigned int min_max_freq = 1;
 
 unsigned int max_capacity = 1024; /* max(rq->capacity) */
 unsigned int min_capacity = 1024; /* min(rq->capacity) */
+unsigned int max_load_scale_factor = 1024; /* max(rq->load_scale_factor) */
 
 static unsigned int sync_cpu;
 static u64 sched_init_jiffy;
@@ -1540,16 +1541,21 @@ static void update_min_max_capacity(void)
 {
 	int i;
 	int max = 0, min = INT_MAX;
+	int max_lsf = 0;
 
 	for_each_possible_cpu(i) {
 		if (cpu_rq(i)->capacity > max)
 			max = cpu_rq(i)->capacity;
 		if (cpu_rq(i)->capacity < min)
 			min = cpu_rq(i)->capacity;
+
+		if (cpu_rq(i)->load_scale_factor > max_lsf)
+			max_lsf = cpu_rq(i)->load_scale_factor;
 	}
 
 	max_capacity = max;
 	min_capacity = min;
+	max_load_scale_factor = max_lsf;
 }
 
 /*
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index ef971c953cf2..b24f68f7204c 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -2717,13 +2717,12 @@ static inline int is_big_task(struct task_struct *p)
 	return load > sched_upmigrate;
 }
 
-/* Is a task "small" on its current cpu */
+/* Is a task "small" on the minimum capacity CPU */
 static inline int is_small_task(struct task_struct *p)
 {
-	unsigned int load = task_load(p);
-
-	load = scale_task_load(load, task_cpu(p));
-
+	u64 load = task_load(p);
+	load *= (u64)max_load_scale_factor;
+	load /= 1024;
 	return load < sched_small_task;
 }
 
@@ -2930,6 +2929,79 @@ static unsigned int power_cost(struct task_struct *p, int cpu)
 	return power_cost_at_freq(cpu, demand);
 }
 
+static int best_small_task_cpu(struct task_struct *p)
+{
+	int best_busy_cpu = -1, best_fallback_cpu = -1;
+	int min_cost_cpu = -1, min_cstate_cpu = -1;
+	int min_busy_load = INT_MAX;
+	int min_cstate = INT_MAX;
+	int min_fallback_cpu_cost = INT_MAX;
+	int min_cost = INT_MAX;
+	int i, load, cstate, cpu_cost;
+	int cost_list[nr_cpu_ids];
+	struct cpumask search_cpus;
+
+	cpumask_and(&search_cpus,  tsk_cpus_allowed(p), cpu_online_mask);
+
+	/* Take a first pass to find the lowest power cost CPU. This
+	   will avoid a potential O(n^2) search */
+	for_each_cpu(i, &search_cpus) {
+		cpu_cost = power_cost(p, i);
+		if (cpu_cost < min_cost) {
+			min_cost = cpu_cost;
+			min_cost_cpu = i;
+		}
+
+		cost_list[i] = cpu_cost;
+	}
+
+	/* Optimization to steer task towards the minimum power
+	   cost CPU. The tradeoff is that we may have to check
+	   the same information again in pass 2 */
+	if (!cpu_rq(min_cost_cpu)->cstate && mostly_idle_cpu(min_cost_cpu))
+		return min_cost_cpu;
+
+	for_each_cpu(i, &search_cpus) {
+		struct rq *rq = cpu_rq(i);
+		cstate = rq->cstate;
+
+		if (power_delta_exceeded(cost_list[i], min_cost)) {
+			if (cost_list[i] < min_fallback_cpu_cost) {
+				best_fallback_cpu = i;
+				min_fallback_cpu_cost = cost_list[i];
+			}
+			continue;
+		}
+
+		if (cstate) {
+			if (cstate < min_cstate) {
+				min_cstate_cpu = i;
+				min_cstate = cstate;
+			}
+			continue;
+		}
+
+		if (mostly_idle_cpu(i))
+			return i;
+
+		load = cpu_load(i);
+		if (!spill_threshold_crossed(p, rq, i)) {
+			if (load < min_busy_load) {
+				min_busy_load = load;
+				best_busy_cpu = i;
+			}
+		}
+	}
+
+	if (min_cstate_cpu != -1)
+		return min_cstate_cpu;
+
+	if (best_busy_cpu != -1)
+		return best_busy_cpu;
+
+	return best_fallback_cpu;
+}
+
 /* return cheapest cpu that can fit this task */
 static int select_best_cpu(struct task_struct *p, int target)
 {
@@ -2941,12 +3013,9 @@ static int select_best_cpu(struct task_struct *p, int target)
 
 	trace_sched_task_load(p);
 
-	/* provide bias for prev_cpu */
-	if (!small_task && mostly_idle_cpu(prev_cpu) &&
-	    task_will_fit(p, prev_cpu)) {
-		best_cpu = prev_cpu;
-		min_cost = power_cost(p, prev_cpu);
-		min_load = cpu_load(prev_cpu);
+	if (small_task) {
+		best_cpu = best_small_task_cpu(p);
+		goto done;
 	}
 
 	/* Todo : Optimize this loop */
@@ -2989,6 +3058,7 @@ static int select_best_cpu(struct task_struct *p, int target)
 		}
 	}
 
+done:
 	if (best_cpu < 0) {
 		if (unlikely(fallback_idle_cpu < 0))
 			best_cpu = prev_cpu;
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index aa7a2c59dd60..5296014cc4c1 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -944,6 +944,7 @@ extern unsigned int max_possible_efficiency;
 extern unsigned int min_possible_efficiency;
 extern unsigned int max_capacity;
 extern unsigned int min_capacity;
+extern unsigned int max_load_scale_factor;
 extern unsigned long capacity_scale_cpu_efficiency(int cpu);
 extern unsigned long capacity_scale_cpu_freq(int cpu);
 extern unsigned int sched_mostly_idle_load;