sched: Make RAVG_HIST_SIZE tunable

Make RAVG_HIST_SIZE available from /proc/sys/kernel/sched_ravg_hist_size
to allow tuning of the size of the history that is used in computation
of task demand.

CRs-fixed: 706138
Change-Id: Id54c1e4b6e974a62d787070a0af1b4e8ce3b4be6
Signed-off-by: Olav Haugan <ohaugan@codeaurora.org>
[joonwoop@codeaurora.org: fixed minor conflict in sysctl.h]
Signed-off-by: Joonwoo Park <joonwoop@codeaurora.org>

4 files changed, 82 insertions, 17 deletions

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 4946d5acebca..ec1b7a036cc2 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -1162,6 +1162,17 @@ static inline void clear_hmp_request(int cpu) { }
 
 #if defined(CONFIG_SCHED_FREQ_INPUT) || defined(CONFIG_SCHED_HMP)
 
+__read_mostly unsigned int sysctl_sched_ravg_hist_size = 5;
+
+/*
+ * copy of sysctl_sched_ravg_hist_size. Required for atomically
+ * changing the ravg history size (see sched_ravg_hist_size_update_handler()
+ * for details).
+ *
+ * Initialize both to same value!!
+ */
+static __read_mostly unsigned int sched_ravg_hist_size = 5;
+
 /* Window size (in ns) */
 __read_mostly unsigned int sched_ravg_window = 10000000;
 
@@ -1284,7 +1295,7 @@ update_history(struct rq *rq, struct task_struct *p, u32 runtime, int samples,
 		return;
 
 	if (!new_window) {
-		for (ridx = 0; ridx < RAVG_HIST_SIZE - 1; ++ridx) {
+		for (ridx = 0; ridx < sched_ravg_hist_size - 1; ++ridx) {
 			sum += hist[ridx];
 			if (hist[ridx] > max)
 				max = hist[ridx];
@@ -1296,20 +1307,24 @@ update_history(struct rq *rq, struct task_struct *p, u32 runtime, int samples,
 	}
 
 	/* Push new 'runtime' value onto stack */
-	widx = RAVG_HIST_SIZE - 1;
+	widx = RAVG_HIST_SIZE_MAX - 1;
 	ridx = widx - samples;
 	for (; ridx >= 0; --widx, --ridx) {
 		hist[widx] = hist[ridx];
-		sum += hist[widx];
-		if (hist[widx] > max)
-			max = hist[widx];
+		if (widx < sched_ravg_hist_size) {
+			sum += hist[widx];
+			if (hist[widx] > max)
+				max = hist[widx];
+		}
 	}
 
-	for (widx = 0; widx < samples && widx < RAVG_HIST_SIZE; widx++) {
+	for (widx = 0; widx < samples && widx < RAVG_HIST_SIZE_MAX; widx++) {
 		hist[widx] = runtime;
-		sum += hist[widx];
-		if (hist[widx] > max)
-			max = hist[widx];
+		if (widx < sched_ravg_hist_size) {
+			sum += hist[widx];
+			if (hist[widx] > max)
+				max = hist[widx];
+		}
 	}
 
 	p->ravg.sum = 0;
@@ -1321,7 +1336,7 @@ update_history(struct rq *rq, struct task_struct *p, u32 runtime, int samples,
 	}
 
 compute_demand:
-	avg = div64_u64(sum, RAVG_HIST_SIZE);
+	avg = div64_u64(sum, sched_ravg_hist_size);
 
 	if (sched_window_stats_policy == WINDOW_STATS_USE_RECENT)
 		demand = runtime;
@@ -1689,7 +1704,8 @@ unsigned long sched_get_busy(int cpu)
 
 /* Called with IRQs disabled */
 void reset_all_window_stats(u64 window_start, unsigned int window_size,
-				 int policy, int acct_wait_time)
+				 int policy, int acct_wait_time,
+				 unsigned int ravg_hist_size)
 {
 	int cpu;
 	u64 wallclock;
@@ -1716,7 +1732,7 @@ void reset_all_window_stats(u64 window_start, unsigned int window_size,
 		p->ravg.partial_demand = 0;
 		p->ravg.prev_window = 0;
 		p->ravg.flags = 0;
-		for (i = 0; i < RAVG_HIST_SIZE; ++i)
+		for (i = 0; i < RAVG_HIST_SIZE_MAX; ++i)
 			p->ravg.sum_history[i] = 0;
 		p->ravg.mark_start = wallclock;
 	}  while_each_thread(g, p);
@@ -1738,6 +1754,9 @@ void reset_all_window_stats(u64 window_start, unsigned int window_size,
 	if (acct_wait_time >= 0)
 		sched_account_wait_time = acct_wait_time;
 
+	if (ravg_hist_size > 0)
+		sched_ravg_hist_size = ravg_hist_size;
+
 	for_each_online_cpu(cpu) {
 		struct rq *rq = cpu_rq(cpu);
 		raw_spin_unlock(&rq->lock);
@@ -1775,7 +1794,7 @@ int sched_set_window(u64 window_start, unsigned int window_size)
 
 	BUG_ON(sched_clock() < ws);
 
-	reset_all_window_stats(ws, window_size, -1, -1);
+	reset_all_window_stats(ws, window_size, -1, -1, 0);
 
 	local_irq_restore(flags);
 
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 7af7e52edbb6..00d9bc11eb09 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -3224,7 +3224,45 @@ int sched_acct_wait_time_update_handler(struct ctl_table *table, int write,
 
 	local_irq_save(flags);
 
-	reset_all_window_stats(0, 0, -1, sysctl_sched_account_wait_time);
+	reset_all_window_stats(0, 0, -1, sysctl_sched_account_wait_time, 0);
+
+	local_irq_restore(flags);
+
+done:
+	mutex_unlock(&policy_mutex);
+
+	return ret;
+}
+
+int sched_ravg_hist_size_update_handler(struct ctl_table *table, int write,
+		void __user *buffer, size_t *lenp,
+		loff_t *ppos)
+{
+	int ret;
+	unsigned int *data = (unsigned int *)table->data;
+	unsigned int old_val;
+	unsigned long flags;
+
+	if (!sched_enable_hmp)
+		return -EINVAL;
+
+	mutex_lock(&policy_mutex);
+
+	old_val = *data;
+
+	ret = proc_dointvec(table, write, buffer, lenp, ppos);
+	if (ret || !write || (write && (old_val == *data)))
+		goto done;
+
+	if (*data > RAVG_HIST_SIZE_MAX || *data < 1) {
+		*data = old_val;
+		ret = -EINVAL;
+		goto done;
+	}
+
+	local_irq_save(flags);
+
+	reset_all_window_stats(0, 0, -1, -1, sysctl_sched_ravg_hist_size);
 
 	local_irq_restore(flags);
 
@@ -3256,7 +3294,7 @@ int sched_window_stats_policy_update_handler(struct ctl_table *table, int write,
 
 	local_irq_save(flags);
 
-	reset_all_window_stats(0, 0, sysctl_sched_window_stats_policy, -1);
+	reset_all_window_stats(0, 0, sysctl_sched_window_stats_policy, -1, 0);
 
 	local_irq_restore(flags);
 
@@ -3554,7 +3592,7 @@ void init_new_task_load(struct task_struct *p)
 	p->ravg.sum		= 0;
 	p->ravg.flags = 0;
 
-	for (i = 0; i < RAVG_HIST_SIZE; ++i)
+	for (i = 0; i < RAVG_HIST_SIZE_MAX; ++i)
 		p->ravg.sum_history[i] = sched_init_task_load_windows;
 	p->se.avg.runnable_avg_sum_scaled = sched_init_task_load_pelt;
 	p->ravg.demand = sched_init_task_load_windows;
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 1d5a78d7a038..dd941c0c3d5f 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -1061,7 +1061,8 @@ extern void dec_nr_big_small_task(struct rq *rq, struct task_struct *p);
 extern void set_hmp_defaults(void);
 extern unsigned int power_cost_at_freq(int cpu, unsigned int freq);
 extern void reset_all_window_stats(u64 window_start, unsigned int window_size,
-				 int policy, int acct_wait_time);
+				 int policy, int acct_wait_time,
+				 unsigned int ravg_hist_size);
 extern void boost_kick(int cpu);
 
 #else /* CONFIG_SCHED_HMP */
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index cbdeb753f1c4..e67ec03d9a97 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -317,6 +317,13 @@ static struct ctl_table kern_table[] = {
 		.proc_handler   = sched_acct_wait_time_update_handler,
 	},
 	{
+		.procname       = "sched_ravg_hist_size",
+		.data           = &sysctl_sched_ravg_hist_size,
+		.maxlen         = sizeof(unsigned int),
+		.mode           = 0644,
+		.proc_handler   = sched_ravg_hist_size_update_handler,
+	},
+	{
 		.procname       = "sched_window_stats_policy",
 		.data           = &sysctl_sched_window_stats_policy,
 		.maxlen         = sizeof(unsigned int),