sched: window-stats: apply scaling to full elapsed windows

In the event that a full window (or multiple full windows) have
elapsed when updating a task's window-based stats, the runtime of
those windows needs to be scaled based on the CPU frequency. This
is currently missing, causing full windows to be accounted as having
elapsed at maximum frequency, erroneously inflating task demand.

Change-Id: I356b4279d44d4f39c8aea881c04327b70ed66183
Signed-off-by: Steve Muckle <smuckle@codeaurora.org>
Signed-off-by: Syed Rameez Mustafa <rameezmustafa@codeaurora.org>

1 files changed, 28 insertions, 19 deletions

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 3a869c46dcbf..500e948dd43a 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -1219,6 +1219,24 @@ static inline void move_window_start(struct rq *rq, u64 wallclock)
 	}
 }
 
+static inline u64 scale_exec_time(u64 delta, struct rq *rq)
+{
+	unsigned int cur_freq = rq->cur_freq;
+	int sf;
+
+	if (unlikely(cur_freq > max_possible_freq ||
+		     (cur_freq == rq->max_freq &&
+		      rq->max_freq < rq->max_possible_freq)))
+		cur_freq = rq->max_possible_freq;
+
+	delta = div64_u64(delta  * cur_freq, max_possible_freq);
+	sf = (rq->efficiency * 1024) / max_possible_efficiency;
+	delta *= sf;
+	delta >>= 10;
+
+	return delta;
+}
+
 void update_task_ravg(struct task_struct *p, struct rq *rq,
 				 int update_sum, u64 wallclock)
 {
@@ -1232,6 +1250,8 @@ void update_task_ravg(struct task_struct *p, struct rq *rq,
 	if (sched_use_pelt || !rq->window_start)
 		return;
 
+	lockdep_assert_held(&rq->lock);
+
 	move_window_start(rq, wallclock);
 	window_start = rq->window_start;
 
@@ -1244,7 +1264,7 @@ void update_task_ravg(struct task_struct *p, struct rq *rq,
 
 	do {
 		s64 delta = 0;
-		int n = 0;
+		int nr_full_windows = 0;
 		u64 now = wallclock;
 		u32 sum = 0;
 
@@ -1253,28 +1273,16 @@ void update_task_ravg(struct task_struct *p, struct rq *rq,
 		if (window_start > mark_start) {
 			delta = window_start - mark_start;
 
-			n = div64_u64(delta, window_size);
-			window_start -= n * window_size;
+			nr_full_windows = div64_u64(delta, window_size);
+			window_start -= nr_full_windows * window_size;
 			now = window_start;
 			new_window = 1;
 		}
 
 		if (update_sum) {
-			unsigned int cur_freq = rq->cur_freq;
-			int sf;
-
 			delta = now - mark_start;
+			delta = scale_exec_time(delta, rq);
 
-			if (unlikely(cur_freq > max_possible_freq ||
-				     (cur_freq == rq->max_freq &&
-				      rq->max_freq < rq->max_possible_freq)))
-				cur_freq = rq->max_possible_freq;
-
-			delta = div64_u64(delta  * cur_freq,
-							max_possible_freq);
-			sf = (rq->efficiency * 1024) / max_possible_efficiency;
-			delta *= sf;
-			delta >>= 10;
 			p->ravg.sum += delta;
 			if (unlikely(p->ravg.sum > window_size))
 				p->ravg.sum = window_size;
@@ -1288,11 +1296,12 @@ void update_task_ravg(struct task_struct *p, struct rq *rq,
 
 		update_history(rq, p, p->ravg.sum, 1);
 
-		if (n) {
-			window_start += n * window_size;
+		if (nr_full_windows) {
+			window_start += nr_full_windows * window_size;
 			if (update_sum)
 				sum = window_size;
-			update_history(rq, p, sum, n);
+			sum = scale_exec_time(sum, rq);
+			update_history(rq, p, sum, nr_full_windows);
 
 			/*
 			 * We will always shift curr_contrib into