diff options
Diffstat (limited to 'kernel/sched/hmp.c')
| -rw-r--r-- | kernel/sched/hmp.c | 88 |
1 files changed, 72 insertions, 16 deletions
diff --git a/kernel/sched/hmp.c b/kernel/sched/hmp.c index 6304c5030137..a8bf39c6d7d7 100644 --- a/kernel/sched/hmp.c +++ b/kernel/sched/hmp.c @@ -74,11 +74,6 @@ inline void clear_ed_task(struct task_struct *p, struct rq *rq) rq->ed_task = NULL; } -inline void set_task_last_wake(struct task_struct *p, u64 wallclock) -{ - p->last_wake_ts = wallclock; -} - inline void set_task_last_switch_out(struct task_struct *p, u64 wallclock) { p->last_switch_out_ts = wallclock; @@ -961,6 +956,16 @@ sched_long_cpu_selection_threshold = 100 * NSEC_PER_MSEC; unsigned int __read_mostly sysctl_sched_restrict_cluster_spill; +/* + * Scheduler tries to avoid waking up idle CPUs for tasks running + * in short bursts. If the task average burst is less than + * sysctl_sched_short_burst nanoseconds and it sleeps on an average + * for more than sysctl_sched_short_sleep nanoseconds, then the + * task is eligible for packing. + */ +unsigned int __read_mostly sysctl_sched_short_burst; +unsigned int __read_mostly sysctl_sched_short_sleep = 1 * NSEC_PER_MSEC; + static void _update_up_down_migrate(unsigned int *up_migrate, unsigned int *down_migrate) { @@ -1552,6 +1557,15 @@ void init_new_task_load(struct task_struct *p, bool idle_task) INIT_LIST_HEAD(&p->grp_list); memset(&p->ravg, 0, sizeof(struct ravg)); p->cpu_cycles = 0; + p->ravg.curr_burst = 0; + /* + * Initialize the avg_burst to twice the threshold, so that + * a task would not be classified as short burst right away + * after fork. It takes at least 6 sleep-wakeup cycles for + * the avg_burst to go below the threshold. + */ + p->ravg.avg_burst = 2 * (u64)sysctl_sched_short_burst; + p->ravg.avg_sleep_time = 0; p->ravg.curr_window_cpu = kcalloc(nr_cpu_ids, sizeof(u32), GFP_KERNEL); p->ravg.prev_window_cpu = kcalloc(nr_cpu_ids, sizeof(u32), GFP_KERNEL); @@ -2738,12 +2752,14 @@ done: trace_sched_update_history(rq, p, runtime, samples, event); } -static void add_to_task_demand(struct rq *rq, struct task_struct *p, u64 delta) +static u64 add_to_task_demand(struct rq *rq, struct task_struct *p, u64 delta) { delta = scale_exec_time(delta, rq); p->ravg.sum += delta; if (unlikely(p->ravg.sum > sched_ravg_window)) p->ravg.sum = sched_ravg_window; + + return delta; } /* @@ -2796,13 +2812,14 @@ static void add_to_task_demand(struct rq *rq, struct task_struct *p, u64 delta) * IMPORTANT : Leave p->ravg.mark_start unchanged, as update_cpu_busy_time() * depends on it! */ -static void update_task_demand(struct task_struct *p, struct rq *rq, +static u64 update_task_demand(struct task_struct *p, struct rq *rq, int event, u64 wallclock) { u64 mark_start = p->ravg.mark_start; u64 delta, window_start = rq->window_start; int new_window, nr_full_windows; u32 window_size = sched_ravg_window; + u64 runtime; new_window = mark_start < window_start; if (!account_busy_for_task_demand(p, event)) { @@ -2816,7 +2833,7 @@ static void update_task_demand(struct task_struct *p, struct rq *rq, * it is not necessary to account those. */ update_history(rq, p, p->ravg.sum, 1, event); - return; + return 0; } if (!new_window) { @@ -2824,8 +2841,7 @@ static void update_task_demand(struct task_struct *p, struct rq *rq, * The simple case - busy time contained within the existing * window. */ - add_to_task_demand(rq, p, wallclock - mark_start); - return; + return add_to_task_demand(rq, p, wallclock - mark_start); } /* @@ -2837,13 +2853,16 @@ static void update_task_demand(struct task_struct *p, struct rq *rq, window_start -= (u64)nr_full_windows * (u64)window_size; /* Process (window_start - mark_start) first */ - add_to_task_demand(rq, p, window_start - mark_start); + runtime = add_to_task_demand(rq, p, window_start - mark_start); /* Push new sample(s) into task's demand history */ update_history(rq, p, p->ravg.sum, 1, event); - if (nr_full_windows) - update_history(rq, p, scale_exec_time(window_size, rq), - nr_full_windows, event); + if (nr_full_windows) { + u64 scaled_window = scale_exec_time(window_size, rq); + + update_history(rq, p, scaled_window, nr_full_windows, event); + runtime += nr_full_windows * scaled_window; + } /* * Roll window_start back to current to process any remainder @@ -2853,13 +2872,31 @@ static void update_task_demand(struct task_struct *p, struct rq *rq, /* Process (wallclock - window_start) next */ mark_start = window_start; - add_to_task_demand(rq, p, wallclock - mark_start); + runtime += add_to_task_demand(rq, p, wallclock - mark_start); + + return runtime; +} + +static inline void +update_task_burst(struct task_struct *p, struct rq *rq, int event, int runtime) +{ + /* + * update_task_demand() has checks for idle task and + * exit task. The runtime may include the wait time, + * so update the burst only for the cases where the + * task is running. + */ + if (event == PUT_PREV_TASK || (event == TASK_UPDATE && + rq->curr == p)) + p->ravg.curr_burst += runtime; } /* Reflect task activity on its demand and cpu's busy time statistics */ void update_task_ravg(struct task_struct *p, struct rq *rq, int event, u64 wallclock, u64 irqtime) { + u64 runtime; + if (!rq->window_start || sched_disable_window_stats || p->ravg.mark_start == wallclock) return; @@ -2874,7 +2911,9 @@ void update_task_ravg(struct task_struct *p, struct rq *rq, int event, } update_task_rq_cpu_cycles(p, rq, event, wallclock, irqtime); - update_task_demand(p, rq, event, wallclock); + runtime = update_task_demand(p, rq, event, wallclock); + if (runtime) + update_task_burst(p, rq, event, runtime); update_cpu_busy_time(p, rq, event, wallclock, irqtime); update_task_pred_demand(rq, p, event); done: @@ -2960,6 +2999,8 @@ void reset_task_stats(struct task_struct *p) p->ravg.curr_window_cpu = curr_window_ptr; p->ravg.prev_window_cpu = prev_window_ptr; + p->ravg.avg_burst = 2 * (u64)sysctl_sched_short_burst; + /* Retain EXITING_TASK marker */ p->ravg.sum_history[0] = sum; } @@ -3029,6 +3070,7 @@ const char *sched_window_reset_reasons[] = { "WINDOW_CHANGE", "POLICY_CHANGE", "HIST_SIZE_CHANGE", + "FREQ_AGGREGATE_CHANGE", }; /* Called with IRQs enabled */ @@ -4462,6 +4504,20 @@ bool early_detection_notify(struct rq *rq, u64 wallclock) return 0; } +void update_avg_burst(struct task_struct *p) +{ + update_avg(&p->ravg.avg_burst, p->ravg.curr_burst); + p->ravg.curr_burst = 0; +} + +void note_task_waking(struct task_struct *p, u64 wallclock) +{ + u64 sleep_time = wallclock - p->last_switch_out_ts; + + p->last_wake_ts = wallclock; + update_avg(&p->ravg.avg_sleep_time, sleep_time); +} + #ifdef CONFIG_CGROUP_SCHED u64 cpu_upmigrate_discourage_read_u64(struct cgroup_subsys_state *css, struct cftype *cft) |