summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--drivers/cpufreq/cpu-boost.c43
-rw-r--r--include/linux/sched.h5
-rw-r--r--kernel/sched/core.c28
-rw-r--r--kernel/sched/fair.c35
-rw-r--r--kernel/sched/sched.h1
5 files changed, 90 insertions, 22 deletions
diff --git a/drivers/cpufreq/cpu-boost.c b/drivers/cpufreq/cpu-boost.c
index 5650bfa83ae2..5a934122277b 100644
--- a/drivers/cpufreq/cpu-boost.c
+++ b/drivers/cpufreq/cpu-boost.c
@@ -37,6 +37,7 @@ struct cpu_sync {
int src_cpu;
unsigned int boost_min;
unsigned int input_boost_min;
+ unsigned int task_load;
};
static DEFINE_PER_CPU(struct cpu_sync, sync_info);
@@ -56,6 +57,12 @@ module_param(input_boost_freq, uint, 0644);
static unsigned int input_boost_ms = 40;
module_param(input_boost_ms, uint, 0644);
+static unsigned int migration_load_threshold = 15;
+module_param(migration_load_threshold, uint, 0644);
+
+static bool load_based_syncs;
+module_param(load_based_syncs, bool, 0644);
+
static u64 last_input_time;
#define MIN_INPUT_INTERVAL (150 * USEC_PER_MSEC)
@@ -139,6 +146,7 @@ static int boost_mig_sync_thread(void *data)
struct cpufreq_policy dest_policy;
struct cpufreq_policy src_policy;
unsigned long flags;
+ unsigned int req_freq;
while (1) {
wait_event(s->sync_wq, s->pending || kthread_should_stop());
@@ -159,17 +167,20 @@ static int boost_mig_sync_thread(void *data)
if (ret)
continue;
- if (src_policy.cur == src_policy.cpuinfo.min_freq) {
- pr_debug("No sync. Source CPU%d@%dKHz at min freq\n",
- src_cpu, src_policy.cur);
+ req_freq = max((dest_policy.max * s->task_load) / 100,
+ src_policy.cur);
+
+ if (req_freq <= dest_policy.cpuinfo.min_freq) {
+ pr_debug("No sync. Sync Freq:%u\n", req_freq);
continue;
}
- cancel_delayed_work_sync(&s->boost_rem);
if (sync_threshold)
- s->boost_min = min(sync_threshold, src_policy.cur);
- else
- s->boost_min = src_policy.cur;
+ req_freq = min(sync_threshold, req_freq);
+
+ cancel_delayed_work_sync(&s->boost_rem);
+
+ s->boost_min = req_freq;
/* Force policy re-evaluation to trigger adjust notifier. */
get_online_cpus();
@@ -187,10 +198,19 @@ static int boost_mig_sync_thread(void *data)
}
static int boost_migration_notify(struct notifier_block *nb,
- unsigned long dest_cpu, void *arg)
+ unsigned long unused, void *arg)
{
+ struct migration_notify_data *mnd = arg;
unsigned long flags;
- struct cpu_sync *s = &per_cpu(sync_info, dest_cpu);
+ struct cpu_sync *s = &per_cpu(sync_info, mnd->dest_cpu);
+
+ if (load_based_syncs && (mnd->load <= migration_load_threshold))
+ return NOTIFY_OK;
+
+ if (load_based_syncs && ((mnd->load < 0) || (mnd->load > 100))) {
+ pr_err("cpu-boost:Invalid load: %d\n", mnd->load);
+ return NOTIFY_OK;
+ }
if (!boost_ms)
return NOTIFY_OK;
@@ -199,10 +219,11 @@ static int boost_migration_notify(struct notifier_block *nb,
if (s->thread == current)
return NOTIFY_OK;
- pr_debug("Migration: CPU%d --> CPU%d\n", (int) arg, (int) dest_cpu);
+ pr_debug("Migration: CPU%d --> CPU%d\n", mnd->src_cpu, mnd->dest_cpu);
spin_lock_irqsave(&s->lock, flags);
s->pending = true;
- s->src_cpu = (int) arg;
+ s->src_cpu = mnd->src_cpu;
+ s->task_load = load_based_syncs ? mnd->load : 0;
spin_unlock_irqrestore(&s->lock, flags);
wake_up(&s->sync_wq);
diff --git a/include/linux/sched.h b/include/linux/sched.h
index aa64a29d03be..0a664c44ee28 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -3158,6 +3158,11 @@ static inline void set_task_cpu(struct task_struct *p, unsigned int cpu)
#endif /* CONFIG_SMP */
extern struct atomic_notifier_head migration_notifier_head;
+struct migration_notify_data {
+ int src_cpu;
+ int dest_cpu;
+ int load;
+};
extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask);
extern long sched_getaffinity(pid_t pid, struct cpumask *mask);
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 3059a938045f..1d71f326bc4e 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -1145,9 +1145,19 @@ static struct rq *__migrate_task(struct rq *rq, struct task_struct *p, int dest_
src_cpu = cpu_of(rq);
rq = move_queued_task(rq, p, dest_cpu);
- if (task_notify_on_migrate(p))
+ if (task_notify_on_migrate(p)) {
+ struct migration_notify_data mnd;
+
+ mnd.src_cpu = src_cpu;
+ mnd.dest_cpu = dest_cpu;
+ if (sysctl_sched_ravg_window)
+ mnd.load = div64_u64((u64)p->se.ravg.demand * 100,
+ (u64)(sysctl_sched_ravg_window));
+ else
+ mnd.load = 0;
atomic_notifier_call_chain(&migration_notifier_head,
- dest_cpu, (void *)(long)src_cpu);
+ 0, (void *)&mnd);
+ }
return rq;
}
@@ -2146,9 +2156,19 @@ stat:
out:
raw_spin_unlock_irqrestore(&p->pi_lock, flags);
- if (src_cpu != cpu && task_notify_on_migrate(p))
+ if (src_cpu != cpu && task_notify_on_migrate(p)) {
+ struct migration_notify_data mnd;
+
+ mnd.src_cpu = src_cpu;
+ mnd.dest_cpu = cpu;
+ if (sysctl_sched_ravg_window)
+ mnd.load = div64_u64((u64)p->se.ravg.demand * 100,
+ (u64)(sysctl_sched_ravg_window));
+ else
+ mnd.load = 0;
atomic_notifier_call_chain(&migration_notifier_head,
- cpu, (void *)(long)src_cpu);
+ 0, (void *)&mnd);
+ }
return success;
}
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 917d0d4eaa71..cbf3d3697322 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -5746,7 +5746,7 @@ static void detach_task(struct task_struct *p, struct lb_env *env)
*
* Returns a task if successful and NULL otherwise.
*/
-static struct task_struct *detach_one_task(struct lb_env *env)
+static struct task_struct *detach_one_task(struct lb_env *env, int *total_run_moved)
{
struct task_struct *p, *n;
@@ -5765,6 +5765,11 @@ static struct task_struct *detach_one_task(struct lb_env *env)
* inside detach_tasks().
*/
schedstat_inc(env->sd, lb_gained[env->idle]);
+
+ if (sysctl_sched_ravg_window)
+ *total_run_moved += div64_u64((u64)p->se.ravg.demand *
+ 100, (u64)(sysctl_sched_ravg_window));
+
return p;
}
return NULL;
@@ -5778,7 +5783,7 @@ static const unsigned int sched_nr_migrate_break = 32;
*
* Returns number of detached tasks if successful and 0 otherwise.
*/
-static int detach_tasks(struct lb_env *env)
+static int detach_tasks(struct lb_env *env, int *total_run_moved)
{
struct list_head *tasks = &env->src_rq->cfs_tasks;
struct task_struct *p;
@@ -5828,6 +5833,9 @@ static int detach_tasks(struct lb_env *env)
detached++;
env->imbalance -= load;
+ if (sysctl_sched_ravg_window)
+ *total_run_moved += div64_u64((u64)p->se.ravg.demand *
+ 100, (u64)(sysctl_sched_ravg_window));
#ifdef CONFIG_PREEMPT
/*
@@ -6967,6 +6975,7 @@ static int load_balance(int this_cpu, struct rq *this_rq,
int *continue_balancing)
{
int ld_moved = 0, cur_ld_moved, active_balance = 0;
+ int total_run_moved = 0;
struct sched_domain *sd_parent = sd->parent;
struct sched_group *group = NULL;
struct rq *busiest = NULL;
@@ -7040,7 +7049,7 @@ more_balance:
* cur_ld_moved - load moved in current iteration
* ld_moved - cumulative load moved across iterations
*/
- cur_ld_moved = detach_tasks(&env);
+ cur_ld_moved = detach_tasks(&env, &total_run_moved);
/*
* We've detached some tasks from busiest_rq. Every
@@ -7176,9 +7185,15 @@ more_balance:
} else {
sd->nr_balance_failed = 0;
if (per_cpu(dbs_boost_needed, this_cpu)) {
+ struct migration_notify_data mnd;
+
per_cpu(dbs_boost_needed, this_cpu) = false;
+
+ mnd.src_cpu = cpu_of(busiest);
+ mnd.dest_cpu = this_cpu;
+ mnd.load = total_run_moved;
atomic_notifier_call_chain(&migration_notifier_head,
- this_cpu, (void *)(long)cpu_of(busiest));
+ 0, (void *)&mnd);
}
}
if (likely(!active_balance)) {
@@ -7376,6 +7391,7 @@ static int active_load_balance_cpu_stop(void *data)
struct rq *busiest_rq = data;
int busiest_cpu = cpu_of(busiest_rq);
int target_cpu = busiest_rq->push_cpu;
+ int total_run_moved = 0;
struct rq *target_rq = cpu_rq(target_cpu);
struct sched_domain *sd;
struct task_struct *p = NULL;
@@ -7418,7 +7434,7 @@ static int active_load_balance_cpu_stop(void *data)
schedstat_inc(sd, alb_count);
- p = detach_one_task(&env);
+ p = detach_one_task(&env, &total_run_moved);
if (p)
schedstat_inc(sd, alb_pushed);
else
@@ -7435,10 +7451,15 @@ out_unlock:
local_irq_enable();
if (per_cpu(dbs_boost_needed, target_cpu)) {
+ struct migration_notify_data mnd;
+
per_cpu(dbs_boost_needed, target_cpu) = false;
+
+ mnd.src_cpu = cpu_of(busiest_rq);
+ mnd.dest_cpu = target_cpu;
+ mnd.load = total_run_moved;
atomic_notifier_call_chain(&migration_notifier_head,
- target_cpu,
- (void *)(long)cpu_of(busiest_rq));
+ 0, (void *)&mnd);
}
return 0;
}
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 0dbe55192ef2..b85da1bb4b49 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -35,6 +35,7 @@ extern void update_cpu_load_active(struct rq *this_rq);
static inline void update_cpu_load_active(struct rq *this_rq) { }
#endif
+extern unsigned int sysctl_sched_ravg_window;
/*
* Helpers for converting nanosecond timing to jiffy resolution
*/
generated by cgit v1.2.3 (git 2.25.1) at 2026-08-04 04:28:17 +0000