From a7dffd7ffbe6aafe8da10f82a922c00e1c65acdc Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Wed, 25 Mar 2015 11:47:53 +0530 Subject: timer: create timer_quiesce_cpu() to isolate CPU from timers To isolate CPUs (isolate from timers) from sysfs using cpusets, we need some support from the timer core. i.e. A routine timer_quiesce_cpu() which would migrates away all the unpinned timers, but shouldn't touch the pinned ones. This patch creates this routine. Change-Id: I8624e0659b86b7b8fa425a3fafdb0784fe005124 Signed-off-by: Viresh Kumar [forward port to 3.18] Signed-off-by: Santosh Shukla [ohaugan@codeaurora.org: Port to 4.4. Fixes for compilation error] Git-commit: 313910b70ea0c73f8789d9189c11e1f339080646 Git-repo: git://git.linaro.org/people/mike.holmes/santosh.shukla/lng-isol.git Signed-off-by: Olav Haugan --- include/linux/timer.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/linux') diff --git a/include/linux/timer.h b/include/linux/timer.h index 7a5602e19e87..b1617e8932b2 100644 --- a/include/linux/timer.h +++ b/include/linux/timer.h @@ -182,6 +182,9 @@ extern void set_timer_slack(struct timer_list *time, int slack_hz); */ #define NEXT_TIMER_MAX_DELTA ((1UL << 30) - 1) +/* To be used from cpusets, only */ +extern void timer_quiesce_cpu(void *cpup); + /* * Timer-statistics info: */ -- cgit v1.2.3 From d0cea65e27b7541234d61b70017ec6cc3cfe3eec Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Wed, 25 Mar 2015 11:57:45 +0530 Subject: hrtimer: update timer->state with 'pinned' information 'Pinned' information would be required in migrate_hrtimers() now, as we can migrate non-pinned timers away without a hotplug (i.e. with cpuset.quiesce). And so we may need to identify pinned timers now, as we can't migrate them. This patch reuses the timer->state variable for setting this flag as there were enough number of free bits available in this variable. And there is no point increasing size of this struct by adding another field. Change-Id: If3b3770e547971809e789ea7c8033c48ec2aa92d Signed-off-by: Viresh Kumar [forward port to 3.18] Signed-off-by: Santosh Shukla [ohaugan@codeaurora.org: Port to 4.4] Git-commit: 62feaf1ed0b64c04868d143d8bdb92d60dc3189b Git-repo: git://git.linaro.org/people/mike.holmes/santosh.shukla/lng-isol.git Signed-off-by: Olav Haugan --- include/linux/hrtimer.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/linux') diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h index 2ead22dd74a0..07766186ee89 100644 --- a/include/linux/hrtimer.h +++ b/include/linux/hrtimer.h @@ -53,6 +53,7 @@ enum hrtimer_restart { * * 0x00 inactive * 0x01 enqueued into rbtree + * 0x02 timer is pinned to a cpu * * The callback state is not part of the timer->state because clearing it would * mean touching the timer after the callback, this makes it impossible to free @@ -72,6 +73,8 @@ enum hrtimer_restart { */ #define HRTIMER_STATE_INACTIVE 0x00 #define HRTIMER_STATE_ENQUEUED 0x01 +#define HRTIMER_PINNED_SHIFT 1 +#define HRTIMER_STATE_PINNED (1 << HRTIMER_PINNED_SHIFT) /** * struct hrtimer - the basic hrtimer structure -- cgit v1.2.3 From f461d408acb522ec6db8ae34a49c2dbbd6092ab1 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Wed, 25 Mar 2015 12:07:31 +0530 Subject: hrtimer: create hrtimer_quiesce_cpu() to isolate CPU from hrtimers To isolate CPUs (isolate from hrtimers) from sysfs using cpusets, we need some support from the hrtimer core. i.e. A routine hrtimer_quiesce_cpu() which would migrate away all the unpinned hrtimers, but shouldn't touch the pinned ones. This patch creates this routine. Change-Id: I51259ea41e3bd5cdba50b718201a6840174a7224 Signed-off-by: Viresh Kumar [forward port to 3.18] Signed-off-by: Santosh Shukla [ohaugan@codeaurora.org: Port to 4.4] Git-commit: d4d50a0ddc35e58ee95137ba4d14e74fea8b682f Git-repo: git://git.linaro.org/people/mike.holmes/santosh.shukla/lng-isol.git Signed-off-by: Olav Haugan --- include/linux/hrtimer.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/linux') diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h index 07766186ee89..952adcacc4cf 100644 --- a/include/linux/hrtimer.h +++ b/include/linux/hrtimer.h @@ -360,6 +360,9 @@ DECLARE_PER_CPU(struct tick_device, tick_cpu_device); /* Exported timer functions: */ +/* To be used from cpusets, only */ +extern void hrtimer_quiesce_cpu(void *cpup); + /* Initialize timers: */ extern void hrtimer_init(struct hrtimer *timer, clockid_t which_clock, enum hrtimer_mode mode); -- cgit v1.2.3 From fc70615291343d031b148bb2676963419a7b672e Mon Sep 17 00:00:00 2001 From: Olav Haugan Date: Sun, 29 May 2016 19:56:37 -0700 Subject: cpumask: Add cpu isolation support Add bitmask and corresponding supporting functions for cpu isolation. Change-Id: Ice1a9503666a2b720bdb324289ca55ceb33097cd Signed-off-by: Olav Haugan --- include/linux/cpumask.h | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'include/linux') diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index 59915ea5373c..0eab4811ee92 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -53,6 +53,7 @@ extern int nr_cpu_ids; * cpu_present_mask - has bit 'cpu' set iff cpu is populated * cpu_online_mask - has bit 'cpu' set iff cpu available to scheduler * cpu_active_mask - has bit 'cpu' set iff cpu available to migration + * cpu_isolated_mask- has bit 'cpu' set iff cpu isolated * * If !CONFIG_HOTPLUG_CPU, present == possible, and active == online. * @@ -89,25 +90,30 @@ extern const struct cpumask *const cpu_possible_mask; extern const struct cpumask *const cpu_online_mask; extern const struct cpumask *const cpu_present_mask; extern const struct cpumask *const cpu_active_mask; +extern const struct cpumask *const cpu_isolated_mask; #if NR_CPUS > 1 #define num_online_cpus() cpumask_weight(cpu_online_mask) #define num_possible_cpus() cpumask_weight(cpu_possible_mask) #define num_present_cpus() cpumask_weight(cpu_present_mask) #define num_active_cpus() cpumask_weight(cpu_active_mask) +#define num_isolated_cpus() cpumask_weight(cpu_isolated_mask) #define cpu_online(cpu) cpumask_test_cpu((cpu), cpu_online_mask) #define cpu_possible(cpu) cpumask_test_cpu((cpu), cpu_possible_mask) #define cpu_present(cpu) cpumask_test_cpu((cpu), cpu_present_mask) #define cpu_active(cpu) cpumask_test_cpu((cpu), cpu_active_mask) +#define cpu_isolated(cpu) cpumask_test_cpu((cpu), cpu_isolated_mask) #else #define num_online_cpus() 1U #define num_possible_cpus() 1U #define num_present_cpus() 1U #define num_active_cpus() 1U +#define num_isolated_cpus() 0U #define cpu_online(cpu) ((cpu) == 0) #define cpu_possible(cpu) ((cpu) == 0) #define cpu_present(cpu) ((cpu) == 0) #define cpu_active(cpu) ((cpu) == 0) +#define cpu_isolated(cpu) ((cpu) == 0) #endif /* verify cpu argument to cpumask_* operators */ @@ -714,12 +720,14 @@ extern const DECLARE_BITMAP(cpu_all_bits, NR_CPUS); #define for_each_possible_cpu(cpu) for_each_cpu((cpu), cpu_possible_mask) #define for_each_online_cpu(cpu) for_each_cpu((cpu), cpu_online_mask) #define for_each_present_cpu(cpu) for_each_cpu((cpu), cpu_present_mask) +#define for_each_isolated_cpu(cpu) for_each_cpu((cpu), cpu_isolated_mask) /* Wrappers for arch boot code to manipulate normally-constant masks */ void set_cpu_possible(unsigned int cpu, bool possible); void set_cpu_present(unsigned int cpu, bool present); void set_cpu_online(unsigned int cpu, bool online); void set_cpu_active(unsigned int cpu, bool active); +void set_cpu_isolated(unsigned int cpu, bool isolated); void init_cpu_present(const struct cpumask *src); void init_cpu_possible(const struct cpumask *src); void init_cpu_online(const struct cpumask *src); -- cgit v1.2.3 From 3fe956359cd3f1cd174285b693b424f89123ff96 Mon Sep 17 00:00:00 2001 From: Olav Haugan Date: Thu, 18 Aug 2016 16:49:44 -0700 Subject: watchdog: Add support for cpu isolation Open up interface to allow external subsystem to enable and disable hard lockup detector. Change-Id: I88a728ee1d54aaa887fab52e5e40d1d4e4fc69ca Signed-off-by: Olav Haugan --- include/linux/device.h | 1 + include/linux/sched.h | 8 ++++++++ 2 files changed, 9 insertions(+) (limited to 'include/linux') diff --git a/include/linux/device.h b/include/linux/device.h index 9f27351c6b9c..4b4e2d5ce6e7 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -1023,6 +1023,7 @@ static inline bool device_supports_offline(struct device *dev) extern void lock_device_hotplug(void); extern void unlock_device_hotplug(void); extern int lock_device_hotplug_sysfs(void); +extern void lock_device_hotplug_assert(void); extern int device_offline(struct device *dev); extern int device_online(struct device *dev); extern void set_primary_fwnode(struct device *dev, struct fwnode_handle *fwnode); diff --git a/include/linux/sched.h b/include/linux/sched.h index 4701e0403167..ac6b519494a2 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -409,6 +409,8 @@ extern int proc_dowatchdog_thresh(struct ctl_table *table, int write, extern unsigned int softlockup_panic; extern unsigned int hardlockup_panic; void lockup_detector_init(void); +extern void watchdog_enable(unsigned int cpu); +extern void watchdog_disable(unsigned int cpu); #else static inline void touch_softlockup_watchdog_sched(void) { @@ -425,6 +427,12 @@ static inline void touch_all_softlockup_watchdogs(void) static inline void lockup_detector_init(void) { } +static inline void watchdog_enable(unsigned int cpu) +{ +} +static inline void watchdog_disable(unsigned int cpu) +{ +} #endif #ifdef CONFIG_DETECT_HUNG_TASK -- cgit v1.2.3 From e33c24bfecde67d7d665bfcf90c7d4c2f231be79 Mon Sep 17 00:00:00 2001 From: Olav Haugan Date: Tue, 31 May 2016 14:34:46 -0700 Subject: sched: add cpu isolation support This adds cpu isolation APIs to the scheduler to isolate and unisolate CPUs. Isolating and unisolating a CPU can be used in place of hotplug. Isolating and unisolating a CPU is faster than hotplug and can thus be used to optimize the performance and power of multi-core CPUs. Isolating works by migrating non-pinned IRQs and tasks to other CPUS and marking the CPU as not available to the scheduler and load balancer. Pinned tasks and IRQs are still allowed to run but it is expected that this would be minimal. Unisolation works by just marking the CPU available for scheduler and load balancer. Change-Id: I0bbddb56238c2958c5987877c5bfc3e79afa67cc Signed-off-by: Olav Haugan --- include/linux/sched.h | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) (limited to 'include/linux') diff --git a/include/linux/sched.h b/include/linux/sched.h index ac6b519494a2..a395d8a9ff73 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -363,6 +363,41 @@ extern cpumask_var_t cpu_isolated_map; extern int runqueue_is_locked(int cpu); +#ifdef CONFIG_HOTPLUG_CPU +extern int sched_isolate_count(const cpumask_t *mask, bool include_offline); +extern int sched_isolate_cpu(int cpu); +extern int sched_unisolate_cpu(int cpu); +extern int sched_unisolate_cpu_unlocked(int cpu); +#else +static inline int sched_isolate_count(const cpumask_t *mask, + bool include_offline) +{ + cpumask_t count_mask; + + if (include_offline) + cpumask_andnot(&count_mask, mask, cpu_online_mask); + else + return 0; + + return cpumask_weight(&count_mask); +} + +static inline int sched_isolate_cpu(int cpu) +{ + return 0; +} + +static inline int sched_unisolate_cpu(int cpu) +{ + return 0; +} + +static inline int sched_unisolate_cpu_unlocked(int cpu) +{ + return 0; +} +#endif + #if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON) extern void nohz_balance_enter_idle(int cpu); extern void set_cpu_sd_state_idle(void); -- cgit v1.2.3 From ca2c4c65188206b0251a8d7576374847b45110e4 Mon Sep 17 00:00:00 2001 From: Olav Haugan Date: Thu, 2 Jun 2016 16:29:27 -0700 Subject: sched/tick: Ensure timers does not get queued on isolated cpus Timers should not be queued on isolated cpus. Instead try to find other cpus to queue timer on. Change-Id: I5d849dfd29aa5bb594454473768d7db1da258028 Signed-off-by: Olav Haugan --- include/linux/tick.h | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/tick.h b/include/linux/tick.h index 5bf3ddade19c..1732697ea419 100644 --- a/include/linux/tick.h +++ b/include/linux/tick.h @@ -161,7 +161,15 @@ extern void __tick_nohz_task_switch(void); #else static inline int housekeeping_any_cpu(void) { - return smp_processor_id(); + cpumask_t available; + int cpu; + + cpumask_andnot(&available, cpu_online_mask, cpu_isolated_mask); + cpu = cpumask_any(&available); + if (cpu >= nr_cpu_ids) + cpu = smp_processor_id(); + + return cpu; } static inline bool tick_nohz_full_enabled(void) { return false; } static inline bool tick_nohz_full_cpu(int cpu) { return false; } @@ -187,7 +195,7 @@ static inline bool is_housekeeping_cpu(int cpu) if (tick_nohz_full_enabled()) return cpumask_test_cpu(cpu, housekeeping_mask); #endif - return true; + return !cpu_isolated(cpu); } static inline void housekeeping_affine(struct task_struct *t) -- cgit v1.2.3