From 6cc98f03adbf517986a90c43af0cbc9a732b8435 Mon Sep 17 00:00:00 2001 From: Olav Haugan Date: Fri, 8 Jul 2016 10:59:08 -0700 Subject: timer: Ensure timers are not running before migrating This is needed to support migration of timers during cpu isolation. A timer might be running on the CPU that we want to isolate so we are unable to migrate the timers at this point. We are adding a spin-loop to wait for the timer to finish before migrating the timers. Change-Id: I24d6e91b6dff468c640c2fe3a37a7f31b6f0c79a Signed-off-by: Olav Haugan --- kernel/time/timer.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) (limited to 'kernel/time') diff --git a/kernel/time/timer.c b/kernel/time/timer.c index 51896272fcde..be750f6b2a68 100644 --- a/kernel/time/timer.c +++ b/kernel/time/timer.c @@ -1635,7 +1635,7 @@ static void migrate_timer_list(struct tvec_base *new_base, struct hlist_head *he } } -static void migrate_timers(int cpu) +static void migrate_timers(int cpu, bool wait) { struct tvec_base *old_base; struct tvec_base *new_base; @@ -1651,7 +1651,18 @@ static void migrate_timers(int cpu) spin_lock_irq(&new_base->lock); spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING); - BUG_ON(old_base->running_timer); + if (wait) { + /* Ensure timers are done running before continuing */ + while (old_base->running_timer) { + spin_unlock(&old_base->lock); + spin_unlock_irq(&new_base->lock); + cpu_relax(); + spin_lock_irq(&new_base->lock); + spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING); + } + } else { + BUG_ON(old_base->running_timer); + } for (i = 0; i < TVR_SIZE; i++) migrate_timer_list(new_base, old_base->tv1.vec + i); @@ -1676,7 +1687,7 @@ static int timer_cpu_notify(struct notifier_block *self, switch (action) { case CPU_DEAD: case CPU_DEAD_FROZEN: - migrate_timers((long)hcpu); + migrate_timers((long)hcpu, false); break; default: break; -- cgit v1.2.3 From a7dffd7ffbe6aafe8da10f82a922c00e1c65acdc Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Wed, 25 Mar 2015 11:47:53 +0530 Subject: timer: create timer_quiesce_cpu() to isolate CPU from timers To isolate CPUs (isolate from timers) from sysfs using cpusets, we need some support from the timer core. i.e. A routine timer_quiesce_cpu() which would migrates away all the unpinned timers, but shouldn't touch the pinned ones. This patch creates this routine. Change-Id: I8624e0659b86b7b8fa425a3fafdb0784fe005124 Signed-off-by: Viresh Kumar [forward port to 3.18] Signed-off-by: Santosh Shukla [ohaugan@codeaurora.org: Port to 4.4. Fixes for compilation error] Git-commit: 313910b70ea0c73f8789d9189c11e1f339080646 Git-repo: git://git.linaro.org/people/mike.holmes/santosh.shukla/lng-isol.git Signed-off-by: Olav Haugan --- kernel/time/timer.c | 56 +++++++++++++++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 21 deletions(-) (limited to 'kernel/time') diff --git a/kernel/time/timer.c b/kernel/time/timer.c index be750f6b2a68..067174a4dde3 100644 --- a/kernel/time/timer.c +++ b/kernel/time/timer.c @@ -1620,44 +1620,49 @@ signed long __sched schedule_timeout_uninterruptible(signed long timeout) } EXPORT_SYMBOL(schedule_timeout_uninterruptible); -#ifdef CONFIG_HOTPLUG_CPU -static void migrate_timer_list(struct tvec_base *new_base, struct hlist_head *head) +#if defined(CONFIG_HOTPLUG_CPU) +static void migrate_timer_list(struct tvec_base *new_base, + struct hlist_head *head, bool remove_pinned) { struct timer_list *timer; int cpu = new_base->cpu; + struct hlist_node *n; + int is_pinned; - while (!hlist_empty(head)) { - timer = hlist_entry(head->first, struct timer_list, entry); - /* We ignore the accounting on the dying cpu */ - detach_timer(timer, false); + hlist_for_each_entry_safe(timer, n, head, entry) { + is_pinned = timer->flags & TIMER_PINNED_ON_CPU; + if (!remove_pinned && is_pinned) + continue; + + detach_if_pending(timer, get_timer_base(timer->flags), false); timer->flags = (timer->flags & ~TIMER_BASEMASK) | cpu; internal_add_timer(new_base, timer); } } -static void migrate_timers(int cpu, bool wait) +static void __migrate_timers(int cpu, bool wait, bool remove_pinned) { struct tvec_base *old_base; struct tvec_base *new_base; + unsigned long flags; int i; - BUG_ON(cpu_online(cpu)); old_base = per_cpu_ptr(&tvec_bases, cpu); new_base = get_cpu_ptr(&tvec_bases); /* * The caller is globally serialized and nobody else * takes two locks at once, deadlock is not possible. */ - spin_lock_irq(&new_base->lock); + spin_lock_irqsave(&new_base->lock, flags); spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING); if (wait) { /* Ensure timers are done running before continuing */ while (old_base->running_timer) { spin_unlock(&old_base->lock); - spin_unlock_irq(&new_base->lock); + spin_unlock_irqrestore(&new_base->lock, flags); cpu_relax(); - spin_lock_irq(&new_base->lock); + spin_lock_irqsave(&new_base->lock, flags); spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING); } } else { @@ -1665,29 +1670,38 @@ static void migrate_timers(int cpu, bool wait) } for (i = 0; i < TVR_SIZE; i++) - migrate_timer_list(new_base, old_base->tv1.vec + i); + migrate_timer_list(new_base, old_base->tv1.vec + i, + remove_pinned); for (i = 0; i < TVN_SIZE; i++) { - migrate_timer_list(new_base, old_base->tv2.vec + i); - migrate_timer_list(new_base, old_base->tv3.vec + i); - migrate_timer_list(new_base, old_base->tv4.vec + i); - migrate_timer_list(new_base, old_base->tv5.vec + i); + migrate_timer_list(new_base, old_base->tv2.vec + i, + remove_pinned); + migrate_timer_list(new_base, old_base->tv3.vec + i, + remove_pinned); + migrate_timer_list(new_base, old_base->tv4.vec + i, + remove_pinned); + migrate_timer_list(new_base, old_base->tv5.vec + i, + remove_pinned); } - old_base->active_timers = 0; - old_base->all_timers = 0; - spin_unlock(&old_base->lock); - spin_unlock_irq(&new_base->lock); + spin_unlock_irqrestore(&new_base->lock, flags); put_cpu_ptr(&tvec_bases); } +/* Migrate timers from 'cpu' to this_cpu */ +static void migrate_timers(int cpu) +{ + BUG_ON(cpu_online(cpu)); + __migrate_timers(cpu, false, true); +} + static int timer_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { switch (action) { case CPU_DEAD: case CPU_DEAD_FROZEN: - migrate_timers((long)hcpu, false); + migrate_timers((long)hcpu); break; default: break; -- cgit v1.2.3 From d0cea65e27b7541234d61b70017ec6cc3cfe3eec Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Wed, 25 Mar 2015 11:57:45 +0530 Subject: hrtimer: update timer->state with 'pinned' information 'Pinned' information would be required in migrate_hrtimers() now, as we can migrate non-pinned timers away without a hotplug (i.e. with cpuset.quiesce). And so we may need to identify pinned timers now, as we can't migrate them. This patch reuses the timer->state variable for setting this flag as there were enough number of free bits available in this variable. And there is no point increasing size of this struct by adding another field. Change-Id: If3b3770e547971809e789ea7c8033c48ec2aa92d Signed-off-by: Viresh Kumar [forward port to 3.18] Signed-off-by: Santosh Shukla [ohaugan@codeaurora.org: Port to 4.4] Git-commit: 62feaf1ed0b64c04868d143d8bdb92d60dc3189b Git-repo: git://git.linaro.org/people/mike.holmes/santosh.shukla/lng-isol.git Signed-off-by: Olav Haugan --- kernel/time/hrtimer.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) (limited to 'kernel/time') diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index fa909f9fd559..83c298cc0533 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -880,7 +880,7 @@ static int enqueue_hrtimer(struct hrtimer *timer, base->cpu_base->active_bases |= 1 << base->index; - timer->state = HRTIMER_STATE_ENQUEUED; + timer->state |= HRTIMER_STATE_ENQUEUED; return timerqueue_add(&base->active, &timer->node); } @@ -900,11 +900,9 @@ static void __remove_hrtimer(struct hrtimer *timer, u8 newstate, int reprogram) { struct hrtimer_cpu_base *cpu_base = base->cpu_base; - u8 state = timer->state; - timer->state = newstate; - if (!(state & HRTIMER_STATE_ENQUEUED)) - return; + if (!(timer->state & HRTIMER_STATE_ENQUEUED)) + goto out; if (!timerqueue_del(&base->active, &timer->node)) cpu_base->active_bases &= ~(1 << base->index); @@ -921,6 +919,13 @@ static void __remove_hrtimer(struct hrtimer *timer, if (reprogram && timer == cpu_base->next_timer) hrtimer_force_reprogram(cpu_base, 1); #endif + +out: + /* + * We need to preserve PINNED state here, otherwise we may end up + * migrating pinned hrtimers as well. + */ + timer->state = newstate | (timer->state & HRTIMER_STATE_PINNED); } /* @@ -1002,6 +1007,10 @@ void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, timer_stats_hrtimer_set_start_info(timer); + /* Update pinned state */ + timer->state &= ~HRTIMER_STATE_PINNED; + timer->state |= !!(mode & HRTIMER_MODE_PINNED) << HRTIMER_PINNED_SHIFT; + leftmost = enqueue_hrtimer(timer, new_base); if (!leftmost) goto unlock; -- cgit v1.2.3 From f461d408acb522ec6db8ae34a49c2dbbd6092ab1 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Wed, 25 Mar 2015 12:07:31 +0530 Subject: hrtimer: create hrtimer_quiesce_cpu() to isolate CPU from hrtimers To isolate CPUs (isolate from hrtimers) from sysfs using cpusets, we need some support from the hrtimer core. i.e. A routine hrtimer_quiesce_cpu() which would migrate away all the unpinned hrtimers, but shouldn't touch the pinned ones. This patch creates this routine. Change-Id: I51259ea41e3bd5cdba50b718201a6840174a7224 Signed-off-by: Viresh Kumar [forward port to 3.18] Signed-off-by: Santosh Shukla [ohaugan@codeaurora.org: Port to 4.4] Git-commit: d4d50a0ddc35e58ee95137ba4d14e74fea8b682f Git-repo: git://git.linaro.org/people/mike.holmes/santosh.shukla/lng-isol.git Signed-off-by: Olav Haugan --- kernel/time/hrtimer.c | 54 +++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 10 deletions(-) (limited to 'kernel/time') diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 83c298cc0533..44ddf403fb01 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -1623,13 +1623,17 @@ static void init_hrtimers_cpu(int cpu) hrtimer_init_hres(cpu_base); } -#ifdef CONFIG_HOTPLUG_CPU - +#if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_CPUSETS) static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base, - struct hrtimer_clock_base *new_base) + struct hrtimer_clock_base *new_base, + bool remove_pinned) { struct hrtimer *timer; struct timerqueue_node *node; + struct timerqueue_head pinned; + int is_pinned; + + timerqueue_init_head(&pinned); while ((node = timerqueue_getnext(&old_base->active))) { timer = container_of(node, struct hrtimer, node); @@ -1642,6 +1646,13 @@ static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base, * under us on another CPU */ __remove_hrtimer(timer, old_base, HRTIMER_STATE_ENQUEUED, 0); + + is_pinned = timer->state & HRTIMER_STATE_PINNED; + if (!remove_pinned && is_pinned) { + timerqueue_add(&pinned, &timer->node); + continue; + } + timer->base = new_base; /* * Enqueue the timers on the new cpu. This does not @@ -1653,17 +1664,23 @@ static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base, */ enqueue_hrtimer(timer, new_base); } + + /* Re-queue pinned timers for non-hotplug usecase */ + while ((node = timerqueue_getnext(&pinned))) { + timer = container_of(node, struct hrtimer, node); + + timerqueue_del(&pinned, &timer->node); + enqueue_hrtimer(timer, old_base); + } } -static void migrate_hrtimers(int scpu) +static void __migrate_hrtimers(int scpu, bool remove_pinned) { struct hrtimer_cpu_base *old_base, *new_base; + unsigned long flags; int i; - BUG_ON(cpu_online(scpu)); - tick_cancel_sched_timer(scpu); - - local_irq_disable(); + local_irq_save(flags); old_base = &per_cpu(hrtimer_bases, scpu); new_base = this_cpu_ptr(&hrtimer_bases); /* @@ -1675,7 +1692,7 @@ static void migrate_hrtimers(int scpu) for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { migrate_hrtimer_list(&old_base->clock_base[i], - &new_base->clock_base[i]); + &new_base->clock_base[i], remove_pinned); } raw_spin_unlock(&old_base->lock); @@ -1683,11 +1700,28 @@ static void migrate_hrtimers(int scpu) /* Check, if we got expired work to do */ __hrtimer_peek_ahead_timers(); - local_irq_enable(); + local_irq_restore(flags); +} +#endif /* CONFIG_HOTPLUG_CPU || CONFIG_CPUSETS */ + +#ifdef CONFIG_HOTPLUG_CPU +static void migrate_hrtimers(int scpu) +{ + BUG_ON(cpu_online(scpu)); + tick_cancel_sched_timer(scpu); + + __migrate_hrtimers(scpu, true); } #endif /* CONFIG_HOTPLUG_CPU */ +#ifdef CONFIG_CPUSETS +void hrtimer_quiesce_cpu(void *cpup) +{ + __migrate_hrtimers(*(int *)cpup, false); +} +#endif /* CONFIG_CPUSETS */ + static int hrtimer_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { -- cgit v1.2.3 From a66156c83c6de8dd2e614d6ad37afe7673ca1473 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Wed, 25 Mar 2015 12:57:46 +0530 Subject: hrtimer: make sure PINNED flag is cleared after removing hrtimer Change-Id: Icc4d1c183e993b4b3c9b96ec9779c234e73ecab7 Signed-off-by: Viresh Kumar [forward port to 3.18] Signed-off-by: Santosh Shukla Git-commit: d6c894e515b4cd93c3a08e7c60cce0aa5118c656 Git-repo: git://git.linaro.org/people/mike.holmes/santosh.shukla/lng-isol.git Signed-off-by: Olav Haugan --- kernel/time/hrtimer.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'kernel/time') diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 44ddf403fb01..ab304f854743 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -954,6 +954,7 @@ remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base, bool rest state = HRTIMER_STATE_INACTIVE; __remove_hrtimer(timer, base, state, reprogram); + timer->state &= ~HRTIMER_STATE_PINNED; return 1; } return 0; @@ -1009,7 +1010,7 @@ void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, /* Update pinned state */ timer->state &= ~HRTIMER_STATE_PINNED; - timer->state |= !!(mode & HRTIMER_MODE_PINNED) << HRTIMER_PINNED_SHIFT; + timer->state |= (!!(mode & HRTIMER_MODE_PINNED)) << HRTIMER_PINNED_SHIFT; leftmost = enqueue_hrtimer(timer, new_base); if (!leftmost) -- cgit v1.2.3 From 84e39dcb3bf4b0ea5ed9b1bcd1b1fb43d2a4bc63 Mon Sep 17 00:00:00 2001 From: "Gary S. Robertson" Date: Wed, 10 Sep 2014 14:57:16 -0500 Subject: hrtimer.h: prevent pinned timer state from breaking inactive test An hrtimer may be pinned to a CPU but inactive, so it is no longer valid to test the hrtimer.state struct member as having no bits set when inactive. Changed the test function to mask out the HRTIMER_STATE_PINNED bit when checking for inactive state. Change-Id: I632f37874ef79887ee1202a028ef734f392d6ed0 Signed-off-by: Gary S. Robertson [ohaugan@codeaurora.org: Port to 4.4] Git-commit: 902e4d4eb0d2158d2792166221a72a829caecf07 Git-repo: git://git.linaro.org/people/mike.holmes/santosh.shukla/lng-isol.git Signed-off-by: Olav Haugan --- kernel/time/hrtimer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel/time') diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index ab304f854743..f3b89de9ca2a 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -1186,8 +1186,8 @@ bool hrtimer_active(const struct hrtimer *timer) cpu_base = READ_ONCE(timer->base->cpu_base); seq = raw_read_seqcount_begin(&cpu_base->seq); - if (timer->state != HRTIMER_STATE_INACTIVE || - cpu_base->running == timer) + if (((timer->state & ~HRTIMER_STATE_PINNED) != + HRTIMER_STATE_INACTIVE) || cpu_base->running == timer) return true; } while (read_seqcount_retry(&cpu_base->seq, seq) || -- cgit v1.2.3 From bba552f4fc0046325f29156720a04623a71821d5 Mon Sep 17 00:00:00 2001 From: Santosh Shukla Date: Wed, 25 Mar 2015 16:09:32 +0530 Subject: timer: Add function to migrate timers Add function to migrate timer that will be used by later patch set. Change-Id: I370e404001344e635a663822b07557abbe0f6f52 Signed-off-by: Santosh Shukla [ohaugan@codeaurora.org: Updated commit text and fixed trivial merge conflict] Git-commit: 3633b88d8fcb4273807574c27c328b6908a741e5 Git-repo: git://git.linaro.org/people/mike.holmes/santosh.shukla/lng-isol.git Signed-off-by: Olav Haugan --- kernel/time/timer.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'kernel/time') diff --git a/kernel/time/timer.c b/kernel/time/timer.c index 067174a4dde3..d38a67a49550 100644 --- a/kernel/time/timer.c +++ b/kernel/time/timer.c @@ -1695,6 +1695,13 @@ static void migrate_timers(int cpu) __migrate_timers(cpu, false, true); } +#ifdef CONFIG_CPUSETS +void timer_quiesce_cpu(void *cpup) +{ + __migrate_timers(*(int *)cpup, true, false); +} +#endif /* CONFIG_CPUSETS */ + static int timer_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { -- cgit v1.2.3 From 922fed628c625b28a50f66267e4b9f99088e2aa4 Mon Sep 17 00:00:00 2001 From: Olav Haugan Date: Sun, 3 Jul 2016 15:02:08 -0700 Subject: timer: Do not require CPUSETS to be enabled for migration Do not require CPUSETS to be enabled to allow migration of timers and hrtimers. Change-Id: Ib911a0d34c250c4df020bdb265b92d2b8df8db93 Signed-off-by: Olav Haugan --- kernel/time/hrtimer.c | 10 +++------- kernel/time/timer.c | 2 -- 2 files changed, 3 insertions(+), 9 deletions(-) (limited to 'kernel/time') diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index f3b89de9ca2a..1b0117198a08 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -1624,7 +1624,7 @@ static void init_hrtimers_cpu(int cpu) hrtimer_init_hres(cpu_base); } -#if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_CPUSETS) +#if defined(CONFIG_HOTPLUG_CPU) static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base, struct hrtimer_clock_base *new_base, bool remove_pinned) @@ -1703,9 +1703,7 @@ static void __migrate_hrtimers(int scpu, bool remove_pinned) __hrtimer_peek_ahead_timers(); local_irq_restore(flags); } -#endif /* CONFIG_HOTPLUG_CPU || CONFIG_CPUSETS */ -#ifdef CONFIG_HOTPLUG_CPU static void migrate_hrtimers(int scpu) { BUG_ON(cpu_online(scpu)); @@ -1714,14 +1712,12 @@ static void migrate_hrtimers(int scpu) __migrate_hrtimers(scpu, true); } -#endif /* CONFIG_HOTPLUG_CPU */ - -#ifdef CONFIG_CPUSETS void hrtimer_quiesce_cpu(void *cpup) { __migrate_hrtimers(*(int *)cpup, false); } -#endif /* CONFIG_CPUSETS */ + +#endif /* CONFIG_HOTPLUG_CPU */ static int hrtimer_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) diff --git a/kernel/time/timer.c b/kernel/time/timer.c index d38a67a49550..0efb3916f5a4 100644 --- a/kernel/time/timer.c +++ b/kernel/time/timer.c @@ -1695,12 +1695,10 @@ static void migrate_timers(int cpu) __migrate_timers(cpu, false, true); } -#ifdef CONFIG_CPUSETS void timer_quiesce_cpu(void *cpup) { __migrate_timers(*(int *)cpup, true, false); } -#endif /* CONFIG_CPUSETS */ static int timer_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) -- cgit v1.2.3