From e18f3ffb9c3ddfc1b4ad8f38f5f2acae8c16f0c9 Mon Sep 17 00:00:00 2001 From: Andrew Morton Date: Wed, 9 May 2007 02:33:50 -0700 Subject: schedule_on_each_cpu(): use preempt_disable() We take workqueue_mutex in there to keep CPU hotplug away. But preempt_disable() will suffice for that. Cc: Ingo Molnar Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/workqueue.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel/workqueue.c') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index b6fa5e63085d..1ea4bcb86974 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -638,7 +638,7 @@ int schedule_on_each_cpu(work_func_t func) if (!works) return -ENOMEM; - mutex_lock(&workqueue_mutex); + preempt_disable(); /* CPU hotplug */ for_each_online_cpu(cpu) { struct work_struct *work = per_cpu_ptr(works, cpu); @@ -646,7 +646,7 @@ int schedule_on_each_cpu(work_func_t func) set_bit(WORK_STRUCT_PENDING, work_data_bits(work)); __queue_work(per_cpu_ptr(keventd_wq->cpu_wq, cpu), work); } - mutex_unlock(&workqueue_mutex); + preempt_enable(); flush_workqueue(keventd_wq); free_percpu(works); return 0; -- cgit v1.2.3 From fc2e4d70410546307344821eed6fd23803a45286 Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Wed, 9 May 2007 02:33:51 -0700 Subject: reimplement flush_workqueue() Remove ->remove_sequence, ->insert_sequence, and ->work_done from struct cpu_workqueue_struct. To implement flush_workqueue() we can queue a barrier work on each CPU and wait for its completition. The barrier is queued under workqueue_mutex to ensure that per cpu wq->cpu_wq is alive, we drop this mutex before going to sleep. If CPU goes down while we are waiting for completition, take_over_work() will move the barrier on another CPU, and the handler will wake up us eventually. Signed-off-by: Oleg Nesterov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/workqueue.c | 70 ++++++++++++++++++++++++------------------------------ 1 file changed, 31 insertions(+), 39 deletions(-) (limited to 'kernel/workqueue.c') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 1ea4bcb86974..b7bb37ab03bc 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -36,23 +36,13 @@ /* * The per-CPU workqueue (if single thread, we always use the first * possible cpu). - * - * The sequence counters are for flush_scheduled_work(). It wants to wait - * until all currently-scheduled works are completed, but it doesn't - * want to be livelocked by new, incoming ones. So it waits until - * remove_sequence is >= the insert_sequence which pertained when - * flush_scheduled_work() was called. */ struct cpu_workqueue_struct { spinlock_t lock; - long remove_sequence; /* Least-recently added (next to run) */ - long insert_sequence; /* Next to add */ - struct list_head worklist; wait_queue_head_t more_work; - wait_queue_head_t work_done; struct workqueue_struct *wq; struct task_struct *thread; @@ -138,8 +128,6 @@ static int __run_work(struct cpu_workqueue_struct *cwq, struct work_struct *work f(work); spin_lock_irqsave(&cwq->lock, flags); - cwq->remove_sequence++; - wake_up(&cwq->work_done); ret = 1; } spin_unlock_irqrestore(&cwq->lock, flags); @@ -187,7 +175,6 @@ static void __queue_work(struct cpu_workqueue_struct *cwq, spin_lock_irqsave(&cwq->lock, flags); set_wq_data(work, cwq); list_add_tail(&work->entry, &cwq->worklist); - cwq->insert_sequence++; wake_up(&cwq->more_work); spin_unlock_irqrestore(&cwq->lock, flags); } @@ -338,8 +325,6 @@ static void run_workqueue(struct cpu_workqueue_struct *cwq) } spin_lock_irqsave(&cwq->lock, flags); - cwq->remove_sequence++; - wake_up(&cwq->work_done); } cwq->run_depth--; spin_unlock_irqrestore(&cwq->lock, flags); @@ -394,6 +379,25 @@ static int worker_thread(void *__cwq) return 0; } +struct wq_barrier { + struct work_struct work; + struct completion done; +}; + +static void wq_barrier_func(struct work_struct *work) +{ + struct wq_barrier *barr = container_of(work, struct wq_barrier, work); + complete(&barr->done); +} + +static inline void init_wq_barrier(struct wq_barrier *barr) +{ + INIT_WORK(&barr->work, wq_barrier_func); + __set_bit(WORK_STRUCT_PENDING, work_data_bits(&barr->work)); + + init_completion(&barr->done); +} + static void flush_cpu_workqueue(struct cpu_workqueue_struct *cwq) { if (cwq->thread == current) { @@ -401,23 +405,18 @@ static void flush_cpu_workqueue(struct cpu_workqueue_struct *cwq) * Probably keventd trying to flush its own queue. So simply run * it by hand rather than deadlocking. */ + mutex_unlock(&workqueue_mutex); run_workqueue(cwq); + mutex_lock(&workqueue_mutex); } else { - DEFINE_WAIT(wait); - long sequence_needed; + struct wq_barrier barr; - spin_lock_irq(&cwq->lock); - sequence_needed = cwq->insert_sequence; + init_wq_barrier(&barr); + __queue_work(cwq, &barr.work); - while (sequence_needed - cwq->remove_sequence > 0) { - prepare_to_wait(&cwq->work_done, &wait, - TASK_UNINTERRUPTIBLE); - spin_unlock_irq(&cwq->lock); - schedule(); - spin_lock_irq(&cwq->lock); - } - finish_wait(&cwq->work_done, &wait); - spin_unlock_irq(&cwq->lock); + mutex_unlock(&workqueue_mutex); + wait_for_completion(&barr.done); + mutex_lock(&workqueue_mutex); } } @@ -428,29 +427,25 @@ static void flush_cpu_workqueue(struct cpu_workqueue_struct *cwq) * Forces execution of the workqueue and blocks until its completion. * This is typically used in driver shutdown handlers. * - * This function will sample each workqueue's current insert_sequence number and - * will sleep until the head sequence is greater than or equal to that. This - * means that we sleep until all works which were queued on entry have been - * handled, but we are not livelocked by new incoming ones. + * We sleep until all works which were queued on entry have been handled, + * but we are not livelocked by new incoming ones. * * This function used to run the workqueues itself. Now we just wait for the * helper threads to do it. */ void fastcall flush_workqueue(struct workqueue_struct *wq) { - might_sleep(); - + mutex_lock(&workqueue_mutex); if (is_single_threaded(wq)) { /* Always use first cpu's area. */ flush_cpu_workqueue(per_cpu_ptr(wq->cpu_wq, singlethread_cpu)); } else { int cpu; - mutex_lock(&workqueue_mutex); for_each_online_cpu(cpu) flush_cpu_workqueue(per_cpu_ptr(wq->cpu_wq, cpu)); - mutex_unlock(&workqueue_mutex); } + mutex_unlock(&workqueue_mutex); } EXPORT_SYMBOL_GPL(flush_workqueue); @@ -463,12 +458,9 @@ static struct task_struct *create_workqueue_thread(struct workqueue_struct *wq, spin_lock_init(&cwq->lock); cwq->wq = wq; cwq->thread = NULL; - cwq->insert_sequence = 0; - cwq->remove_sequence = 0; cwq->freezeable = freezeable; INIT_LIST_HEAD(&cwq->worklist); init_waitqueue_head(&cwq->more_work); - init_waitqueue_head(&cwq->work_done); if (is_single_threaded(wq)) p = kthread_create(worker_thread, cwq, "%s", wq->name); -- cgit v1.2.3 From b89deed32ccc96098bd6bc953c64bba6b847774f Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Wed, 9 May 2007 02:33:52 -0700 Subject: implement flush_work() A basic problem with flush_scheduled_work() is that it blocks behind _all_ presently-queued works, rather than just the work whcih the caller wants to flush. If the caller holds some lock, and if one of the queued work happens to want that lock as well then accidental deadlocks can occur. One example of this is the phy layer: it wants to flush work while holding rtnl_lock(). But if a linkwatch event happens to be queued, the phy code will deadlock because the linkwatch callback function takes rtnl_lock. So we implement a new function which will flush a *single* work - just the one which the caller wants to free up. Thus we avoid the accidental deadlocks which can arise from unrelated subsystems' callbacks taking shared locks. flush_work() non-blockingly dequeues the work_struct which we want to kill, then it waits for its handler to complete on all CPUs. Add ->current_work to the "struct cpu_workqueue_struct", it points to currently running "struct work_struct". When flush_work(work) detects ->current_work == work, it inserts a barrier at the _head_ of ->worklist (and thus right _after_ that work) and waits for completition. This means that the next work fired on that CPU will be this barrier, or another barrier queued by concurrent flush_work(), so the caller of flush_work() will be woken before any "regular" work has a chance to run. When wait_on_work() unlocks workqueue_mutex (or whatever we choose to protect against CPU hotplug), CPU may go away. But in that case take_over_work() will move a barrier we queued to another CPU, it will be fired sometime, and wait_on_work() will be woken. Actually, we are doing cleanup_workqueue_thread()->kthread_stop() before take_over_work(), so cwq->thread should complete its ->worklist (and thus the barrier), because currently we don't check kthread_should_stop() in run_workqueue(). But even if we did, everything should be ok. [akpm@osdl.org: cleanup] [akpm@osdl.org: add flush_work_keventd() wrapper] Signed-off-by: Oleg Nesterov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/workqueue.c | 95 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 92 insertions(+), 3 deletions(-) (limited to 'kernel/workqueue.c') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index b7bb37ab03bc..918d55267a12 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -46,6 +46,7 @@ struct cpu_workqueue_struct { struct workqueue_struct *wq; struct task_struct *thread; + struct work_struct *current_work; int run_depth; /* Detect run_workqueue() recursion depth */ @@ -120,6 +121,7 @@ static int __run_work(struct cpu_workqueue_struct *cwq, struct work_struct *work && work_pending(work) && !list_empty(&work->entry)) { work_func_t f = work->func; + cwq->current_work = work; list_del_init(&work->entry); spin_unlock_irqrestore(&cwq->lock, flags); @@ -128,6 +130,7 @@ static int __run_work(struct cpu_workqueue_struct *cwq, struct work_struct *work f(work); spin_lock_irqsave(&cwq->lock, flags); + cwq->current_work = NULL; ret = 1; } spin_unlock_irqrestore(&cwq->lock, flags); @@ -166,6 +169,17 @@ int fastcall run_scheduled_work(struct work_struct *work) } EXPORT_SYMBOL(run_scheduled_work); +static void insert_work(struct cpu_workqueue_struct *cwq, + struct work_struct *work, int tail) +{ + set_wq_data(work, cwq); + if (tail) + list_add_tail(&work->entry, &cwq->worklist); + else + list_add(&work->entry, &cwq->worklist); + wake_up(&cwq->more_work); +} + /* Preempt must be disabled. */ static void __queue_work(struct cpu_workqueue_struct *cwq, struct work_struct *work) @@ -173,9 +187,7 @@ static void __queue_work(struct cpu_workqueue_struct *cwq, unsigned long flags; spin_lock_irqsave(&cwq->lock, flags); - set_wq_data(work, cwq); - list_add_tail(&work->entry, &cwq->worklist); - wake_up(&cwq->more_work); + insert_work(cwq, work, 1); spin_unlock_irqrestore(&cwq->lock, flags); } @@ -305,6 +317,7 @@ static void run_workqueue(struct cpu_workqueue_struct *cwq) struct work_struct, entry); work_func_t f = work->func; + cwq->current_work = work; list_del_init(cwq->worklist.next); spin_unlock_irqrestore(&cwq->lock, flags); @@ -325,6 +338,7 @@ static void run_workqueue(struct cpu_workqueue_struct *cwq) } spin_lock_irqsave(&cwq->lock, flags); + cwq->current_work = NULL; } cwq->run_depth--; spin_unlock_irqrestore(&cwq->lock, flags); @@ -449,6 +463,75 @@ void fastcall flush_workqueue(struct workqueue_struct *wq) } EXPORT_SYMBOL_GPL(flush_workqueue); +static void wait_on_work(struct cpu_workqueue_struct *cwq, + struct work_struct *work) +{ + struct wq_barrier barr; + int running = 0; + + spin_lock_irq(&cwq->lock); + if (unlikely(cwq->current_work == work)) { + init_wq_barrier(&barr); + insert_work(cwq, &barr.work, 0); + running = 1; + } + spin_unlock_irq(&cwq->lock); + + if (unlikely(running)) { + mutex_unlock(&workqueue_mutex); + wait_for_completion(&barr.done); + mutex_lock(&workqueue_mutex); + } +} + +/** + * flush_work - block until a work_struct's callback has terminated + * @wq: the workqueue on which the work is queued + * @work: the work which is to be flushed + * + * flush_work() will attempt to cancel the work if it is queued. If the work's + * callback appears to be running, flush_work() will block until it has + * completed. + * + * flush_work() is designed to be used when the caller is tearing down data + * structures which the callback function operates upon. It is expected that, + * prior to calling flush_work(), the caller has arranged for the work to not + * be requeued. + */ +void flush_work(struct workqueue_struct *wq, struct work_struct *work) +{ + struct cpu_workqueue_struct *cwq; + + mutex_lock(&workqueue_mutex); + cwq = get_wq_data(work); + /* Was it ever queued ? */ + if (!cwq) + goto out; + + /* + * This work can't be re-queued, and the lock above protects us + * from take_over_work(), no need to re-check that get_wq_data() + * is still the same when we take cwq->lock. + */ + spin_lock_irq(&cwq->lock); + list_del_init(&work->entry); + work_release(work); + spin_unlock_irq(&cwq->lock); + + if (is_single_threaded(wq)) { + /* Always use first cpu's area. */ + wait_on_work(per_cpu_ptr(wq->cpu_wq, singlethread_cpu), work); + } else { + int cpu; + + for_each_online_cpu(cpu) + wait_on_work(per_cpu_ptr(wq->cpu_wq, cpu), work); + } +out: + mutex_unlock(&workqueue_mutex); +} +EXPORT_SYMBOL_GPL(flush_work); + static struct task_struct *create_workqueue_thread(struct workqueue_struct *wq, int cpu, int freezeable) { @@ -650,6 +733,12 @@ void flush_scheduled_work(void) } EXPORT_SYMBOL(flush_scheduled_work); +void flush_work_keventd(struct work_struct *work) +{ + flush_work(keventd_wq, work); +} +EXPORT_SYMBOL(flush_work_keventd); + /** * cancel_rearming_delayed_workqueue - reliably kill off a delayed work whose handler rearms the delayed work. * @wq: the controlling workqueue structure -- cgit v1.2.3 From edab2516a6c1752e8e5e3d55727cabf12346e5df Mon Sep 17 00:00:00 2001 From: Andrew Morton Date: Wed, 9 May 2007 02:33:53 -0700 Subject: flush_workqueue(): use preempt_disable to hold off cpu hotplug Cc: Oleg Nesterov Cc: Srivatsa Vaddagiri Cc: Gautham Shenoy Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/workqueue.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) (limited to 'kernel/workqueue.c') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 918d55267a12..5176d51bcc2a 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -419,18 +419,22 @@ static void flush_cpu_workqueue(struct cpu_workqueue_struct *cwq) * Probably keventd trying to flush its own queue. So simply run * it by hand rather than deadlocking. */ - mutex_unlock(&workqueue_mutex); + preempt_enable(); + /* + * We can still touch *cwq here because we are keventd, and + * hot-unplug will be waiting us to exit. + */ run_workqueue(cwq); - mutex_lock(&workqueue_mutex); + preempt_disable(); } else { struct wq_barrier barr; init_wq_barrier(&barr); __queue_work(cwq, &barr.work); - mutex_unlock(&workqueue_mutex); + preempt_enable(); /* Can no longer touch *cwq */ wait_for_completion(&barr.done); - mutex_lock(&workqueue_mutex); + preempt_disable(); } } @@ -449,7 +453,7 @@ static void flush_cpu_workqueue(struct cpu_workqueue_struct *cwq) */ void fastcall flush_workqueue(struct workqueue_struct *wq) { - mutex_lock(&workqueue_mutex); + preempt_disable(); /* CPU hotplug */ if (is_single_threaded(wq)) { /* Always use first cpu's area. */ flush_cpu_workqueue(per_cpu_ptr(wq->cpu_wq, singlethread_cpu)); @@ -459,7 +463,7 @@ void fastcall flush_workqueue(struct workqueue_struct *wq) for_each_online_cpu(cpu) flush_cpu_workqueue(per_cpu_ptr(wq->cpu_wq, cpu)); } - mutex_unlock(&workqueue_mutex); + preempt_enable(); } EXPORT_SYMBOL_GPL(flush_workqueue); -- cgit v1.2.3 From 83c22520c51bf67529367e8237f95c03fe44e2da Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Wed, 9 May 2007 02:33:54 -0700 Subject: flush_cpu_workqueue: don't flush an empty ->worklist Now when we have ->current_work we can avoid adding a barrier and waiting for its completition when cwq's queue is empty. Note: this change is also useful if we change flush_workqueue() to also check the dead CPUs. Signed-off-by: Oleg Nesterov Cc: Srivatsa Vaddagiri Cc: Gautham Shenoy Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/workqueue.c | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) (limited to 'kernel/workqueue.c') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 5176d51bcc2a..5ecf4984e382 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -404,12 +404,15 @@ static void wq_barrier_func(struct work_struct *work) complete(&barr->done); } -static inline void init_wq_barrier(struct wq_barrier *barr) +static void insert_wq_barrier(struct cpu_workqueue_struct *cwq, + struct wq_barrier *barr, int tail) { INIT_WORK(&barr->work, wq_barrier_func); __set_bit(WORK_STRUCT_PENDING, work_data_bits(&barr->work)); init_completion(&barr->done); + + insert_work(cwq, &barr->work, tail); } static void flush_cpu_workqueue(struct cpu_workqueue_struct *cwq) @@ -428,13 +431,20 @@ static void flush_cpu_workqueue(struct cpu_workqueue_struct *cwq) preempt_disable(); } else { struct wq_barrier barr; + int active = 0; - init_wq_barrier(&barr); - __queue_work(cwq, &barr.work); + spin_lock_irq(&cwq->lock); + if (!list_empty(&cwq->worklist) || cwq->current_work != NULL) { + insert_wq_barrier(cwq, &barr, 1); + active = 1; + } + spin_unlock_irq(&cwq->lock); - preempt_enable(); /* Can no longer touch *cwq */ - wait_for_completion(&barr.done); - preempt_disable(); + if (active) { + preempt_enable(); + wait_for_completion(&barr.done); + preempt_disable(); + } } } @@ -475,8 +485,7 @@ static void wait_on_work(struct cpu_workqueue_struct *cwq, spin_lock_irq(&cwq->lock); if (unlikely(cwq->current_work == work)) { - init_wq_barrier(&barr); - insert_work(cwq, &barr.work, 0); + insert_wq_barrier(cwq, &barr, 0); running = 1; } spin_unlock_irq(&cwq->lock); -- cgit v1.2.3 From 319c2a986eb45989690c955d9667b814ef0ed56f Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Wed, 9 May 2007 02:34:06 -0700 Subject: workqueue: fix freezeable workqueues implementation Currently ->freezeable is per-cpu, this is wrong. CPU_UP_PREPARE creates cwq->thread which is not freezeable. Move ->freezeable to workqueue_struct. Signed-off-by: Oleg Nesterov Cc: Srivatsa Vaddagiri Cc: "Pallipadi, Venkatesh" Cc: Gautham shenoy Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/workqueue.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'kernel/workqueue.c') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 5ecf4984e382..d80dbdceadb8 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -49,8 +49,6 @@ struct cpu_workqueue_struct { struct work_struct *current_work; int run_depth; /* Detect run_workqueue() recursion depth */ - - int freezeable; /* Freeze the thread during suspend */ } ____cacheline_aligned; /* @@ -61,6 +59,7 @@ struct workqueue_struct { struct cpu_workqueue_struct *cpu_wq; const char *name; struct list_head list; /* Empty if single thread */ + int freezeable; /* Freeze threads during suspend */ }; /* All the per-cpu workqueues on the system, for hotplug cpu to add/remove @@ -351,7 +350,7 @@ static int worker_thread(void *__cwq) struct k_sigaction sa; sigset_t blocked; - if (!cwq->freezeable) + if (!cwq->wq->freezeable) current->flags |= PF_NOFREEZE; set_user_nice(current, -5); @@ -375,7 +374,7 @@ static int worker_thread(void *__cwq) set_current_state(TASK_INTERRUPTIBLE); while (!kthread_should_stop()) { - if (cwq->freezeable) + if (cwq->wq->freezeable) try_to_freeze(); add_wait_queue(&cwq->more_work, &wait); @@ -546,7 +545,7 @@ out: EXPORT_SYMBOL_GPL(flush_work); static struct task_struct *create_workqueue_thread(struct workqueue_struct *wq, - int cpu, int freezeable) + int cpu) { struct cpu_workqueue_struct *cwq = per_cpu_ptr(wq->cpu_wq, cpu); struct task_struct *p; @@ -554,7 +553,6 @@ static struct task_struct *create_workqueue_thread(struct workqueue_struct *wq, spin_lock_init(&cwq->lock); cwq->wq = wq; cwq->thread = NULL; - cwq->freezeable = freezeable; INIT_LIST_HEAD(&cwq->worklist); init_waitqueue_head(&cwq->more_work); @@ -586,10 +584,12 @@ struct workqueue_struct *__create_workqueue(const char *name, } wq->name = name; + wq->freezeable = freezeable; + mutex_lock(&workqueue_mutex); if (singlethread) { INIT_LIST_HEAD(&wq->list); - p = create_workqueue_thread(wq, singlethread_cpu, freezeable); + p = create_workqueue_thread(wq, singlethread_cpu); if (!p) destroy = 1; else @@ -597,7 +597,7 @@ struct workqueue_struct *__create_workqueue(const char *name, } else { list_add(&wq->list, &workqueues); for_each_online_cpu(cpu) { - p = create_workqueue_thread(wq, cpu, freezeable); + p = create_workqueue_thread(wq, cpu); if (p) { kthread_bind(p, cpu); wake_up_process(p); @@ -854,7 +854,7 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, mutex_lock(&workqueue_mutex); /* Create a new workqueue thread for it. */ list_for_each_entry(wq, &workqueues, list) { - if (!create_workqueue_thread(wq, hotcpu, 0)) { + if (!create_workqueue_thread(wq, hotcpu)) { printk("workqueue for %i failed\n", hotcpu); return NOTIFY_BAD; } -- cgit v1.2.3 From d721304dce0ced0b3b0366996cc02929669708a8 Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Wed, 9 May 2007 02:34:07 -0700 Subject: workqueue: fix flush_workqueue() vs CPU_DEAD race Many thanks to Srivatsa Vaddagiri for the helpful discussion and for spotting the bug in my previous attempt. work->func() (and thus flush_workqueue()) must not use workqueue_mutex, this leads to deadlock when CPU_DEAD does kthread_stop(). However without this mutex held we can't detect CPU_DEAD in progress, which can move pending works to another CPU while the dead one is not on cpu_online_map. Change flush_workqueue() to use for_each_possible_cpu(). This means that flush_cpu_workqueue() may hit CPU which is already dead. However in that case !list_empty(&cwq->worklist) || cwq->current_work != NULL means that CPU_DEAD in progress, it will do kthread_stop() + take_over_work() so we can proceed and insert a barrier. We hold cwq->lock, so we are safe. Also, add migrate_sequence incremented by take_over_work() under cwq->lock. If take_over_work() happened before we checked this CPU, we should see the new value after spin_unlock(). Further possible changes: remove CPU_DEAD handling (along with take_over_work, migrate_sequence) from workqueue.c. CPU_DEAD just sets cwq->please_exit_after_flush flag. CPU_UP_PREPARE->create_workqueue_thread() clears this flag, and creates the new thread if cwq->thread == NULL. This way the workqueue/cpu-hotplug interaction is almost zero, workqueue_mutex just protects "workqueues" list, CPU_LOCK_ACQUIRE/CPU_LOCK_RELEASE go away. Signed-off-by: Oleg Nesterov Cc: Srivatsa Vaddagiri Cc: "Pallipadi, Venkatesh" Cc: Gautham shenoy Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/workqueue.c | 44 +++++++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 19 deletions(-) (limited to 'kernel/workqueue.c') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index d80dbdceadb8..1d1933cf3778 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -64,6 +64,7 @@ struct workqueue_struct { /* All the per-cpu workqueues on the system, for hotplug cpu to add/remove threads to each one as cpus come/go. */ +static long migrate_sequence __read_mostly; static DEFINE_MUTEX(workqueue_mutex); static LIST_HEAD(workqueues); @@ -421,13 +422,7 @@ static void flush_cpu_workqueue(struct cpu_workqueue_struct *cwq) * Probably keventd trying to flush its own queue. So simply run * it by hand rather than deadlocking. */ - preempt_enable(); - /* - * We can still touch *cwq here because we are keventd, and - * hot-unplug will be waiting us to exit. - */ run_workqueue(cwq); - preempt_disable(); } else { struct wq_barrier barr; int active = 0; @@ -439,11 +434,8 @@ static void flush_cpu_workqueue(struct cpu_workqueue_struct *cwq) } spin_unlock_irq(&cwq->lock); - if (active) { - preempt_enable(); + if (active) wait_for_completion(&barr.done); - preempt_disable(); - } } } @@ -462,17 +454,21 @@ static void flush_cpu_workqueue(struct cpu_workqueue_struct *cwq) */ void fastcall flush_workqueue(struct workqueue_struct *wq) { - preempt_disable(); /* CPU hotplug */ if (is_single_threaded(wq)) { /* Always use first cpu's area. */ flush_cpu_workqueue(per_cpu_ptr(wq->cpu_wq, singlethread_cpu)); } else { + long sequence; int cpu; +again: + sequence = migrate_sequence; - for_each_online_cpu(cpu) + for_each_possible_cpu(cpu) flush_cpu_workqueue(per_cpu_ptr(wq->cpu_wq, cpu)); + + if (unlikely(sequence != migrate_sequence)) + goto again; } - preempt_enable(); } EXPORT_SYMBOL_GPL(flush_workqueue); @@ -544,17 +540,21 @@ out: } EXPORT_SYMBOL_GPL(flush_work); -static struct task_struct *create_workqueue_thread(struct workqueue_struct *wq, - int cpu) +static void init_cpu_workqueue(struct workqueue_struct *wq, int cpu) { struct cpu_workqueue_struct *cwq = per_cpu_ptr(wq->cpu_wq, cpu); - struct task_struct *p; - spin_lock_init(&cwq->lock); cwq->wq = wq; - cwq->thread = NULL; + spin_lock_init(&cwq->lock); INIT_LIST_HEAD(&cwq->worklist); init_waitqueue_head(&cwq->more_work); +} + +static struct task_struct *create_workqueue_thread(struct workqueue_struct *wq, + int cpu) +{ + struct cpu_workqueue_struct *cwq = per_cpu_ptr(wq->cpu_wq, cpu); + struct task_struct *p; if (is_single_threaded(wq)) p = kthread_create(worker_thread, cwq, "%s", wq->name); @@ -589,6 +589,7 @@ struct workqueue_struct *__create_workqueue(const char *name, mutex_lock(&workqueue_mutex); if (singlethread) { INIT_LIST_HEAD(&wq->list); + init_cpu_workqueue(wq, singlethread_cpu); p = create_workqueue_thread(wq, singlethread_cpu); if (!p) destroy = 1; @@ -596,7 +597,11 @@ struct workqueue_struct *__create_workqueue(const char *name, wake_up_process(p); } else { list_add(&wq->list, &workqueues); - for_each_online_cpu(cpu) { + for_each_possible_cpu(cpu) { + init_cpu_workqueue(wq, cpu); + if (!cpu_online(cpu)) + continue; + p = create_workqueue_thread(wq, cpu); if (p) { kthread_bind(p, cpu); @@ -831,6 +836,7 @@ static void take_over_work(struct workqueue_struct *wq, unsigned int cpu) spin_lock_irq(&cwq->lock); list_replace_init(&cwq->worklist, &list); + migrate_sequence++; while (!list_empty(&list)) { printk("Taking work for %s\n", wq->name); -- cgit v1.2.3 From 36aa9dfc39bf473780439f5629c30f59d677e793 Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Wed, 9 May 2007 02:34:08 -0700 Subject: workqueue: don't clear cwq->thread until it exits Pointed out by Srivatsa Vaddagiri. cleanup_workqueue_thread() sets cwq->thread = NULL and does kthread_stop(). This breaks the "if (cwq->thread == current)" logic in flush_cpu_workqueue() and leads to deadlock. Kill the thead first, then clear cwq->thread. workqueue_mutex protects us from create_workqueue_thread() so we don't need cwq->lock. Signed-off-by: Oleg Nesterov Cc: Srivatsa Vaddagiri Cc: "Pallipadi, Venkatesh" Cc: Gautham shenoy Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/workqueue.c | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) (limited to 'kernel/workqueue.c') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 1d1933cf3778..398c34ff6a54 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -625,17 +625,12 @@ EXPORT_SYMBOL_GPL(__create_workqueue); static void cleanup_workqueue_thread(struct workqueue_struct *wq, int cpu) { - struct cpu_workqueue_struct *cwq; - unsigned long flags; - struct task_struct *p; + struct cpu_workqueue_struct *cwq = per_cpu_ptr(wq->cpu_wq, cpu); - cwq = per_cpu_ptr(wq->cpu_wq, cpu); - spin_lock_irqsave(&cwq->lock, flags); - p = cwq->thread; - cwq->thread = NULL; - spin_unlock_irqrestore(&cwq->lock, flags); - if (p) - kthread_stop(p); + if (cwq->thread) { + kthread_stop(cwq->thread); + cwq->thread = NULL; + } } /** -- cgit v1.2.3 From 3af24433efac62f451bfdb1cf1edb7181fb73645 Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Wed, 9 May 2007 02:34:09 -0700 Subject: workqueue: don't migrate pending works from the dead CPU Currently CPU_DEAD uses kthread_stop() to stop cwq->thread and then transfers cwq->worklist to another CPU. However, it is very unlikely that worker_thread() will notice kthread_should_stop() before flushing cwq->worklist. It is only possible if worker_thread() was preempted after run_workqueue(cwq), a new work_struct was added, and CPU_DEAD happened before cwq->thread has a chance to run. This means that take_over_work() mostly adds unneeded complications. Note also that kthread_stop() is not good per se, wake_up_process() may confuse work->func() if it sleeps waiting for some event. Remove take_over_work() and migrate_sequence complications. CPU_DEAD sets the cwq->should_stop flag (introduced by this patch) and waits for cwq->thread to flush cwq->worklist and exit. Because the dead CPU is not on cpu_online_map, no more works can be added to that cwq. cpu_populated_map was introduced to optimize for_each_possible_cpu(), it is not strictly needed, and it is more a documentation in fact. Saves 418 bytes. Signed-off-by: Oleg Nesterov Cc: Srivatsa Vaddagiri Cc: "Pallipadi, Venkatesh" Cc: Gautham shenoy Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/workqueue.c | 430 ++++++++++++++++++++++++++--------------------------- 1 file changed, 211 insertions(+), 219 deletions(-) (limited to 'kernel/workqueue.c') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 398c34ff6a54..a981add58fb9 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -43,10 +43,11 @@ struct cpu_workqueue_struct { struct list_head worklist; wait_queue_head_t more_work; + struct work_struct *current_work; struct workqueue_struct *wq; struct task_struct *thread; - struct work_struct *current_work; + int should_stop; int run_depth; /* Detect run_workqueue() recursion depth */ } ____cacheline_aligned; @@ -64,11 +65,12 @@ struct workqueue_struct { /* All the per-cpu workqueues on the system, for hotplug cpu to add/remove threads to each one as cpus come/go. */ -static long migrate_sequence __read_mostly; static DEFINE_MUTEX(workqueue_mutex); static LIST_HEAD(workqueues); -static int singlethread_cpu; +static int singlethread_cpu __read_mostly; +/* optimization, we could use cpu_possible_map */ +static cpumask_t cpu_populated_map __read_mostly; /* If it's single threaded, it isn't in the list of workqueues. */ static inline int is_single_threaded(struct workqueue_struct *wq) @@ -344,10 +346,28 @@ static void run_workqueue(struct cpu_workqueue_struct *cwq) spin_unlock_irqrestore(&cwq->lock, flags); } +/* + * NOTE: the caller must not touch *cwq if this func returns true + */ +static int cwq_should_stop(struct cpu_workqueue_struct *cwq) +{ + int should_stop = cwq->should_stop; + + if (unlikely(should_stop)) { + spin_lock_irq(&cwq->lock); + should_stop = cwq->should_stop && list_empty(&cwq->worklist); + if (should_stop) + cwq->thread = NULL; + spin_unlock_irq(&cwq->lock); + } + + return should_stop; +} + static int worker_thread(void *__cwq) { struct cpu_workqueue_struct *cwq = __cwq; - DECLARE_WAITQUEUE(wait, current); + DEFINE_WAIT(wait); struct k_sigaction sa; sigset_t blocked; @@ -373,23 +393,21 @@ static int worker_thread(void *__cwq) siginitset(&sa.sa.sa_mask, sigmask(SIGCHLD)); do_sigaction(SIGCHLD, &sa, (struct k_sigaction *)0); - set_current_state(TASK_INTERRUPTIBLE); - while (!kthread_should_stop()) { + for (;;) { if (cwq->wq->freezeable) try_to_freeze(); - add_wait_queue(&cwq->more_work, &wait); - if (list_empty(&cwq->worklist)) + prepare_to_wait(&cwq->more_work, &wait, TASK_INTERRUPTIBLE); + if (!cwq->should_stop && list_empty(&cwq->worklist)) schedule(); - else - __set_current_state(TASK_RUNNING); - remove_wait_queue(&cwq->more_work, &wait); + finish_wait(&cwq->more_work, &wait); + + if (cwq_should_stop(cwq)) + break; - if (!list_empty(&cwq->worklist)) - run_workqueue(cwq); - set_current_state(TASK_INTERRUPTIBLE); + run_workqueue(cwq); } - __set_current_state(TASK_RUNNING); + return 0; } @@ -454,20 +472,13 @@ static void flush_cpu_workqueue(struct cpu_workqueue_struct *cwq) */ void fastcall flush_workqueue(struct workqueue_struct *wq) { - if (is_single_threaded(wq)) { - /* Always use first cpu's area. */ + if (is_single_threaded(wq)) flush_cpu_workqueue(per_cpu_ptr(wq->cpu_wq, singlethread_cpu)); - } else { - long sequence; + else { int cpu; -again: - sequence = migrate_sequence; - for_each_possible_cpu(cpu) + for_each_cpu_mask(cpu, cpu_populated_map) flush_cpu_workqueue(per_cpu_ptr(wq->cpu_wq, cpu)); - - if (unlikely(sequence != migrate_sequence)) - goto again; } } EXPORT_SYMBOL_GPL(flush_workqueue); @@ -485,11 +496,8 @@ static void wait_on_work(struct cpu_workqueue_struct *cwq, } spin_unlock_irq(&cwq->lock); - if (unlikely(running)) { - mutex_unlock(&workqueue_mutex); + if (unlikely(running)) wait_for_completion(&barr.done); - mutex_lock(&workqueue_mutex); - } } /** @@ -510,155 +518,31 @@ void flush_work(struct workqueue_struct *wq, struct work_struct *work) { struct cpu_workqueue_struct *cwq; - mutex_lock(&workqueue_mutex); cwq = get_wq_data(work); /* Was it ever queued ? */ if (!cwq) - goto out; + return; /* - * This work can't be re-queued, and the lock above protects us - * from take_over_work(), no need to re-check that get_wq_data() - * is still the same when we take cwq->lock. + * This work can't be re-queued, no need to re-check that + * get_wq_data() is still the same when we take cwq->lock. */ spin_lock_irq(&cwq->lock); list_del_init(&work->entry); work_release(work); spin_unlock_irq(&cwq->lock); - if (is_single_threaded(wq)) { - /* Always use first cpu's area. */ + if (is_single_threaded(wq)) wait_on_work(per_cpu_ptr(wq->cpu_wq, singlethread_cpu), work); - } else { + else { int cpu; - for_each_online_cpu(cpu) + for_each_cpu_mask(cpu, cpu_populated_map) wait_on_work(per_cpu_ptr(wq->cpu_wq, cpu), work); } -out: - mutex_unlock(&workqueue_mutex); } EXPORT_SYMBOL_GPL(flush_work); -static void init_cpu_workqueue(struct workqueue_struct *wq, int cpu) -{ - struct cpu_workqueue_struct *cwq = per_cpu_ptr(wq->cpu_wq, cpu); - - cwq->wq = wq; - spin_lock_init(&cwq->lock); - INIT_LIST_HEAD(&cwq->worklist); - init_waitqueue_head(&cwq->more_work); -} - -static struct task_struct *create_workqueue_thread(struct workqueue_struct *wq, - int cpu) -{ - struct cpu_workqueue_struct *cwq = per_cpu_ptr(wq->cpu_wq, cpu); - struct task_struct *p; - - if (is_single_threaded(wq)) - p = kthread_create(worker_thread, cwq, "%s", wq->name); - else - p = kthread_create(worker_thread, cwq, "%s/%d", wq->name, cpu); - if (IS_ERR(p)) - return NULL; - cwq->thread = p; - return p; -} - -struct workqueue_struct *__create_workqueue(const char *name, - int singlethread, int freezeable) -{ - int cpu, destroy = 0; - struct workqueue_struct *wq; - struct task_struct *p; - - wq = kzalloc(sizeof(*wq), GFP_KERNEL); - if (!wq) - return NULL; - - wq->cpu_wq = alloc_percpu(struct cpu_workqueue_struct); - if (!wq->cpu_wq) { - kfree(wq); - return NULL; - } - - wq->name = name; - wq->freezeable = freezeable; - - mutex_lock(&workqueue_mutex); - if (singlethread) { - INIT_LIST_HEAD(&wq->list); - init_cpu_workqueue(wq, singlethread_cpu); - p = create_workqueue_thread(wq, singlethread_cpu); - if (!p) - destroy = 1; - else - wake_up_process(p); - } else { - list_add(&wq->list, &workqueues); - for_each_possible_cpu(cpu) { - init_cpu_workqueue(wq, cpu); - if (!cpu_online(cpu)) - continue; - - p = create_workqueue_thread(wq, cpu); - if (p) { - kthread_bind(p, cpu); - wake_up_process(p); - } else - destroy = 1; - } - } - mutex_unlock(&workqueue_mutex); - - /* - * Was there any error during startup? If yes then clean up: - */ - if (destroy) { - destroy_workqueue(wq); - wq = NULL; - } - return wq; -} -EXPORT_SYMBOL_GPL(__create_workqueue); - -static void cleanup_workqueue_thread(struct workqueue_struct *wq, int cpu) -{ - struct cpu_workqueue_struct *cwq = per_cpu_ptr(wq->cpu_wq, cpu); - - if (cwq->thread) { - kthread_stop(cwq->thread); - cwq->thread = NULL; - } -} - -/** - * destroy_workqueue - safely terminate a workqueue - * @wq: target workqueue - * - * Safely destroy a workqueue. All work currently pending will be done first. - */ -void destroy_workqueue(struct workqueue_struct *wq) -{ - int cpu; - - flush_workqueue(wq); - - /* We don't need the distraction of CPUs appearing and vanishing. */ - mutex_lock(&workqueue_mutex); - if (is_single_threaded(wq)) - cleanup_workqueue_thread(wq, singlethread_cpu); - else { - for_each_online_cpu(cpu) - cleanup_workqueue_thread(wq, cpu); - list_del(&wq->list); - } - mutex_unlock(&workqueue_mutex); - free_percpu(wq->cpu_wq); - kfree(wq); -} -EXPORT_SYMBOL_GPL(destroy_workqueue); static struct workqueue_struct *keventd_wq; @@ -822,85 +706,193 @@ int current_is_keventd(void) } -/* Take the work from this (downed) CPU. */ -static void take_over_work(struct workqueue_struct *wq, unsigned int cpu) +static struct cpu_workqueue_struct * +init_cpu_workqueue(struct workqueue_struct *wq, int cpu) { struct cpu_workqueue_struct *cwq = per_cpu_ptr(wq->cpu_wq, cpu); - struct list_head list; - struct work_struct *work; - spin_lock_irq(&cwq->lock); - list_replace_init(&cwq->worklist, &list); - migrate_sequence++; - - while (!list_empty(&list)) { - printk("Taking work for %s\n", wq->name); - work = list_entry(list.next,struct work_struct,entry); - list_del(&work->entry); - __queue_work(per_cpu_ptr(wq->cpu_wq, smp_processor_id()), work); - } - spin_unlock_irq(&cwq->lock); + cwq->wq = wq; + spin_lock_init(&cwq->lock); + INIT_LIST_HEAD(&cwq->worklist); + init_waitqueue_head(&cwq->more_work); + + return cwq; } -/* We're holding the cpucontrol mutex here */ -static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, - unsigned long action, - void *hcpu) +static int create_workqueue_thread(struct cpu_workqueue_struct *cwq, int cpu) +{ + struct workqueue_struct *wq = cwq->wq; + const char *fmt = is_single_threaded(wq) ? "%s" : "%s/%d"; + struct task_struct *p; + + p = kthread_create(worker_thread, cwq, fmt, wq->name, cpu); + /* + * Nobody can add the work_struct to this cwq, + * if (caller is __create_workqueue) + * nobody should see this wq + * else // caller is CPU_UP_PREPARE + * cpu is not on cpu_online_map + * so we can abort safely. + */ + if (IS_ERR(p)) + return PTR_ERR(p); + + cwq->thread = p; + cwq->should_stop = 0; + if (!is_single_threaded(wq)) + kthread_bind(p, cpu); + + if (is_single_threaded(wq) || cpu_online(cpu)) + wake_up_process(p); + + return 0; +} + +struct workqueue_struct *__create_workqueue(const char *name, + int singlethread, int freezeable) { - unsigned int hotcpu = (unsigned long)hcpu; struct workqueue_struct *wq; + struct cpu_workqueue_struct *cwq; + int err = 0, cpu; - switch (action) { - case CPU_UP_PREPARE: + wq = kzalloc(sizeof(*wq), GFP_KERNEL); + if (!wq) + return NULL; + + wq->cpu_wq = alloc_percpu(struct cpu_workqueue_struct); + if (!wq->cpu_wq) { + kfree(wq); + return NULL; + } + + wq->name = name; + wq->freezeable = freezeable; + + if (singlethread) { + INIT_LIST_HEAD(&wq->list); + cwq = init_cpu_workqueue(wq, singlethread_cpu); + err = create_workqueue_thread(cwq, singlethread_cpu); + } else { mutex_lock(&workqueue_mutex); - /* Create a new workqueue thread for it. */ - list_for_each_entry(wq, &workqueues, list) { - if (!create_workqueue_thread(wq, hotcpu)) { - printk("workqueue for %i failed\n", hotcpu); - return NOTIFY_BAD; - } + list_add(&wq->list, &workqueues); + + for_each_possible_cpu(cpu) { + cwq = init_cpu_workqueue(wq, cpu); + if (err || !cpu_online(cpu)) + continue; + err = create_workqueue_thread(cwq, cpu); } - break; + mutex_unlock(&workqueue_mutex); + } + + if (err) { + destroy_workqueue(wq); + wq = NULL; + } + return wq; +} +EXPORT_SYMBOL_GPL(__create_workqueue); - case CPU_ONLINE: - /* Kick off worker threads. */ - list_for_each_entry(wq, &workqueues, list) { - struct cpu_workqueue_struct *cwq; +static void cleanup_workqueue_thread(struct cpu_workqueue_struct *cwq, int cpu) +{ + struct wq_barrier barr; + int alive = 0; - cwq = per_cpu_ptr(wq->cpu_wq, hotcpu); - kthread_bind(cwq->thread, hotcpu); - wake_up_process(cwq->thread); - } + spin_lock_irq(&cwq->lock); + if (cwq->thread != NULL) { + insert_wq_barrier(cwq, &barr, 1); + cwq->should_stop = 1; + alive = 1; + } + spin_unlock_irq(&cwq->lock); + + if (alive) { + wait_for_completion(&barr.done); + + while (unlikely(cwq->thread != NULL)) + cpu_relax(); + /* + * Wait until cwq->thread unlocks cwq->lock, + * it won't touch *cwq after that. + */ + smp_rmb(); + spin_unlock_wait(&cwq->lock); + } +} + +/** + * destroy_workqueue - safely terminate a workqueue + * @wq: target workqueue + * + * Safely destroy a workqueue. All work currently pending will be done first. + */ +void destroy_workqueue(struct workqueue_struct *wq) +{ + struct cpu_workqueue_struct *cwq; + + if (is_single_threaded(wq)) { + cwq = per_cpu_ptr(wq->cpu_wq, singlethread_cpu); + cleanup_workqueue_thread(cwq, singlethread_cpu); + } else { + int cpu; + + mutex_lock(&workqueue_mutex); + list_del(&wq->list); mutex_unlock(&workqueue_mutex); - break; - case CPU_UP_CANCELED: - list_for_each_entry(wq, &workqueues, list) { - if (!per_cpu_ptr(wq->cpu_wq, hotcpu)->thread) - continue; - /* Unbind so it can run. */ - kthread_bind(per_cpu_ptr(wq->cpu_wq, hotcpu)->thread, - any_online_cpu(cpu_online_map)); - cleanup_workqueue_thread(wq, hotcpu); + for_each_cpu_mask(cpu, cpu_populated_map) { + cwq = per_cpu_ptr(wq->cpu_wq, cpu); + cleanup_workqueue_thread(cwq, cpu); } - mutex_unlock(&workqueue_mutex); - break; + } - case CPU_DOWN_PREPARE: + free_percpu(wq->cpu_wq); + kfree(wq); +} +EXPORT_SYMBOL_GPL(destroy_workqueue); + +static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, + unsigned long action, + void *hcpu) +{ + unsigned int cpu = (unsigned long)hcpu; + struct cpu_workqueue_struct *cwq; + struct workqueue_struct *wq; + + switch (action) { + case CPU_LOCK_ACQUIRE: mutex_lock(&workqueue_mutex); - break; + return NOTIFY_OK; - case CPU_DOWN_FAILED: + case CPU_LOCK_RELEASE: mutex_unlock(&workqueue_mutex); - break; + return NOTIFY_OK; - case CPU_DEAD: - list_for_each_entry(wq, &workqueues, list) - cleanup_workqueue_thread(wq, hotcpu); - list_for_each_entry(wq, &workqueues, list) - take_over_work(wq, hotcpu); - mutex_unlock(&workqueue_mutex); - break; + case CPU_UP_PREPARE: + cpu_set(cpu, cpu_populated_map); + } + + list_for_each_entry(wq, &workqueues, list) { + cwq = per_cpu_ptr(wq->cpu_wq, cpu); + + switch (action) { + case CPU_UP_PREPARE: + if (!create_workqueue_thread(cwq, cpu)) + break; + printk(KERN_ERR "workqueue for %i failed\n", cpu); + return NOTIFY_BAD; + + case CPU_ONLINE: + wake_up_process(cwq->thread); + break; + + case CPU_UP_CANCELED: + if (cwq->thread) + wake_up_process(cwq->thread); + case CPU_DEAD: + cleanup_workqueue_thread(cwq, cpu); + break; + } } return NOTIFY_OK; @@ -908,9 +900,9 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, void init_workqueues(void) { + cpu_populated_map = cpu_online_map; singlethread_cpu = first_cpu(cpu_possible_map); hotcpu_notifier(workqueue_cpu_callback, 0); keventd_wq = create_workqueue("events"); BUG_ON(!keventd_wq); } - -- cgit v1.2.3 From 7097a87afe937a5879528d52880c2d95f089e96c Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Wed, 9 May 2007 02:34:10 -0700 Subject: workqueue: kill run_scheduled_work() Because it has no callers. Actually, I think the whole idea of run_scheduled_work() was not right, not good to mix "unqueue this work and execute its ->func()" in one function. Signed-off-by: Oleg Nesterov Cc: Ingo Molnar Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/workqueue.c | 73 ------------------------------------------------------ 1 file changed, 73 deletions(-) (limited to 'kernel/workqueue.c') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index a981add58fb9..ea422254f8bf 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -98,79 +98,6 @@ static inline void *get_wq_data(struct work_struct *work) return (void *) (atomic_long_read(&work->data) & WORK_STRUCT_WQ_DATA_MASK); } -static int __run_work(struct cpu_workqueue_struct *cwq, struct work_struct *work) -{ - int ret = 0; - unsigned long flags; - - spin_lock_irqsave(&cwq->lock, flags); - /* - * We need to re-validate the work info after we've gotten - * the cpu_workqueue lock. We can run the work now iff: - * - * - the wq_data still matches the cpu_workqueue_struct - * - AND the work is still marked pending - * - AND the work is still on a list (which will be this - * workqueue_struct list) - * - * All these conditions are important, because we - * need to protect against the work being run right - * now on another CPU (all but the last one might be - * true if it's currently running and has not been - * released yet, for example). - */ - if (get_wq_data(work) == cwq - && work_pending(work) - && !list_empty(&work->entry)) { - work_func_t f = work->func; - cwq->current_work = work; - list_del_init(&work->entry); - spin_unlock_irqrestore(&cwq->lock, flags); - - if (!test_bit(WORK_STRUCT_NOAUTOREL, work_data_bits(work))) - work_release(work); - f(work); - - spin_lock_irqsave(&cwq->lock, flags); - cwq->current_work = NULL; - ret = 1; - } - spin_unlock_irqrestore(&cwq->lock, flags); - return ret; -} - -/** - * run_scheduled_work - run scheduled work synchronously - * @work: work to run - * - * This checks if the work was pending, and runs it - * synchronously if so. It returns a boolean to indicate - * whether it had any scheduled work to run or not. - * - * NOTE! This _only_ works for normal work_structs. You - * CANNOT use this for delayed work, because the wq data - * for delayed work will not point properly to the per- - * CPU workqueue struct, but will change! - */ -int fastcall run_scheduled_work(struct work_struct *work) -{ - for (;;) { - struct cpu_workqueue_struct *cwq; - - if (!work_pending(work)) - return 0; - if (list_empty(&work->entry)) - return 0; - /* NOTE! This depends intimately on __queue_work! */ - cwq = get_wq_data(work); - if (!cwq) - return 0; - if (__run_work(cwq, work)) - return 1; - } -} -EXPORT_SYMBOL(run_scheduled_work); - static void insert_work(struct cpu_workqueue_struct *cwq, struct work_struct *work, int tail) { -- cgit v1.2.3 From f293ea92007419e4f9c52db0cf57af17f45b9f94 Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Wed, 9 May 2007 02:34:10 -0700 Subject: workqueue: don't save interrupts in run_workqueue() work->func() may sleep, it's a bug to call run_workqueue() with irqs disabled. Signed-off-by: Oleg Nesterov Cc: Ingo Molnar Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/workqueue.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) (limited to 'kernel/workqueue.c') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index ea422254f8bf..74f3f7825229 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -227,13 +227,7 @@ EXPORT_SYMBOL_GPL(queue_delayed_work_on); static void run_workqueue(struct cpu_workqueue_struct *cwq) { - unsigned long flags; - - /* - * Keep taking off work from the queue until - * done. - */ - spin_lock_irqsave(&cwq->lock, flags); + spin_lock_irq(&cwq->lock); cwq->run_depth++; if (cwq->run_depth > 3) { /* morton gets to eat his hat */ @@ -248,7 +242,7 @@ static void run_workqueue(struct cpu_workqueue_struct *cwq) cwq->current_work = work; list_del_init(cwq->worklist.next); - spin_unlock_irqrestore(&cwq->lock, flags); + spin_unlock_irq(&cwq->lock); BUG_ON(get_wq_data(work) != cwq); if (!test_bit(WORK_STRUCT_NOAUTOREL, work_data_bits(work))) @@ -266,11 +260,11 @@ static void run_workqueue(struct cpu_workqueue_struct *cwq) dump_stack(); } - spin_lock_irqsave(&cwq->lock, flags); + spin_lock_irq(&cwq->lock); cwq->current_work = NULL; } cwq->run_depth--; - spin_unlock_irqrestore(&cwq->lock, flags); + spin_unlock_irq(&cwq->lock); } /* @@ -399,6 +393,8 @@ static void flush_cpu_workqueue(struct cpu_workqueue_struct *cwq) */ void fastcall flush_workqueue(struct workqueue_struct *wq) { + might_sleep(); + if (is_single_threaded(wq)) flush_cpu_workqueue(per_cpu_ptr(wq->cpu_wq, singlethread_cpu)); else { @@ -445,6 +441,8 @@ void flush_work(struct workqueue_struct *wq, struct work_struct *work) { struct cpu_workqueue_struct *cwq; + might_sleep(); + cwq = get_wq_data(work); /* Was it ever queued ? */ if (!cwq) -- cgit v1.2.3 From dfb4b82e1c631b1a6057e77212996a890aa515b7 Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Wed, 9 May 2007 02:34:11 -0700 Subject: workqueue: make cancel_rearming_delayed_workqueue() work on idle dwork cancel_rearming_delayed_workqueue(dwork) will hang forever if dwork was not scheduled, because in that case cancel_delayed_work()->del_timer_sync() never returns true. I don't know if there are any callers which may have problems, but this is not so convenient, and the fix is very simple. Q: looks like we don't need "struct workqueue_struct *wq" parameter. If the timer was aborted successfully, get_wq_data() == wq. Is it worth to add the new function? Signed-off-by: Oleg Nesterov Cc: Ingo Molnar Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/workqueue.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'kernel/workqueue.c') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 74f3f7825229..ce72d45c7fd8 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -569,6 +569,10 @@ EXPORT_SYMBOL(flush_work_keventd); void cancel_rearming_delayed_workqueue(struct workqueue_struct *wq, struct delayed_work *dwork) { + /* Was it ever queued ? */ + if (!get_wq_data(&dwork->work)) + return; + while (!cancel_delayed_work(dwork)) flush_workqueue(wq); } -- cgit v1.2.3 From b1f4ec172f75bc2f5cc4f4be69b5587660a955d2 Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Wed, 9 May 2007 02:34:12 -0700 Subject: workqueue: introduce cpu_singlethread_map The code like if (is_single_threaded(wq)) do_something(singlethread_cpu); else { for_each_cpu_mask(cpu, cpu_populated_map) do_something(cpu); } looks very annoying. We can add "static cpumask_t cpu_singlethread_map" and simplify the code. Lessens .text a bit, and imho makes the code more readable. Signed-off-by: Oleg Nesterov Cc: Ingo Molnar Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/workqueue.c | 55 +++++++++++++++++++++++++----------------------------- 1 file changed, 25 insertions(+), 30 deletions(-) (limited to 'kernel/workqueue.c') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index ce72d45c7fd8..6308a4bc6a82 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -69,6 +69,7 @@ static DEFINE_MUTEX(workqueue_mutex); static LIST_HEAD(workqueues); static int singlethread_cpu __read_mostly; +static cpumask_t cpu_singlethread_map __read_mostly; /* optimization, we could use cpu_possible_map */ static cpumask_t cpu_populated_map __read_mostly; @@ -78,6 +79,12 @@ static inline int is_single_threaded(struct workqueue_struct *wq) return list_empty(&wq->list); } +static const cpumask_t *wq_cpu_map(struct workqueue_struct *wq) +{ + return is_single_threaded(wq) + ? &cpu_singlethread_map : &cpu_populated_map; +} + /* * Set the workqueue on which a work item is to be run * - Must *only* be called if the pending flag is set @@ -393,16 +400,12 @@ static void flush_cpu_workqueue(struct cpu_workqueue_struct *cwq) */ void fastcall flush_workqueue(struct workqueue_struct *wq) { - might_sleep(); - - if (is_single_threaded(wq)) - flush_cpu_workqueue(per_cpu_ptr(wq->cpu_wq, singlethread_cpu)); - else { - int cpu; + const cpumask_t *cpu_map = wq_cpu_map(wq); + int cpu - for_each_cpu_mask(cpu, cpu_populated_map) - flush_cpu_workqueue(per_cpu_ptr(wq->cpu_wq, cpu)); - } + might_sleep(); + for_each_cpu_mask(cpu, *cpu_map) + flush_cpu_workqueue(per_cpu_ptr(wq->cpu_wq, cpu)); } EXPORT_SYMBOL_GPL(flush_workqueue); @@ -439,7 +442,9 @@ static void wait_on_work(struct cpu_workqueue_struct *cwq, */ void flush_work(struct workqueue_struct *wq, struct work_struct *work) { + const cpumask_t *cpu_map = wq_cpu_map(wq); struct cpu_workqueue_struct *cwq; + int cpu; might_sleep(); @@ -457,14 +462,8 @@ void flush_work(struct workqueue_struct *wq, struct work_struct *work) work_release(work); spin_unlock_irq(&cwq->lock); - if (is_single_threaded(wq)) - wait_on_work(per_cpu_ptr(wq->cpu_wq, singlethread_cpu), work); - else { - int cpu; - - for_each_cpu_mask(cpu, cpu_populated_map) - wait_on_work(per_cpu_ptr(wq->cpu_wq, cpu), work); - } + for_each_cpu_mask(cpu, *cpu_map) + wait_on_work(per_cpu_ptr(wq->cpu_wq, cpu), work); } EXPORT_SYMBOL_GPL(flush_work); @@ -757,22 +756,17 @@ static void cleanup_workqueue_thread(struct cpu_workqueue_struct *cwq, int cpu) */ void destroy_workqueue(struct workqueue_struct *wq) { + const cpumask_t *cpu_map = wq_cpu_map(wq); struct cpu_workqueue_struct *cwq; + int cpu; - if (is_single_threaded(wq)) { - cwq = per_cpu_ptr(wq->cpu_wq, singlethread_cpu); - cleanup_workqueue_thread(cwq, singlethread_cpu); - } else { - int cpu; + mutex_lock(&workqueue_mutex); + list_del(&wq->list); + mutex_unlock(&workqueue_mutex); - mutex_lock(&workqueue_mutex); - list_del(&wq->list); - mutex_unlock(&workqueue_mutex); - - for_each_cpu_mask(cpu, cpu_populated_map) { - cwq = per_cpu_ptr(wq->cpu_wq, cpu); - cleanup_workqueue_thread(cwq, cpu); - } + for_each_cpu_mask(cpu, *cpu_map) { + cwq = per_cpu_ptr(wq->cpu_wq, cpu); + cleanup_workqueue_thread(cwq, cpu); } free_percpu(wq->cpu_wq); @@ -831,6 +825,7 @@ void init_workqueues(void) { cpu_populated_map = cpu_online_map; singlethread_cpu = first_cpu(cpu_possible_map); + cpu_singlethread_map = cpumask_of_cpu(singlethread_cpu); hotcpu_notifier(workqueue_cpu_callback, 0); keventd_wq = create_workqueue("events"); BUG_ON(!keventd_wq); -- cgit v1.2.3 From cce1a1656c9a3fdc6c6c1029b576e4ab6ecaac37 Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Wed, 9 May 2007 02:34:13 -0700 Subject: workqueue: introduce workqueue_struct->singlethread Add explicit workqueue_struct->singlethread flag. This lessens .text a little, but most importantly this allows us to manipulate wq->list without changine the meaning of is_single_threaded(). Signed-off-by: Oleg Nesterov Cc: Ingo Molnar Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/workqueue.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'kernel/workqueue.c') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 6308a4bc6a82..32b1091f21ef 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -58,8 +58,9 @@ struct cpu_workqueue_struct { */ struct workqueue_struct { struct cpu_workqueue_struct *cpu_wq; + struct list_head list; const char *name; - struct list_head list; /* Empty if single thread */ + int singlethread; int freezeable; /* Freeze threads during suspend */ }; @@ -76,7 +77,7 @@ static cpumask_t cpu_populated_map __read_mostly; /* If it's single threaded, it isn't in the list of workqueues. */ static inline int is_single_threaded(struct workqueue_struct *wq) { - return list_empty(&wq->list); + return wq->singlethread; } static const cpumask_t *wq_cpu_map(struct workqueue_struct *wq) @@ -401,7 +402,7 @@ static void flush_cpu_workqueue(struct cpu_workqueue_struct *cwq) void fastcall flush_workqueue(struct workqueue_struct *wq) { const cpumask_t *cpu_map = wq_cpu_map(wq); - int cpu + int cpu; might_sleep(); for_each_cpu_mask(cpu, *cpu_map) @@ -694,10 +695,11 @@ struct workqueue_struct *__create_workqueue(const char *name, } wq->name = name; + wq->singlethread = singlethread; wq->freezeable = freezeable; + INIT_LIST_HEAD(&wq->list); if (singlethread) { - INIT_LIST_HEAD(&wq->list); cwq = init_cpu_workqueue(wq, singlethread_cpu); err = create_workqueue_thread(cwq, singlethread_cpu); } else { -- cgit v1.2.3 From c12920d19078eb8fd99560ec232a6e05c6ff1aa8 Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Wed, 9 May 2007 02:34:14 -0700 Subject: workqueue: make init_workqueues() __init The only caller of init_workqueues() is do_basic_setup(). Signed-off-by: Oleg Nesterov Cc: Ingo Molnar Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/workqueue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel/workqueue.c') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 32b1091f21ef..e858e93886e3 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -823,7 +823,7 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, return NOTIFY_OK; } -void init_workqueues(void) +void __init init_workqueues(void) { cpu_populated_map = cpu_online_map; singlethread_cpu = first_cpu(cpu_possible_map); -- cgit v1.2.3 From 06ba38a9a0f6ceffe70343f684c5a690e3710ef4 Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Wed, 9 May 2007 02:34:15 -0700 Subject: workqueues: shift kthread_bind() from CPU_UP_PREPARE to CPU_ONLINE CPU_UP_PREPARE binds cwq->thread to the new CPU. So CPU_UP_CANCELED tries to wake up the task which is bound to the failed CPU. With this patch we don't bind cwq->thread until CPU becomes online. The first wake_up() after kthread_create() is a bit special, make a simple helper for that. Signed-off-by: Oleg Nesterov Cc: Gautham R Shenoy Cc: "Rafael J. Wysocki" Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/workqueue.c | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) (limited to 'kernel/workqueue.c') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index e858e93886e3..7d1ebfc1a995 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -668,15 +668,21 @@ static int create_workqueue_thread(struct cpu_workqueue_struct *cwq, int cpu) cwq->thread = p; cwq->should_stop = 0; - if (!is_single_threaded(wq)) - kthread_bind(p, cpu); - - if (is_single_threaded(wq) || cpu_online(cpu)) - wake_up_process(p); return 0; } +static void start_workqueue_thread(struct cpu_workqueue_struct *cwq, int cpu) +{ + struct task_struct *p = cwq->thread; + + if (p != NULL) { + if (cpu >= 0) + kthread_bind(p, cpu); + wake_up_process(p); + } +} + struct workqueue_struct *__create_workqueue(const char *name, int singlethread, int freezeable) { @@ -702,6 +708,7 @@ struct workqueue_struct *__create_workqueue(const char *name, if (singlethread) { cwq = init_cpu_workqueue(wq, singlethread_cpu); err = create_workqueue_thread(cwq, singlethread_cpu); + start_workqueue_thread(cwq, -1); } else { mutex_lock(&workqueue_mutex); list_add(&wq->list, &workqueues); @@ -711,6 +718,7 @@ struct workqueue_struct *__create_workqueue(const char *name, if (err || !cpu_online(cpu)) continue; err = create_workqueue_thread(cwq, cpu); + start_workqueue_thread(cwq, cpu); } mutex_unlock(&workqueue_mutex); } @@ -808,12 +816,11 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, return NOTIFY_BAD; case CPU_ONLINE: - wake_up_process(cwq->thread); + start_workqueue_thread(cwq, cpu); break; case CPU_UP_CANCELED: - if (cwq->thread) - wake_up_process(cwq->thread); + start_workqueue_thread(cwq, -1); case CPU_DEAD: cleanup_workqueue_thread(cwq, cpu); break; -- cgit v1.2.3 From ed7c0feede39d70092d048ec30f59bb1df69eec6 Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Wed, 9 May 2007 02:34:16 -0700 Subject: make queue_delayed_work() friendly to flush_fork() Currently typeof(delayed_work->work.data) is "struct workqueue_struct" when the timer is pending "struct cpu_workqueue_struct" whe the work is queued This makes impossible to use flush_fork(delayed_work->work) in addition to cancel_delayed_work/cancel_rearming_delayed_work, not good. Change queue_delayed_work/delayed_work_timer_fn to use cwq, not wq. This complicates (and uglifies) these functions a little bit, but alows us to use flush_fork(dwork) and imho makes the whole code more consistent. Also, document the fact that cancel_rearming_delayed_work() doesn't garantee the completion of work->func() upon return. Signed-off-by: Oleg Nesterov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/workqueue.c | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) (limited to 'kernel/workqueue.c') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 7d1ebfc1a995..d107e1c3b071 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -90,18 +90,20 @@ static const cpumask_t *wq_cpu_map(struct workqueue_struct *wq) * Set the workqueue on which a work item is to be run * - Must *only* be called if the pending flag is set */ -static inline void set_wq_data(struct work_struct *work, void *wq) +static inline void set_wq_data(struct work_struct *work, + struct cpu_workqueue_struct *cwq) { unsigned long new; BUG_ON(!work_pending(work)); - new = (unsigned long) wq | (1UL << WORK_STRUCT_PENDING); + new = (unsigned long) cwq | (1UL << WORK_STRUCT_PENDING); new |= WORK_STRUCT_FLAG_MASK & *work_data_bits(work); atomic_long_set(&work->data, new); } -static inline void *get_wq_data(struct work_struct *work) +static inline +struct cpu_workqueue_struct *get_wq_data(struct work_struct *work) { return (void *) (atomic_long_read(&work->data) & WORK_STRUCT_WQ_DATA_MASK); } @@ -157,7 +159,8 @@ EXPORT_SYMBOL_GPL(queue_work); void delayed_work_timer_fn(unsigned long __data) { struct delayed_work *dwork = (struct delayed_work *)__data; - struct workqueue_struct *wq = get_wq_data(&dwork->work); + struct cpu_workqueue_struct *cwq = get_wq_data(&dwork->work); + struct workqueue_struct *wq = cwq->wq; int cpu = smp_processor_id(); if (unlikely(is_single_threaded(wq))) @@ -189,8 +192,9 @@ int fastcall queue_delayed_work(struct workqueue_struct *wq, BUG_ON(timer_pending(timer)); BUG_ON(!list_empty(&work->entry)); - /* This stores wq for the moment, for the timer_fn */ - set_wq_data(work, wq); + /* This stores cwq for the moment, for the timer_fn */ + set_wq_data(work, + per_cpu_ptr(wq->cpu_wq, raw_smp_processor_id())); timer->expires = jiffies + delay; timer->data = (unsigned long)dwork; timer->function = delayed_work_timer_fn; @@ -221,8 +225,9 @@ int queue_delayed_work_on(int cpu, struct workqueue_struct *wq, BUG_ON(timer_pending(timer)); BUG_ON(!list_empty(&work->entry)); - /* This stores wq for the moment, for the timer_fn */ - set_wq_data(work, wq); + /* This stores cwq for the moment, for the timer_fn */ + set_wq_data(work, + per_cpu_ptr(wq->cpu_wq, raw_smp_processor_id())); timer->expires = jiffies + delay; timer->data = (unsigned long)dwork; timer->function = delayed_work_timer_fn; @@ -562,9 +567,12 @@ void flush_work_keventd(struct work_struct *work) EXPORT_SYMBOL(flush_work_keventd); /** - * cancel_rearming_delayed_workqueue - reliably kill off a delayed work whose handler rearms the delayed work. + * cancel_rearming_delayed_workqueue - kill off a delayed work whose handler rearms the delayed work. * @wq: the controlling workqueue structure * @dwork: the delayed work struct + * + * Note that the work callback function may still be running on return from + * cancel_delayed_work(). Run flush_workqueue() or flush_work() to wait on it. */ void cancel_rearming_delayed_workqueue(struct workqueue_struct *wq, struct delayed_work *dwork) @@ -579,7 +587,7 @@ void cancel_rearming_delayed_workqueue(struct workqueue_struct *wq, EXPORT_SYMBOL(cancel_rearming_delayed_workqueue); /** - * cancel_rearming_delayed_work - reliably kill off a delayed keventd work whose handler rearms the delayed work. + * cancel_rearming_delayed_work - kill off a delayed keventd work whose handler rearms the delayed work. * @dwork: the delayed work struct */ void cancel_rearming_delayed_work(struct delayed_work *dwork) -- cgit v1.2.3 From 63bc0362521cbaae3ed17b8de7b094f9492453f0 Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Wed, 9 May 2007 02:34:16 -0700 Subject: unify queue_delayed_work() and queue_delayed_work_on() Change queue_delayed_work() to use queue_delayed_work_on() to avoid the code duplication (saves 133 bytes). Q: queue_delayed_work() enqueues &dwork->work directly when delay == 0, why? [jirislaby@gmail.com: oops fix] Signed-off-by: Oleg Nesterov Signed-off-by: Jiri Slaby Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/workqueue.c | 32 ++++++++++---------------------- 1 file changed, 10 insertions(+), 22 deletions(-) (limited to 'kernel/workqueue.c') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index d107e1c3b071..0eb9b33f1d91 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -180,28 +180,11 @@ void delayed_work_timer_fn(unsigned long __data) int fastcall queue_delayed_work(struct workqueue_struct *wq, struct delayed_work *dwork, unsigned long delay) { - int ret = 0; - struct timer_list *timer = &dwork->timer; - struct work_struct *work = &dwork->work; - - timer_stats_timer_set_start_info(timer); + timer_stats_timer_set_start_info(&dwork->timer); if (delay == 0) - return queue_work(wq, work); - - if (!test_and_set_bit(WORK_STRUCT_PENDING, work_data_bits(work))) { - BUG_ON(timer_pending(timer)); - BUG_ON(!list_empty(&work->entry)); + return queue_work(wq, &dwork->work); - /* This stores cwq for the moment, for the timer_fn */ - set_wq_data(work, - per_cpu_ptr(wq->cpu_wq, raw_smp_processor_id())); - timer->expires = jiffies + delay; - timer->data = (unsigned long)dwork; - timer->function = delayed_work_timer_fn; - add_timer(timer); - ret = 1; - } - return ret; + return queue_delayed_work_on(-1, wq, dwork, delay); } EXPORT_SYMBOL_GPL(queue_delayed_work); @@ -227,11 +210,16 @@ int queue_delayed_work_on(int cpu, struct workqueue_struct *wq, /* This stores cwq for the moment, for the timer_fn */ set_wq_data(work, - per_cpu_ptr(wq->cpu_wq, raw_smp_processor_id())); + per_cpu_ptr(wq->cpu_wq, wq->singlethread ? + singlethread_cpu : raw_smp_processor_id())); timer->expires = jiffies + delay; timer->data = (unsigned long)dwork; timer->function = delayed_work_timer_fn; - add_timer_on(timer, cpu); + + if (unlikely(cpu >= 0)) + add_timer_on(timer, cpu); + else + add_timer(timer); ret = 1; } return ret; -- cgit v1.2.3 From a848e3b67c07ed79374bd0f9b82f9ce45a419643 Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Wed, 9 May 2007 02:34:17 -0700 Subject: workqueue: introduce wq_per_cpu() helper Cleanup. A number of per_cpu_ptr(wq->cpu_wq, cpu) users have to check that cpu is valid for this wq. Make a simple helper. Signed-off-by: Oleg Nesterov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/workqueue.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) (limited to 'kernel/workqueue.c') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 0eb9b33f1d91..985902e2e071 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -86,6 +86,14 @@ static const cpumask_t *wq_cpu_map(struct workqueue_struct *wq) ? &cpu_singlethread_map : &cpu_populated_map; } +static +struct cpu_workqueue_struct *wq_per_cpu(struct workqueue_struct *wq, int cpu) +{ + if (unlikely(is_single_threaded(wq))) + cpu = singlethread_cpu; + return per_cpu_ptr(wq->cpu_wq, cpu); +} + /* * Set the workqueue on which a work item is to be run * - Must *only* be called if the pending flag is set @@ -142,16 +150,14 @@ static void __queue_work(struct cpu_workqueue_struct *cwq, */ int fastcall queue_work(struct workqueue_struct *wq, struct work_struct *work) { - int ret = 0, cpu = get_cpu(); + int ret = 0; if (!test_and_set_bit(WORK_STRUCT_PENDING, work_data_bits(work))) { - if (unlikely(is_single_threaded(wq))) - cpu = singlethread_cpu; BUG_ON(!list_empty(&work->entry)); - __queue_work(per_cpu_ptr(wq->cpu_wq, cpu), work); + __queue_work(wq_per_cpu(wq, get_cpu()), work); + put_cpu(); ret = 1; } - put_cpu(); return ret; } EXPORT_SYMBOL_GPL(queue_work); @@ -161,12 +167,8 @@ void delayed_work_timer_fn(unsigned long __data) struct delayed_work *dwork = (struct delayed_work *)__data; struct cpu_workqueue_struct *cwq = get_wq_data(&dwork->work); struct workqueue_struct *wq = cwq->wq; - int cpu = smp_processor_id(); - - if (unlikely(is_single_threaded(wq))) - cpu = singlethread_cpu; - __queue_work(per_cpu_ptr(wq->cpu_wq, cpu), &dwork->work); + __queue_work(wq_per_cpu(wq, smp_processor_id()), &dwork->work); } /** @@ -209,9 +211,7 @@ int queue_delayed_work_on(int cpu, struct workqueue_struct *wq, BUG_ON(!list_empty(&work->entry)); /* This stores cwq for the moment, for the timer_fn */ - set_wq_data(work, - per_cpu_ptr(wq->cpu_wq, wq->singlethread ? - singlethread_cpu : raw_smp_processor_id())); + set_wq_data(work, wq_per_cpu(wq, raw_smp_processor_id())); timer->expires = jiffies + delay; timer->data = (unsigned long)dwork; timer->function = delayed_work_timer_fn; -- cgit v1.2.3 From 1634c48f8b85dcb05101f1eb2eab9af40b5976da Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Wed, 9 May 2007 02:34:18 -0700 Subject: make cancel_rearming_delayed_work() work on any workqueue, not just keventd_wq cancel_rearming_delayed_workqueue(wq, dwork) doesn't need the first parameter. We don't hang on un-queued dwork any longer, and work->data doesn't change its type. This means we can always figure out "wq" from dwork when it is needed. Remove this parameter, and rename the function to cancel_rearming_delayed_work(). Re-create an inline "obsolete" cancel_rearming_delayed_workqueue(wq) which just calls cancel_rearming_delayed_work(). Signed-off-by: Oleg Nesterov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/workqueue.c | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) (limited to 'kernel/workqueue.c') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 985902e2e071..41eaffd125ca 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -555,32 +555,23 @@ void flush_work_keventd(struct work_struct *work) EXPORT_SYMBOL(flush_work_keventd); /** - * cancel_rearming_delayed_workqueue - kill off a delayed work whose handler rearms the delayed work. - * @wq: the controlling workqueue structure + * cancel_rearming_delayed_work - kill off a delayed work whose handler rearms the delayed work. * @dwork: the delayed work struct * * Note that the work callback function may still be running on return from * cancel_delayed_work(). Run flush_workqueue() or flush_work() to wait on it. */ -void cancel_rearming_delayed_workqueue(struct workqueue_struct *wq, - struct delayed_work *dwork) +void cancel_rearming_delayed_work(struct delayed_work *dwork) { - /* Was it ever queued ? */ - if (!get_wq_data(&dwork->work)) - return; + struct cpu_workqueue_struct *cwq = get_wq_data(&dwork->work); - while (!cancel_delayed_work(dwork)) - flush_workqueue(wq); -} -EXPORT_SYMBOL(cancel_rearming_delayed_workqueue); + /* Was it ever queued ? */ + if (cwq != NULL) { + struct workqueue_struct *wq = cwq->wq; -/** - * cancel_rearming_delayed_work - kill off a delayed keventd work whose handler rearms the delayed work. - * @dwork: the delayed work struct - */ -void cancel_rearming_delayed_work(struct delayed_work *dwork) -{ - cancel_rearming_delayed_workqueue(keventd_wq, dwork); + while (!cancel_delayed_work(dwork)) + flush_workqueue(wq); + } } EXPORT_SYMBOL(cancel_rearming_delayed_work); -- cgit v1.2.3 From 23b2e5991afde5af91a1a661d7f47ee56120759e Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Wed, 9 May 2007 02:34:19 -0700 Subject: workqueue: kill NOAUTOREL works We don't have any users, and it is not so trivial to use NOAUTOREL works correctly. It is better to simplify API. Delete NOAUTOREL support and rename work_release to work_clear_pending to avoid a confusion. Signed-off-by: Oleg Nesterov Acked-by: David Howells Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/workqueue.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'kernel/workqueue.c') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 41eaffd125ca..0611de815a8f 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -246,8 +246,7 @@ static void run_workqueue(struct cpu_workqueue_struct *cwq) spin_unlock_irq(&cwq->lock); BUG_ON(get_wq_data(work) != cwq); - if (!test_bit(WORK_STRUCT_NOAUTOREL, work_data_bits(work))) - work_release(work); + work_clear_pending(work); f(work); if (unlikely(in_atomic() || lockdep_depth(current) > 0)) { @@ -453,7 +452,7 @@ void flush_work(struct workqueue_struct *wq, struct work_struct *work) */ spin_lock_irq(&cwq->lock); list_del_init(&work->entry); - work_release(work); + work_clear_pending(work); spin_unlock_irq(&cwq->lock); for_each_cpu_mask(cpu, *cpu_map) -- cgit v1.2.3 From b9aac8e0d32499217417ff0b494731811f185b18 Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Wed, 9 May 2007 02:34:20 -0700 Subject: worker_thread: don't play with signals worker_thread() doesn't need to "Block and flush all signals", this was already done by its caller, kthread(). Signed-off-by: Oleg Nesterov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/workqueue.c | 7 ------- 1 file changed, 7 deletions(-) (limited to 'kernel/workqueue.c') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 0611de815a8f..87693b37d017 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -290,18 +290,11 @@ static int worker_thread(void *__cwq) struct cpu_workqueue_struct *cwq = __cwq; DEFINE_WAIT(wait); struct k_sigaction sa; - sigset_t blocked; if (!cwq->wq->freezeable) current->flags |= PF_NOFREEZE; set_user_nice(current, -5); - - /* Block and flush all signals */ - sigfillset(&blocked); - sigprocmask(SIG_BLOCK, &blocked, NULL); - flush_signals(current); - /* * We inherited MPOL_INTERLEAVE from the booting kernel. * Set MPOL_DEFAULT to insure node local allocations. -- cgit v1.2.3 From 85f4186af944c1240c84934a9ab578743df2d69b Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Wed, 9 May 2007 02:34:20 -0700 Subject: worker_thread: fix racy try_to_freeze() usage worker_thread() can miss freeze_process()->signal_wake_up() if it happens between try_to_freeze() and prepare_to_wait(). We should check freezing() before entering schedule(). This race was introduced by me in [PATCH 1/1] workqueue: don't migrate pending works from the dead CPU Looks like mm/vmscan.c:kswapd() has the same race. Signed-off-by: Oleg Nesterov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/workqueue.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'kernel/workqueue.c') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 87693b37d017..63885abf1ba0 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -308,14 +308,14 @@ static int worker_thread(void *__cwq) do_sigaction(SIGCHLD, &sa, (struct k_sigaction *)0); for (;;) { - if (cwq->wq->freezeable) - try_to_freeze(); - prepare_to_wait(&cwq->more_work, &wait, TASK_INTERRUPTIBLE); - if (!cwq->should_stop && list_empty(&cwq->worklist)) + if (!freezing(current) && !cwq->should_stop + && list_empty(&cwq->worklist)) schedule(); finish_wait(&cwq->more_work, &wait); + try_to_freeze(); + if (cwq_should_stop(cwq)) break; -- cgit v1.2.3 From 28e53bddf814485699a4142bc056fd37d4e11dd4 Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Wed, 9 May 2007 02:34:22 -0700 Subject: unify flush_work/flush_work_keventd and rename it to cancel_work_sync flush_work(wq, work) doesn't need the first parameter, we can use cwq->wq (this was possible from the very beginnig, I missed this). So we can unify flush_work_keventd and flush_work. Also, rename flush_work() to cancel_work_sync() and fix all callers. Perhaps this is not the best name, but "flush_work" is really bad. (akpm: this is why the earlier patches bypassed maintainers) Signed-off-by: Oleg Nesterov Cc: Jeff Garzik Cc: "David S. Miller" Cc: Jens Axboe Cc: Tejun Heo Cc: Auke Kok , Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/workqueue.c | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) (limited to 'kernel/workqueue.c') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 63885abf1ba0..c9ab4293904f 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -413,23 +413,23 @@ static void wait_on_work(struct cpu_workqueue_struct *cwq, } /** - * flush_work - block until a work_struct's callback has terminated - * @wq: the workqueue on which the work is queued + * cancel_work_sync - block until a work_struct's callback has terminated * @work: the work which is to be flushed * - * flush_work() will attempt to cancel the work if it is queued. If the work's - * callback appears to be running, flush_work() will block until it has - * completed. + * cancel_work_sync() will attempt to cancel the work if it is queued. If the + * work's callback appears to be running, cancel_work_sync() will block until + * it has completed. * - * flush_work() is designed to be used when the caller is tearing down data - * structures which the callback function operates upon. It is expected that, - * prior to calling flush_work(), the caller has arranged for the work to not - * be requeued. + * cancel_work_sync() is designed to be used when the caller is tearing down + * data structures which the callback function operates upon. It is expected + * that, prior to calling cancel_work_sync(), the caller has arranged for the + * work to not be requeued. */ -void flush_work(struct workqueue_struct *wq, struct work_struct *work) +void cancel_work_sync(struct work_struct *work) { - const cpumask_t *cpu_map = wq_cpu_map(wq); struct cpu_workqueue_struct *cwq; + struct workqueue_struct *wq; + const cpumask_t *cpu_map; int cpu; might_sleep(); @@ -448,10 +448,13 @@ void flush_work(struct workqueue_struct *wq, struct work_struct *work) work_clear_pending(work); spin_unlock_irq(&cwq->lock); + wq = cwq->wq; + cpu_map = wq_cpu_map(wq); + for_each_cpu_mask(cpu, *cpu_map) wait_on_work(per_cpu_ptr(wq->cpu_wq, cpu), work); } -EXPORT_SYMBOL_GPL(flush_work); +EXPORT_SYMBOL_GPL(cancel_work_sync); static struct workqueue_struct *keventd_wq; @@ -540,18 +543,13 @@ void flush_scheduled_work(void) } EXPORT_SYMBOL(flush_scheduled_work); -void flush_work_keventd(struct work_struct *work) -{ - flush_work(keventd_wq, work); -} -EXPORT_SYMBOL(flush_work_keventd); - /** * cancel_rearming_delayed_work - kill off a delayed work whose handler rearms the delayed work. * @dwork: the delayed work struct * * Note that the work callback function may still be running on return from - * cancel_delayed_work(). Run flush_workqueue() or flush_work() to wait on it. + * cancel_delayed_work(). Run flush_workqueue() or cancel_work_sync() to wait + * on it. */ void cancel_rearming_delayed_work(struct delayed_work *dwork) { -- cgit v1.2.3 From 5de18d169739293e27e0cf9acfc75a2d2f4aa572 Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Wed, 9 May 2007 02:34:36 -0700 Subject: worker_thread: don't play with SIGCHLD and numa policy worker_thread() inherits ignored SIGCHLD and numa_default_policy() from its parent, kthreadd. No need to setup this again. Signed-off-by: Oleg Nesterov Acked-by: "Eric W. Biederman" Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/workqueue.c | 12 ------------ 1 file changed, 12 deletions(-) (limited to 'kernel/workqueue.c') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index c9ab4293904f..25cee1afe6fb 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -289,23 +289,11 @@ static int worker_thread(void *__cwq) { struct cpu_workqueue_struct *cwq = __cwq; DEFINE_WAIT(wait); - struct k_sigaction sa; if (!cwq->wq->freezeable) current->flags |= PF_NOFREEZE; set_user_nice(current, -5); - /* - * We inherited MPOL_INTERLEAVE from the booting kernel. - * Set MPOL_DEFAULT to insure node local allocations. - */ - numa_default_policy(); - - /* SIG_IGN makes children autoreap: see do_notify_parent(). */ - sa.sa.sa_handler = SIG_IGN; - sa.sa.sa_flags = 0; - siginitset(&sa.sa.sa_mask, sigmask(SIGCHLD)); - do_sigaction(SIGCHLD, &sa, (struct k_sigaction *)0); for (;;) { prepare_to_wait(&cwq->more_work, &wait, TASK_INTERRUPTIBLE); -- cgit v1.2.3 From 6e84d644b5929789398914b0ccf447355dec6fb0 Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Wed, 9 May 2007 02:34:46 -0700 Subject: make cancel_rearming_delayed_work() reliable Thanks to Jarek Poplawski for the ideas and for spotting the bug in the initial draft patch. cancel_rearming_delayed_work() currently has many limitations, because it requires that dwork always re-arms itself via queue_delayed_work(). So it hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/ cancel_delayed_work was already called. It uses flush_workqueue() in a loop, so it can't be used if workqueue was freezed, and it is potentially live- lockable on busy system if delay is small. With this patch cancel_rearming_delayed_work() doesn't make any assumptions about dwork, it can re-arm itself via queue_delayed_work(), or queue_work(), or do nothing. As a "side effect", cancel_work_sync() was changed to handle re-arming works as well. Disadvantages: - this patch adds wmb() to insert_work(). - slowdowns the fast path (when del_timer() succeeds on entry) of cancel_rearming_delayed_work(), because wait_on_work() is called unconditionally. In that case, compared to the old version, we are doing "unneeded" lock/unlock for each online CPU. On the other hand, this means we don't need to use cancel_work_sync() after cancel_rearming_delayed_work(). - complicates the code (.text grows by 130 bytes). [akpm@linux-foundation.org: fix speling] Signed-off-by: Oleg Nesterov Cc: David Chinner Cc: David Howells Cc: Gautham Shenoy Acked-by: Jarek Poplawski Cc: Srivatsa Vaddagiri Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/workqueue.c | 140 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 91 insertions(+), 49 deletions(-) (limited to 'kernel/workqueue.c') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 25cee1afe6fb..b976ed87dd37 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -120,6 +120,11 @@ static void insert_work(struct cpu_workqueue_struct *cwq, struct work_struct *work, int tail) { set_wq_data(work, cwq); + /* + * Ensure that we get the right work->data if we see the + * result of list_add() below, see try_to_grab_pending(). + */ + smp_wmb(); if (tail) list_add_tail(&work->entry, &cwq->worklist); else @@ -383,7 +388,46 @@ void fastcall flush_workqueue(struct workqueue_struct *wq) } EXPORT_SYMBOL_GPL(flush_workqueue); -static void wait_on_work(struct cpu_workqueue_struct *cwq, +/* + * Upon a successful return, the caller "owns" WORK_STRUCT_PENDING bit, + * so this work can't be re-armed in any way. + */ +static int try_to_grab_pending(struct work_struct *work) +{ + struct cpu_workqueue_struct *cwq; + int ret = 0; + + if (!test_and_set_bit(WORK_STRUCT_PENDING, work_data_bits(work))) + return 1; + + /* + * The queueing is in progress, or it is already queued. Try to + * steal it from ->worklist without clearing WORK_STRUCT_PENDING. + */ + + cwq = get_wq_data(work); + if (!cwq) + return ret; + + spin_lock_irq(&cwq->lock); + if (!list_empty(&work->entry)) { + /* + * This work is queued, but perhaps we locked the wrong cwq. + * In that case we must see the new value after rmb(), see + * insert_work()->wmb(). + */ + smp_rmb(); + if (cwq == get_wq_data(work)) { + list_del_init(&work->entry); + ret = 1; + } + } + spin_unlock_irq(&cwq->lock); + + return ret; +} + +static void wait_on_cpu_work(struct cpu_workqueue_struct *cwq, struct work_struct *work) { struct wq_barrier barr; @@ -400,20 +444,7 @@ static void wait_on_work(struct cpu_workqueue_struct *cwq, wait_for_completion(&barr.done); } -/** - * cancel_work_sync - block until a work_struct's callback has terminated - * @work: the work which is to be flushed - * - * cancel_work_sync() will attempt to cancel the work if it is queued. If the - * work's callback appears to be running, cancel_work_sync() will block until - * it has completed. - * - * cancel_work_sync() is designed to be used when the caller is tearing down - * data structures which the callback function operates upon. It is expected - * that, prior to calling cancel_work_sync(), the caller has arranged for the - * work to not be requeued. - */ -void cancel_work_sync(struct work_struct *work) +static void wait_on_work(struct work_struct *work) { struct cpu_workqueue_struct *cwq; struct workqueue_struct *wq; @@ -423,29 +454,62 @@ void cancel_work_sync(struct work_struct *work) might_sleep(); cwq = get_wq_data(work); - /* Was it ever queued ? */ if (!cwq) return; - /* - * This work can't be re-queued, no need to re-check that - * get_wq_data() is still the same when we take cwq->lock. - */ - spin_lock_irq(&cwq->lock); - list_del_init(&work->entry); - work_clear_pending(work); - spin_unlock_irq(&cwq->lock); - wq = cwq->wq; cpu_map = wq_cpu_map(wq); for_each_cpu_mask(cpu, *cpu_map) - wait_on_work(per_cpu_ptr(wq->cpu_wq, cpu), work); + wait_on_cpu_work(per_cpu_ptr(wq->cpu_wq, cpu), work); +} + +/** + * cancel_work_sync - block until a work_struct's callback has terminated + * @work: the work which is to be flushed + * + * cancel_work_sync() will cancel the work if it is queued. If the work's + * callback appears to be running, cancel_work_sync() will block until it + * has completed. + * + * It is possible to use this function if the work re-queues itself. It can + * cancel the work even if it migrates to another workqueue, however in that + * case it only guarantees that work->func() has completed on the last queued + * workqueue. + * + * cancel_work_sync(&delayed_work->work) should be used only if ->timer is not + * pending, otherwise it goes into a busy-wait loop until the timer expires. + * + * The caller must ensure that workqueue_struct on which this work was last + * queued can't be destroyed before this function returns. + */ +void cancel_work_sync(struct work_struct *work) +{ + while (!try_to_grab_pending(work)) + cpu_relax(); + wait_on_work(work); + work_clear_pending(work); } EXPORT_SYMBOL_GPL(cancel_work_sync); +/** + * cancel_rearming_delayed_work - reliably kill off a delayed work. + * @dwork: the delayed work struct + * + * It is possible to use this function if @dwork rearms itself via queue_work() + * or queue_delayed_work(). See also the comment for cancel_work_sync(). + */ +void cancel_rearming_delayed_work(struct delayed_work *dwork) +{ + while (!del_timer(&dwork->timer) && + !try_to_grab_pending(&dwork->work)) + cpu_relax(); + wait_on_work(&dwork->work); + work_clear_pending(&dwork->work); +} +EXPORT_SYMBOL(cancel_rearming_delayed_work); -static struct workqueue_struct *keventd_wq; +static struct workqueue_struct *keventd_wq __read_mostly; /** * schedule_work - put work task in global workqueue @@ -531,28 +595,6 @@ void flush_scheduled_work(void) } EXPORT_SYMBOL(flush_scheduled_work); -/** - * cancel_rearming_delayed_work - kill off a delayed work whose handler rearms the delayed work. - * @dwork: the delayed work struct - * - * Note that the work callback function may still be running on return from - * cancel_delayed_work(). Run flush_workqueue() or cancel_work_sync() to wait - * on it. - */ -void cancel_rearming_delayed_work(struct delayed_work *dwork) -{ - struct cpu_workqueue_struct *cwq = get_wq_data(&dwork->work); - - /* Was it ever queued ? */ - if (cwq != NULL) { - struct workqueue_struct *wq = cwq->wq; - - while (!cancel_delayed_work(dwork)) - flush_workqueue(wq); - } -} -EXPORT_SYMBOL(cancel_rearming_delayed_work); - /** * execute_in_process_context - reliably execute the routine with user context * @fn: the function to execute -- cgit v1.2.3 From 8bb7844286fb8c9fce6f65d8288aeb09d03a5e0d Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Wed, 9 May 2007 02:35:10 -0700 Subject: Add suspend-related notifications for CPU hotplug Since nonboot CPUs are now disabled after tasks and devices have been frozen and the CPU hotplug infrastructure is used for this purpose, we need special CPU hotplug notifications that will help the CPU-hotplug-aware subsystems distinguish normal CPU hotplug events from CPU hotplug events related to a system-wide suspend or resume operation in progress. This patch introduces such notifications and causes them to be used during suspend and resume transitions. It also changes all of the CPU-hotplug-aware subsystems to take these notifications into consideration (for now they are handled in the same way as the corresponding "normal" ones). [oleg@tv-sign.ru: cleanups] Signed-off-by: Rafael J. Wysocki Cc: Gautham R Shenoy Cc: Pavel Machek Signed-off-by: Oleg Nesterov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/workqueue.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'kernel/workqueue.c') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index b976ed87dd37..fb56fedd5c02 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -799,6 +799,8 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb, struct cpu_workqueue_struct *cwq; struct workqueue_struct *wq; + action &= ~CPU_TASKS_FROZEN; + switch (action) { case CPU_LOCK_ACQUIRE: mutex_lock(&workqueue_mutex); -- cgit v1.2.3