From c4a0a5d964e90b93eb4101c3927b788e083e530f Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Wed, 18 Dec 2013 13:53:32 -0700 Subject: PCI: Move device_del() from pci_stop_dev() to pci_destroy_dev() After commit bcdde7e221a8 (sysfs: make __sysfs_remove_dir() recursive) I'm seeing traces analogous to the one below in Thunderbolt testing: WARNING: CPU: 3 PID: 76 at /scratch/rafael/work/linux-pm/fs/sysfs/group.c:214 sysfs_remove_group+0x59/0xe0() sysfs group ffffffff81c6c500 not found for kobject '0000:08' Modules linked in: ... CPU: 3 PID: 76 Comm: kworker/u16:7 Not tainted 3.13.0-rc1+ #76 Hardware name: Acer Aspire S5-391/Venus , BIOS V1.02 05/29/2012 Workqueue: kacpi_hotplug acpi_hotplug_work_fn 0000000000000009 ffff8801644b9ac8 ffffffff816b23bf 0000000000000007 ffff8801644b9b18 ffff8801644b9b08 ffffffff81046607 ffff88016925b800 0000000000000000 ffffffff81c6c500 ffff88016924f928 ffff88016924f800 Call Trace: [] dump_stack+0x4e/0x71 [] warn_slowpath_common+0x87/0xb0 [] warn_slowpath_fmt+0x41/0x50 [] ? sysfs_get_dirent_ns+0x6f/0x80 [] sysfs_remove_group+0x59/0xe0 [] dpm_sysfs_remove+0x3b/0x50 [] device_del+0x58/0x1c0 [] device_unregister+0x48/0x60 [] pci_remove_bus+0x6e/0x80 [] pci_remove_bus_device+0x38/0x110 [] pci_remove_bus_device+0x4d/0x110 [] pci_stop_and_remove_bus_device+0x19/0x20 [] disable_slot+0x20/0xe0 [] acpiphp_check_bridge+0xa8/0xd0 [] hotplug_event+0x17d/0x220 [] hotplug_event_work+0x30/0x70 [] acpi_hotplug_work_fn+0x18/0x24 [] process_one_work+0x261/0x450 [] worker_thread+0x21e/0x370 [] ? rescuer_thread+0x300/0x300 [] kthread+0xd2/0xe0 [] ? flush_kthread_worker+0x70/0x70 [] ret_from_fork+0x7c/0xb0 [] ? flush_kthread_worker+0x70/0x70 (Mika Westerberg sees them too in his tests). Some investigation documented in kernel bug #65281 led me to the conclusion that the source of the problem is the device_del() in pci_stop_dev() as it now causes the sysfs directory of the device to be removed recursively along with all of its subdirectories. That includes the sysfs directory of the device's subordinate bus (dev->subordinate) and its "power" group. Consequently, when pci_remove_bus() is called for dev->subordinate in pci_remove_bus_device(), it calls device_unregister(&bus->dev), but at this point the sysfs directory of bus->dev doesn't exist any more and its "power" group doesn't exist either. Thus, when dpm_sysfs_remove() called from device_del() tries to remove that group, it triggers the above warning. That indicates a logical mistake in the design of pci_stop_and_remove_bus_device(), which causes bus device objects to be left behind their parents (bridge device objects) and can be fixed by moving the device_del() from pci_stop_dev() into pci_destroy_dev(), so pci_remove_bus() can be called for the device's subordinate bus before the device itself is unregistered from the hierarchy. Still, the driver, if any, should be detached from the device in pci_stop_dev(), so use device_release_driver() directly from there. References: https://bugzilla.kernel.org/show_bug.cgi?id=65281#c6 Reported-by: Mika Westerberg Signed-off-by: Rafael J. Wysocki Signed-off-by: Bjorn Helgaas --- drivers/pci/remove.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'drivers/pci/remove.c') diff --git a/drivers/pci/remove.c b/drivers/pci/remove.c index 1576851028db..cc9337a71529 100644 --- a/drivers/pci/remove.c +++ b/drivers/pci/remove.c @@ -24,7 +24,7 @@ static void pci_stop_dev(struct pci_dev *dev) if (dev->is_added) { pci_proc_detach_device(dev); pci_remove_sysfs_dev_files(dev); - device_del(&dev->dev); + device_release_driver(&dev->dev); dev->is_added = 0; } @@ -34,6 +34,8 @@ static void pci_stop_dev(struct pci_dev *dev) static void pci_destroy_dev(struct pci_dev *dev) { + device_del(&dev->dev); + down_write(&pci_bus_sem); list_del(&dev->bus_list); up_write(&pci_bus_sem); -- cgit v1.2.3 From e3b439e1d315aff59c1b4f0fc43c5cd2d82b4138 Mon Sep 17 00:00:00 2001 From: Yinghai Lu Date: Sat, 30 Nov 2013 14:40:27 -0800 Subject: PCI: Use device_release_driver() in pci_stop_root_bus() To be consistent with 4bff6749905d ("PCI: Move device_del() from pci_stop_dev() to pci_destroy_dev()", this changes pci_stop_root_bus() to use device_release_driver() instead of device_del(). This also changes pci_remove_root_bus() to use device_unregister() instead of put_device() so it corresponds with the device_register() call in pci_create_root_bus(). [bhelgaas: changelog] Signed-off-by: Yinghai Lu Signed-off-by: Bjorn Helgaas Acked-by: Rafael J. Wysocki --- drivers/pci/remove.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers/pci/remove.c') diff --git a/drivers/pci/remove.c b/drivers/pci/remove.c index cc9337a71529..692f4c39ac48 100644 --- a/drivers/pci/remove.c +++ b/drivers/pci/remove.c @@ -128,7 +128,7 @@ void pci_stop_root_bus(struct pci_bus *bus) pci_stop_bus_device(child); /* stop the host bridge */ - device_del(&host_bridge->dev); + device_release_driver(&host_bridge->dev); } void pci_remove_root_bus(struct pci_bus *bus) @@ -147,5 +147,5 @@ void pci_remove_root_bus(struct pci_bus *bus) host_bridge->bus = NULL; /* remove the host bridge */ - put_device(&host_bridge->dev); + device_unregister(&host_bridge->dev); } -- cgit v1.2.3 From ef83b0781a73f9efcb1228256bfdfb97fc9533a8 Mon Sep 17 00:00:00 2001 From: Yinghai Lu Date: Sat, 30 Nov 2013 14:40:29 -0800 Subject: PCI: Remove from bus_list and release resources in pci_release_dev() Previously we removed the pci_dev from the bus_list and released its resources in pci_destroy_dev(). But that's too early: it's possible to call pci_destroy_dev() twice for the same device (e.g., via sysfs), and that will cause an oops when we try to remove it from bus_list the second time. We should remove it from the bus_list only when the last reference to the pci_dev has been released, i.e., in pci_release_dev(). [bhelgaas: changelog] Signed-off-by: Yinghai Lu Signed-off-by: Bjorn Helgaas --- drivers/pci/remove.c | 19 ------------------- 1 file changed, 19 deletions(-) (limited to 'drivers/pci/remove.c') diff --git a/drivers/pci/remove.c b/drivers/pci/remove.c index 692f4c39ac48..f452148e6d55 100644 --- a/drivers/pci/remove.c +++ b/drivers/pci/remove.c @@ -3,20 +3,6 @@ #include #include "pci.h" -static void pci_free_resources(struct pci_dev *dev) -{ - int i; - - msi_remove_pci_irq_vectors(dev); - - pci_cleanup_rom(dev); - for (i = 0; i < PCI_NUM_RESOURCES; i++) { - struct resource *res = dev->resource + i; - if (res->parent) - release_resource(res); - } -} - static void pci_stop_dev(struct pci_dev *dev) { pci_pme_active(dev, false); @@ -36,11 +22,6 @@ static void pci_destroy_dev(struct pci_dev *dev) { device_del(&dev->dev); - down_write(&pci_bus_sem); - list_del(&dev->bus_list); - up_write(&pci_bus_sem); - - pci_free_resources(dev); put_device(&dev->dev); } -- cgit v1.2.3 From 9d16947b75831acd317ab9a53e0e94d160731d33 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Fri, 10 Jan 2014 15:22:18 +0100 Subject: PCI: Add global pci_lock_rescan_remove() There are multiple PCI device addition and removal code paths that may be run concurrently with the generic PCI bus rescan and device removal that can be triggered via sysfs. If that happens, it may lead to multiple different, potentially dangerous race conditions. The most straightforward way to address those problems is to run the code in question under the same lock that is used by the generic rescan/remove code in pci-sysfs.c. To prepare for those changes, move the definition of the global PCI remove/rescan lock to probe.c and provide global wrappers, pci_lock_rescan_remove() and pci_unlock_rescan_remove(), allowing drivers to manipulate that lock. Also provide pci_stop_and_remove_bus_device_locked() for the callers of pci_stop_and_remove_bus_device() who only need to hold the rescan/remove lock around it. Signed-off-by: Rafael J. Wysocki Signed-off-by: Bjorn Helgaas --- drivers/pci/remove.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'drivers/pci/remove.c') diff --git a/drivers/pci/remove.c b/drivers/pci/remove.c index f452148e6d55..10fa13f9e309 100644 --- a/drivers/pci/remove.c +++ b/drivers/pci/remove.c @@ -95,6 +95,14 @@ void pci_stop_and_remove_bus_device(struct pci_dev *dev) } EXPORT_SYMBOL(pci_stop_and_remove_bus_device); +void pci_stop_and_remove_bus_device_locked(struct pci_dev *dev) +{ + pci_lock_rescan_remove(); + pci_stop_and_remove_bus_device(dev); + pci_unlock_rescan_remove(); +} +EXPORT_SYMBOL_GPL(pci_stop_and_remove_bus_device_locked); + void pci_stop_root_bus(struct pci_bus *bus) { struct pci_dev *child, *tmp; -- cgit v1.2.3 From 8a4c5c329de716996eea03d93753ccbb5406072b Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Tue, 14 Jan 2014 12:04:51 -0700 Subject: PCI: Check parent kobject in pci_destroy_dev() If pci_stop_and_remove_bus_device() is run concurrently for a device and its parent bridge via remove_callback(), both code paths attempt to acquire pci_rescan_remove_lock. If the child device removal acquires it first, there will be no problems. However, if the parent bridge removal acquires it first, it will eventually execute pci_destroy_dev() for the child device, but that device object will not be freed yet due to the reference held by the concurrent child removal. Consequently, both pci_stop_bus_device() and pci_remove_bus_device() will be executed for that device unnecessarily and pci_destroy_dev() will see a corrupted list head in that object. Moreover, an excess put_device() will be executed for that device in that case which may lead to a use-after-free in the final kobject_put() done by sysfs_schedule_callback_work(). To avoid that problem, make pci_destroy_dev() check if the device's parent kobject is NULL, which only happens after device_del() has already run for it. Make pci_destroy_dev() return immediately whithout doing anything in that case. Signed-off-by: Rafael J. Wysocki Signed-off-by: Bjorn Helgaas --- drivers/pci/remove.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'drivers/pci/remove.c') diff --git a/drivers/pci/remove.c b/drivers/pci/remove.c index 10fa13f9e309..4ff36bfa785e 100644 --- a/drivers/pci/remove.c +++ b/drivers/pci/remove.c @@ -20,6 +20,9 @@ static void pci_stop_dev(struct pci_dev *dev) static void pci_destroy_dev(struct pci_dev *dev) { + if (!dev->dev.kobj.parent) + return; + device_del(&dev->dev); put_device(&dev->dev); -- cgit v1.2.3 From 04480094de7242d08bb62088e713fd7fe00443b4 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Sat, 1 Feb 2014 15:38:29 +0100 Subject: Revert "PCI: Remove from bus_list and release resources in pci_release_dev()" Revert commit ef83b0781a73 "PCI: Remove from bus_list and release resources in pci_release_dev()" that made some nasty race conditions become possible. For example, if a Thunderbolt link is unplugged and then replugged immediately, the pci_release_dev() resulting from the hot-remove code path may be racing with the hot-add code path which after that commit causes various kinds of breakage to happen (up to and including a hard crash of the whole system). Moreover, the problem that commit ef83b0781a73 attempted to address cannot happen any more after commit 8a4c5c329de7 "PCI: Check parent kobject in pci_destroy_dev()", because pci_destroy_dev() will now return immediately if it has already been executed for the given device. Note, however, that the invocation of msi_remove_pci_irq_vectors() removed by commit ef83b0781a73 from pci_free_resources() along with the other changes made by it is not added back because of subsequent code changes depending on that modification. Fixes: ef83b0781a73 (PCI: Remove from bus_list and release resources in pci_release_dev()) Reported-by: Mika Westerberg Signed-off-by: Rafael J. Wysocki Signed-off-by: Linus Torvalds --- drivers/pci/remove.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'drivers/pci/remove.c') diff --git a/drivers/pci/remove.c b/drivers/pci/remove.c index 4ff36bfa785e..8bd76c9ba21c 100644 --- a/drivers/pci/remove.c +++ b/drivers/pci/remove.c @@ -3,6 +3,18 @@ #include #include "pci.h" +static void pci_free_resources(struct pci_dev *dev) +{ + int i; + + pci_cleanup_rom(dev); + for (i = 0; i < PCI_NUM_RESOURCES; i++) { + struct resource *res = dev->resource + i; + if (res->parent) + release_resource(res); + } +} + static void pci_stop_dev(struct pci_dev *dev) { pci_pme_active(dev, false); @@ -25,6 +37,11 @@ static void pci_destroy_dev(struct pci_dev *dev) device_del(&dev->dev); + down_write(&pci_bus_sem); + list_del(&dev->bus_list); + up_write(&pci_bus_sem); + + pci_free_resources(dev); put_device(&dev->dev); } -- cgit v1.2.3