driver core: Probe devices asynchronously instead of the driver
[sfrench/cifs-2.6.git] / drivers / base / dd.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * drivers/base/dd.c - The core device/driver interactions.
4  *
5  * This file contains the (sometimes tricky) code that controls the
6  * interactions between devices and drivers, which primarily includes
7  * driver binding and unbinding.
8  *
9  * All of this code used to exist in drivers/base/bus.c, but was
10  * relocated to here in the name of compartmentalization (since it wasn't
11  * strictly code just for the 'struct bus_type'.
12  *
13  * Copyright (c) 2002-5 Patrick Mochel
14  * Copyright (c) 2002-3 Open Source Development Labs
15  * Copyright (c) 2007-2009 Greg Kroah-Hartman <gregkh@suse.de>
16  * Copyright (c) 2007-2009 Novell Inc.
17  */
18
19 #include <linux/debugfs.h>
20 #include <linux/device.h>
21 #include <linux/delay.h>
22 #include <linux/dma-mapping.h>
23 #include <linux/init.h>
24 #include <linux/module.h>
25 #include <linux/kthread.h>
26 #include <linux/wait.h>
27 #include <linux/async.h>
28 #include <linux/pm_runtime.h>
29 #include <linux/pinctrl/devinfo.h>
30
31 #include "base.h"
32 #include "power/power.h"
33
34 /*
35  * Deferred Probe infrastructure.
36  *
37  * Sometimes driver probe order matters, but the kernel doesn't always have
38  * dependency information which means some drivers will get probed before a
39  * resource it depends on is available.  For example, an SDHCI driver may
40  * first need a GPIO line from an i2c GPIO controller before it can be
41  * initialized.  If a required resource is not available yet, a driver can
42  * request probing to be deferred by returning -EPROBE_DEFER from its probe hook
43  *
44  * Deferred probe maintains two lists of devices, a pending list and an active
45  * list.  A driver returning -EPROBE_DEFER causes the device to be added to the
46  * pending list.  A successful driver probe will trigger moving all devices
47  * from the pending to the active list so that the workqueue will eventually
48  * retry them.
49  *
50  * The deferred_probe_mutex must be held any time the deferred_probe_*_list
51  * of the (struct device*)->p->deferred_probe pointers are manipulated
52  */
53 static DEFINE_MUTEX(deferred_probe_mutex);
54 static LIST_HEAD(deferred_probe_pending_list);
55 static LIST_HEAD(deferred_probe_active_list);
56 static atomic_t deferred_trigger_count = ATOMIC_INIT(0);
57 static struct dentry *deferred_devices;
58 static bool initcalls_done;
59
60 /*
61  * In some cases, like suspend to RAM or hibernation, It might be reasonable
62  * to prohibit probing of devices as it could be unsafe.
63  * Once defer_all_probes is true all drivers probes will be forcibly deferred.
64  */
65 static bool defer_all_probes;
66
67 /*
68  * deferred_probe_work_func() - Retry probing devices in the active list.
69  */
70 static void deferred_probe_work_func(struct work_struct *work)
71 {
72         struct device *dev;
73         struct device_private *private;
74         /*
75          * This block processes every device in the deferred 'active' list.
76          * Each device is removed from the active list and passed to
77          * bus_probe_device() to re-attempt the probe.  The loop continues
78          * until every device in the active list is removed and retried.
79          *
80          * Note: Once the device is removed from the list and the mutex is
81          * released, it is possible for the device get freed by another thread
82          * and cause a illegal pointer dereference.  This code uses
83          * get/put_device() to ensure the device structure cannot disappear
84          * from under our feet.
85          */
86         mutex_lock(&deferred_probe_mutex);
87         while (!list_empty(&deferred_probe_active_list)) {
88                 private = list_first_entry(&deferred_probe_active_list,
89                                         typeof(*dev->p), deferred_probe);
90                 dev = private->device;
91                 list_del_init(&private->deferred_probe);
92
93                 get_device(dev);
94
95                 /*
96                  * Drop the mutex while probing each device; the probe path may
97                  * manipulate the deferred list
98                  */
99                 mutex_unlock(&deferred_probe_mutex);
100
101                 /*
102                  * Force the device to the end of the dpm_list since
103                  * the PM code assumes that the order we add things to
104                  * the list is a good order for suspend but deferred
105                  * probe makes that very unsafe.
106                  */
107                 device_pm_move_to_tail(dev);
108
109                 dev_dbg(dev, "Retrying from deferred list\n");
110                 bus_probe_device(dev);
111                 mutex_lock(&deferred_probe_mutex);
112
113                 put_device(dev);
114         }
115         mutex_unlock(&deferred_probe_mutex);
116 }
117 static DECLARE_WORK(deferred_probe_work, deferred_probe_work_func);
118
119 static void driver_deferred_probe_add(struct device *dev)
120 {
121         mutex_lock(&deferred_probe_mutex);
122         if (list_empty(&dev->p->deferred_probe)) {
123                 dev_dbg(dev, "Added to deferred list\n");
124                 list_add_tail(&dev->p->deferred_probe, &deferred_probe_pending_list);
125         }
126         mutex_unlock(&deferred_probe_mutex);
127 }
128
129 void driver_deferred_probe_del(struct device *dev)
130 {
131         mutex_lock(&deferred_probe_mutex);
132         if (!list_empty(&dev->p->deferred_probe)) {
133                 dev_dbg(dev, "Removed from deferred list\n");
134                 list_del_init(&dev->p->deferred_probe);
135         }
136         mutex_unlock(&deferred_probe_mutex);
137 }
138
139 static bool driver_deferred_probe_enable = false;
140 /**
141  * driver_deferred_probe_trigger() - Kick off re-probing deferred devices
142  *
143  * This functions moves all devices from the pending list to the active
144  * list and schedules the deferred probe workqueue to process them.  It
145  * should be called anytime a driver is successfully bound to a device.
146  *
147  * Note, there is a race condition in multi-threaded probe. In the case where
148  * more than one device is probing at the same time, it is possible for one
149  * probe to complete successfully while another is about to defer. If the second
150  * depends on the first, then it will get put on the pending list after the
151  * trigger event has already occurred and will be stuck there.
152  *
153  * The atomic 'deferred_trigger_count' is used to determine if a successful
154  * trigger has occurred in the midst of probing a driver. If the trigger count
155  * changes in the midst of a probe, then deferred processing should be triggered
156  * again.
157  */
158 static void driver_deferred_probe_trigger(void)
159 {
160         if (!driver_deferred_probe_enable)
161                 return;
162
163         /*
164          * A successful probe means that all the devices in the pending list
165          * should be triggered to be reprobed.  Move all the deferred devices
166          * into the active list so they can be retried by the workqueue
167          */
168         mutex_lock(&deferred_probe_mutex);
169         atomic_inc(&deferred_trigger_count);
170         list_splice_tail_init(&deferred_probe_pending_list,
171                               &deferred_probe_active_list);
172         mutex_unlock(&deferred_probe_mutex);
173
174         /*
175          * Kick the re-probe thread.  It may already be scheduled, but it is
176          * safe to kick it again.
177          */
178         schedule_work(&deferred_probe_work);
179 }
180
181 /**
182  * device_block_probing() - Block/defer device's probes
183  *
184  *      It will disable probing of devices and defer their probes instead.
185  */
186 void device_block_probing(void)
187 {
188         defer_all_probes = true;
189         /* sync with probes to avoid races. */
190         wait_for_device_probe();
191 }
192
193 /**
194  * device_unblock_probing() - Unblock/enable device's probes
195  *
196  *      It will restore normal behavior and trigger re-probing of deferred
197  * devices.
198  */
199 void device_unblock_probing(void)
200 {
201         defer_all_probes = false;
202         driver_deferred_probe_trigger();
203 }
204
205 /*
206  * deferred_devs_show() - Show the devices in the deferred probe pending list.
207  */
208 static int deferred_devs_show(struct seq_file *s, void *data)
209 {
210         struct device_private *curr;
211
212         mutex_lock(&deferred_probe_mutex);
213
214         list_for_each_entry(curr, &deferred_probe_pending_list, deferred_probe)
215                 seq_printf(s, "%s\n", dev_name(curr->device));
216
217         mutex_unlock(&deferred_probe_mutex);
218
219         return 0;
220 }
221 DEFINE_SHOW_ATTRIBUTE(deferred_devs);
222
223 static int deferred_probe_timeout = -1;
224 static int __init deferred_probe_timeout_setup(char *str)
225 {
226         int timeout;
227
228         if (!kstrtoint(str, 10, &timeout))
229                 deferred_probe_timeout = timeout;
230         return 1;
231 }
232 __setup("deferred_probe_timeout=", deferred_probe_timeout_setup);
233
234 /**
235  * driver_deferred_probe_check_state() - Check deferred probe state
236  * @dev: device to check
237  *
238  * Returns -ENODEV if init is done and all built-in drivers have had a chance
239  * to probe (i.e. initcalls are done), -ETIMEDOUT if deferred probe debug
240  * timeout has expired, or -EPROBE_DEFER if none of those conditions are met.
241  *
242  * Drivers or subsystems can opt-in to calling this function instead of directly
243  * returning -EPROBE_DEFER.
244  */
245 int driver_deferred_probe_check_state(struct device *dev)
246 {
247         if (initcalls_done) {
248                 if (!deferred_probe_timeout) {
249                         dev_WARN(dev, "deferred probe timeout, ignoring dependency");
250                         return -ETIMEDOUT;
251                 }
252                 dev_warn(dev, "ignoring dependency for device, assuming no driver");
253                 return -ENODEV;
254         }
255         return -EPROBE_DEFER;
256 }
257
258 static void deferred_probe_timeout_work_func(struct work_struct *work)
259 {
260         struct device_private *private, *p;
261
262         deferred_probe_timeout = 0;
263         driver_deferred_probe_trigger();
264         flush_work(&deferred_probe_work);
265
266         list_for_each_entry_safe(private, p, &deferred_probe_pending_list, deferred_probe)
267                 dev_info(private->device, "deferred probe pending");
268 }
269 static DECLARE_DELAYED_WORK(deferred_probe_timeout_work, deferred_probe_timeout_work_func);
270
271 /**
272  * deferred_probe_initcall() - Enable probing of deferred devices
273  *
274  * We don't want to get in the way when the bulk of drivers are getting probed.
275  * Instead, this initcall makes sure that deferred probing is delayed until
276  * late_initcall time.
277  */
278 static int deferred_probe_initcall(void)
279 {
280         deferred_devices = debugfs_create_file("devices_deferred", 0444, NULL,
281                                                NULL, &deferred_devs_fops);
282
283         driver_deferred_probe_enable = true;
284         driver_deferred_probe_trigger();
285         /* Sort as many dependencies as possible before exiting initcalls */
286         flush_work(&deferred_probe_work);
287         initcalls_done = true;
288
289         /*
290          * Trigger deferred probe again, this time we won't defer anything
291          * that is optional
292          */
293         driver_deferred_probe_trigger();
294         flush_work(&deferred_probe_work);
295
296         if (deferred_probe_timeout > 0) {
297                 schedule_delayed_work(&deferred_probe_timeout_work,
298                         deferred_probe_timeout * HZ);
299         }
300         return 0;
301 }
302 late_initcall(deferred_probe_initcall);
303
304 static void __exit deferred_probe_exit(void)
305 {
306         debugfs_remove_recursive(deferred_devices);
307 }
308 __exitcall(deferred_probe_exit);
309
310 /**
311  * device_is_bound() - Check if device is bound to a driver
312  * @dev: device to check
313  *
314  * Returns true if passed device has already finished probing successfully
315  * against a driver.
316  *
317  * This function must be called with the device lock held.
318  */
319 bool device_is_bound(struct device *dev)
320 {
321         return dev->p && klist_node_attached(&dev->p->knode_driver);
322 }
323
324 static void driver_bound(struct device *dev)
325 {
326         if (device_is_bound(dev)) {
327                 printk(KERN_WARNING "%s: device %s already bound\n",
328                         __func__, kobject_name(&dev->kobj));
329                 return;
330         }
331
332         pr_debug("driver: '%s': %s: bound to device '%s'\n", dev->driver->name,
333                  __func__, dev_name(dev));
334
335         klist_add_tail(&dev->p->knode_driver, &dev->driver->p->klist_devices);
336         device_links_driver_bound(dev);
337
338         device_pm_check_callbacks(dev);
339
340         /*
341          * Make sure the device is no longer in one of the deferred lists and
342          * kick off retrying all pending devices
343          */
344         driver_deferred_probe_del(dev);
345         driver_deferred_probe_trigger();
346
347         if (dev->bus)
348                 blocking_notifier_call_chain(&dev->bus->p->bus_notifier,
349                                              BUS_NOTIFY_BOUND_DRIVER, dev);
350
351         kobject_uevent(&dev->kobj, KOBJ_BIND);
352 }
353
354 static ssize_t coredump_store(struct device *dev, struct device_attribute *attr,
355                             const char *buf, size_t count)
356 {
357         device_lock(dev);
358         dev->driver->coredump(dev);
359         device_unlock(dev);
360
361         return count;
362 }
363 static DEVICE_ATTR_WO(coredump);
364
365 static int driver_sysfs_add(struct device *dev)
366 {
367         int ret;
368
369         if (dev->bus)
370                 blocking_notifier_call_chain(&dev->bus->p->bus_notifier,
371                                              BUS_NOTIFY_BIND_DRIVER, dev);
372
373         ret = sysfs_create_link(&dev->driver->p->kobj, &dev->kobj,
374                                 kobject_name(&dev->kobj));
375         if (ret)
376                 goto fail;
377
378         ret = sysfs_create_link(&dev->kobj, &dev->driver->p->kobj,
379                                 "driver");
380         if (ret)
381                 goto rm_dev;
382
383         if (!IS_ENABLED(CONFIG_DEV_COREDUMP) || !dev->driver->coredump ||
384             !device_create_file(dev, &dev_attr_coredump))
385                 return 0;
386
387         sysfs_remove_link(&dev->kobj, "driver");
388
389 rm_dev:
390         sysfs_remove_link(&dev->driver->p->kobj,
391                           kobject_name(&dev->kobj));
392
393 fail:
394         return ret;
395 }
396
397 static void driver_sysfs_remove(struct device *dev)
398 {
399         struct device_driver *drv = dev->driver;
400
401         if (drv) {
402                 if (drv->coredump)
403                         device_remove_file(dev, &dev_attr_coredump);
404                 sysfs_remove_link(&drv->p->kobj, kobject_name(&dev->kobj));
405                 sysfs_remove_link(&dev->kobj, "driver");
406         }
407 }
408
409 /**
410  * device_bind_driver - bind a driver to one device.
411  * @dev: device.
412  *
413  * Allow manual attachment of a driver to a device.
414  * Caller must have already set @dev->driver.
415  *
416  * Note that this does not modify the bus reference count
417  * nor take the bus's rwsem. Please verify those are accounted
418  * for before calling this. (It is ok to call with no other effort
419  * from a driver's probe() method.)
420  *
421  * This function must be called with the device lock held.
422  */
423 int device_bind_driver(struct device *dev)
424 {
425         int ret;
426
427         ret = driver_sysfs_add(dev);
428         if (!ret)
429                 driver_bound(dev);
430         else if (dev->bus)
431                 blocking_notifier_call_chain(&dev->bus->p->bus_notifier,
432                                              BUS_NOTIFY_DRIVER_NOT_BOUND, dev);
433         return ret;
434 }
435 EXPORT_SYMBOL_GPL(device_bind_driver);
436
437 static atomic_t probe_count = ATOMIC_INIT(0);
438 static DECLARE_WAIT_QUEUE_HEAD(probe_waitqueue);
439
440 static void driver_deferred_probe_add_trigger(struct device *dev,
441                                               int local_trigger_count)
442 {
443         driver_deferred_probe_add(dev);
444         /* Did a trigger occur while probing? Need to re-trigger if yes */
445         if (local_trigger_count != atomic_read(&deferred_trigger_count))
446                 driver_deferred_probe_trigger();
447 }
448
449 static int really_probe(struct device *dev, struct device_driver *drv)
450 {
451         int ret = -EPROBE_DEFER;
452         int local_trigger_count = atomic_read(&deferred_trigger_count);
453         bool test_remove = IS_ENABLED(CONFIG_DEBUG_TEST_DRIVER_REMOVE) &&
454                            !drv->suppress_bind_attrs;
455
456         if (defer_all_probes) {
457                 /*
458                  * Value of defer_all_probes can be set only by
459                  * device_block_probing() which, in turn, will call
460                  * wait_for_device_probe() right after that to avoid any races.
461                  */
462                 dev_dbg(dev, "Driver %s force probe deferral\n", drv->name);
463                 driver_deferred_probe_add(dev);
464                 return ret;
465         }
466
467         ret = device_links_check_suppliers(dev);
468         if (ret == -EPROBE_DEFER)
469                 driver_deferred_probe_add_trigger(dev, local_trigger_count);
470         if (ret)
471                 return ret;
472
473         atomic_inc(&probe_count);
474         pr_debug("bus: '%s': %s: probing driver %s with device %s\n",
475                  drv->bus->name, __func__, drv->name, dev_name(dev));
476         WARN_ON(!list_empty(&dev->devres_head));
477
478 re_probe:
479         dev->driver = drv;
480
481         /* If using pinctrl, bind pins now before probing */
482         ret = pinctrl_bind_pins(dev);
483         if (ret)
484                 goto pinctrl_bind_failed;
485
486         if (dev->bus->dma_configure) {
487                 ret = dev->bus->dma_configure(dev);
488                 if (ret)
489                         goto dma_failed;
490         }
491
492         if (driver_sysfs_add(dev)) {
493                 printk(KERN_ERR "%s: driver_sysfs_add(%s) failed\n",
494                         __func__, dev_name(dev));
495                 goto probe_failed;
496         }
497
498         if (dev->pm_domain && dev->pm_domain->activate) {
499                 ret = dev->pm_domain->activate(dev);
500                 if (ret)
501                         goto probe_failed;
502         }
503
504         if (dev->bus->probe) {
505                 ret = dev->bus->probe(dev);
506                 if (ret)
507                         goto probe_failed;
508         } else if (drv->probe) {
509                 ret = drv->probe(dev);
510                 if (ret)
511                         goto probe_failed;
512         }
513
514         if (test_remove) {
515                 test_remove = false;
516
517                 if (dev->bus->remove)
518                         dev->bus->remove(dev);
519                 else if (drv->remove)
520                         drv->remove(dev);
521
522                 devres_release_all(dev);
523                 driver_sysfs_remove(dev);
524                 dev->driver = NULL;
525                 dev_set_drvdata(dev, NULL);
526                 if (dev->pm_domain && dev->pm_domain->dismiss)
527                         dev->pm_domain->dismiss(dev);
528                 pm_runtime_reinit(dev);
529
530                 goto re_probe;
531         }
532
533         pinctrl_init_done(dev);
534
535         if (dev->pm_domain && dev->pm_domain->sync)
536                 dev->pm_domain->sync(dev);
537
538         driver_bound(dev);
539         ret = 1;
540         pr_debug("bus: '%s': %s: bound device %s to driver %s\n",
541                  drv->bus->name, __func__, dev_name(dev), drv->name);
542         goto done;
543
544 probe_failed:
545         arch_teardown_dma_ops(dev);
546 dma_failed:
547         if (dev->bus)
548                 blocking_notifier_call_chain(&dev->bus->p->bus_notifier,
549                                              BUS_NOTIFY_DRIVER_NOT_BOUND, dev);
550 pinctrl_bind_failed:
551         device_links_no_driver(dev);
552         devres_release_all(dev);
553         driver_sysfs_remove(dev);
554         dev->driver = NULL;
555         dev_set_drvdata(dev, NULL);
556         if (dev->pm_domain && dev->pm_domain->dismiss)
557                 dev->pm_domain->dismiss(dev);
558         pm_runtime_reinit(dev);
559         dev_pm_set_driver_flags(dev, 0);
560
561         switch (ret) {
562         case -EPROBE_DEFER:
563                 /* Driver requested deferred probing */
564                 dev_dbg(dev, "Driver %s requests probe deferral\n", drv->name);
565                 driver_deferred_probe_add_trigger(dev, local_trigger_count);
566                 break;
567         case -ENODEV:
568         case -ENXIO:
569                 pr_debug("%s: probe of %s rejects match %d\n",
570                          drv->name, dev_name(dev), ret);
571                 break;
572         default:
573                 /* driver matched but the probe failed */
574                 printk(KERN_WARNING
575                        "%s: probe of %s failed with error %d\n",
576                        drv->name, dev_name(dev), ret);
577         }
578         /*
579          * Ignore errors returned by ->probe so that the next driver can try
580          * its luck.
581          */
582         ret = 0;
583 done:
584         atomic_dec(&probe_count);
585         wake_up(&probe_waitqueue);
586         return ret;
587 }
588
589 /*
590  * For initcall_debug, show the driver probe time.
591  */
592 static int really_probe_debug(struct device *dev, struct device_driver *drv)
593 {
594         ktime_t calltime, delta, rettime;
595         int ret;
596
597         calltime = ktime_get();
598         ret = really_probe(dev, drv);
599         rettime = ktime_get();
600         delta = ktime_sub(rettime, calltime);
601         printk(KERN_DEBUG "probe of %s returned %d after %lld usecs\n",
602                dev_name(dev), ret, (s64) ktime_to_us(delta));
603         return ret;
604 }
605
606 /**
607  * driver_probe_done
608  * Determine if the probe sequence is finished or not.
609  *
610  * Should somehow figure out how to use a semaphore, not an atomic variable...
611  */
612 int driver_probe_done(void)
613 {
614         pr_debug("%s: probe_count = %d\n", __func__,
615                  atomic_read(&probe_count));
616         if (atomic_read(&probe_count))
617                 return -EBUSY;
618         return 0;
619 }
620
621 /**
622  * wait_for_device_probe
623  * Wait for device probing to be completed.
624  */
625 void wait_for_device_probe(void)
626 {
627         /* wait for the deferred probe workqueue to finish */
628         flush_work(&deferred_probe_work);
629
630         /* wait for the known devices to complete their probing */
631         wait_event(probe_waitqueue, atomic_read(&probe_count) == 0);
632         async_synchronize_full();
633 }
634 EXPORT_SYMBOL_GPL(wait_for_device_probe);
635
636 /**
637  * driver_probe_device - attempt to bind device & driver together
638  * @drv: driver to bind a device to
639  * @dev: device to try to bind to the driver
640  *
641  * This function returns -ENODEV if the device is not registered,
642  * 1 if the device is bound successfully and 0 otherwise.
643  *
644  * This function must be called with @dev lock held.  When called for a
645  * USB interface, @dev->parent lock must be held as well.
646  *
647  * If the device has a parent, runtime-resume the parent before driver probing.
648  */
649 int driver_probe_device(struct device_driver *drv, struct device *dev)
650 {
651         int ret = 0;
652
653         if (!device_is_registered(dev))
654                 return -ENODEV;
655
656         pr_debug("bus: '%s': %s: matched device %s with driver %s\n",
657                  drv->bus->name, __func__, dev_name(dev), drv->name);
658
659         pm_runtime_get_suppliers(dev);
660         if (dev->parent)
661                 pm_runtime_get_sync(dev->parent);
662
663         pm_runtime_barrier(dev);
664         if (initcall_debug)
665                 ret = really_probe_debug(dev, drv);
666         else
667                 ret = really_probe(dev, drv);
668         pm_request_idle(dev);
669
670         if (dev->parent)
671                 pm_runtime_put(dev->parent);
672
673         pm_runtime_put_suppliers(dev);
674         return ret;
675 }
676
677 bool driver_allows_async_probing(struct device_driver *drv)
678 {
679         switch (drv->probe_type) {
680         case PROBE_PREFER_ASYNCHRONOUS:
681                 return true;
682
683         case PROBE_FORCE_SYNCHRONOUS:
684                 return false;
685
686         default:
687                 if (module_requested_async_probing(drv->owner))
688                         return true;
689
690                 return false;
691         }
692 }
693
694 struct device_attach_data {
695         struct device *dev;
696
697         /*
698          * Indicates whether we are are considering asynchronous probing or
699          * not. Only initial binding after device or driver registration
700          * (including deferral processing) may be done asynchronously, the
701          * rest is always synchronous, as we expect it is being done by
702          * request from userspace.
703          */
704         bool check_async;
705
706         /*
707          * Indicates if we are binding synchronous or asynchronous drivers.
708          * When asynchronous probing is enabled we'll execute 2 passes
709          * over drivers: first pass doing synchronous probing and second
710          * doing asynchronous probing (if synchronous did not succeed -
711          * most likely because there was no driver requiring synchronous
712          * probing - and we found asynchronous driver during first pass).
713          * The 2 passes are done because we can't shoot asynchronous
714          * probe for given device and driver from bus_for_each_drv() since
715          * driver pointer is not guaranteed to stay valid once
716          * bus_for_each_drv() iterates to the next driver on the bus.
717          */
718         bool want_async;
719
720         /*
721          * We'll set have_async to 'true' if, while scanning for matching
722          * driver, we'll encounter one that requests asynchronous probing.
723          */
724         bool have_async;
725 };
726
727 static int __device_attach_driver(struct device_driver *drv, void *_data)
728 {
729         struct device_attach_data *data = _data;
730         struct device *dev = data->dev;
731         bool async_allowed;
732         int ret;
733
734         ret = driver_match_device(drv, dev);
735         if (ret == 0) {
736                 /* no match */
737                 return 0;
738         } else if (ret == -EPROBE_DEFER) {
739                 dev_dbg(dev, "Device match requests probe deferral\n");
740                 driver_deferred_probe_add(dev);
741         } else if (ret < 0) {
742                 dev_dbg(dev, "Bus failed to match device: %d", ret);
743                 return ret;
744         } /* ret > 0 means positive match */
745
746         async_allowed = driver_allows_async_probing(drv);
747
748         if (async_allowed)
749                 data->have_async = true;
750
751         if (data->check_async && async_allowed != data->want_async)
752                 return 0;
753
754         return driver_probe_device(drv, dev);
755 }
756
757 static void __device_attach_async_helper(void *_dev, async_cookie_t cookie)
758 {
759         struct device *dev = _dev;
760         struct device_attach_data data = {
761                 .dev            = dev,
762                 .check_async    = true,
763                 .want_async     = true,
764         };
765
766         device_lock(dev);
767
768         /*
769          * Check if device has already been removed or claimed. This may
770          * happen with driver loading, device discovery/registration,
771          * and deferred probe processing happens all at once with
772          * multiple threads.
773          */
774         if (dev->p->dead || dev->driver)
775                 goto out_unlock;
776
777         if (dev->parent)
778                 pm_runtime_get_sync(dev->parent);
779
780         bus_for_each_drv(dev->bus, NULL, &data, __device_attach_driver);
781         dev_dbg(dev, "async probe completed\n");
782
783         pm_request_idle(dev);
784
785         if (dev->parent)
786                 pm_runtime_put(dev->parent);
787 out_unlock:
788         device_unlock(dev);
789
790         put_device(dev);
791 }
792
793 static int __device_attach(struct device *dev, bool allow_async)
794 {
795         int ret = 0;
796
797         device_lock(dev);
798         if (dev->driver) {
799                 if (device_is_bound(dev)) {
800                         ret = 1;
801                         goto out_unlock;
802                 }
803                 ret = device_bind_driver(dev);
804                 if (ret == 0)
805                         ret = 1;
806                 else {
807                         dev->driver = NULL;
808                         ret = 0;
809                 }
810         } else {
811                 struct device_attach_data data = {
812                         .dev = dev,
813                         .check_async = allow_async,
814                         .want_async = false,
815                 };
816
817                 if (dev->parent)
818                         pm_runtime_get_sync(dev->parent);
819
820                 ret = bus_for_each_drv(dev->bus, NULL, &data,
821                                         __device_attach_driver);
822                 if (!ret && allow_async && data.have_async) {
823                         /*
824                          * If we could not find appropriate driver
825                          * synchronously and we are allowed to do
826                          * async probes and there are drivers that
827                          * want to probe asynchronously, we'll
828                          * try them.
829                          */
830                         dev_dbg(dev, "scheduling asynchronous probe\n");
831                         get_device(dev);
832                         async_schedule(__device_attach_async_helper, dev);
833                 } else {
834                         pm_request_idle(dev);
835                 }
836
837                 if (dev->parent)
838                         pm_runtime_put(dev->parent);
839         }
840 out_unlock:
841         device_unlock(dev);
842         return ret;
843 }
844
845 /**
846  * device_attach - try to attach device to a driver.
847  * @dev: device.
848  *
849  * Walk the list of drivers that the bus has and call
850  * driver_probe_device() for each pair. If a compatible
851  * pair is found, break out and return.
852  *
853  * Returns 1 if the device was bound to a driver;
854  * 0 if no matching driver was found;
855  * -ENODEV if the device is not registered.
856  *
857  * When called for a USB interface, @dev->parent lock must be held.
858  */
859 int device_attach(struct device *dev)
860 {
861         return __device_attach(dev, false);
862 }
863 EXPORT_SYMBOL_GPL(device_attach);
864
865 void device_initial_probe(struct device *dev)
866 {
867         __device_attach(dev, true);
868 }
869
870 /*
871  * __device_driver_lock - acquire locks needed to manipulate dev->drv
872  * @dev: Device we will update driver info for
873  * @parent: Parent device. Needed if the bus requires parent lock
874  *
875  * This function will take the required locks for manipulating dev->drv.
876  * Normally this will just be the @dev lock, but when called for a USB
877  * interface, @parent lock will be held as well.
878  */
879 static void __device_driver_lock(struct device *dev, struct device *parent)
880 {
881         if (parent && dev->bus->need_parent_lock)
882                 device_lock(parent);
883         device_lock(dev);
884 }
885
886 /*
887  * __device_driver_unlock - release locks needed to manipulate dev->drv
888  * @dev: Device we will update driver info for
889  * @parent: Parent device. Needed if the bus requires parent lock
890  *
891  * This function will release the required locks for manipulating dev->drv.
892  * Normally this will just be the the @dev lock, but when called for a
893  * USB interface, @parent lock will be released as well.
894  */
895 static void __device_driver_unlock(struct device *dev, struct device *parent)
896 {
897         device_unlock(dev);
898         if (parent && dev->bus->need_parent_lock)
899                 device_unlock(parent);
900 }
901
902 /**
903  * device_driver_attach - attach a specific driver to a specific device
904  * @drv: Driver to attach
905  * @dev: Device to attach it to
906  *
907  * Manually attach driver to a device. Will acquire both @dev lock and
908  * @dev->parent lock if needed.
909  */
910 int device_driver_attach(struct device_driver *drv, struct device *dev)
911 {
912         int ret = 0;
913
914         __device_driver_lock(dev, dev->parent);
915
916         /*
917          * If device has been removed or someone has already successfully
918          * bound a driver before us just skip the driver probe call.
919          */
920         if (!dev->p->dead && !dev->driver)
921                 ret = driver_probe_device(drv, dev);
922
923         __device_driver_unlock(dev, dev->parent);
924
925         return ret;
926 }
927
928 static void __driver_attach_async_helper(void *_dev, async_cookie_t cookie)
929 {
930         struct device *dev = _dev;
931         struct device_driver *drv;
932         int ret = 0;
933
934         __device_driver_lock(dev, dev->parent);
935
936         drv = dev->p->async_driver;
937
938         /*
939          * If device has been removed or someone has already successfully
940          * bound a driver before us just skip the driver probe call.
941          */
942         if (!dev->p->dead && !dev->driver)
943                 ret = driver_probe_device(drv, dev);
944
945         __device_driver_unlock(dev, dev->parent);
946
947         dev_dbg(dev, "driver %s async attach completed: %d\n", drv->name, ret);
948
949         put_device(dev);
950 }
951
952 static int __driver_attach(struct device *dev, void *data)
953 {
954         struct device_driver *drv = data;
955         int ret;
956
957         /*
958          * Lock device and try to bind to it. We drop the error
959          * here and always return 0, because we need to keep trying
960          * to bind to devices and some drivers will return an error
961          * simply if it didn't support the device.
962          *
963          * driver_probe_device() will spit a warning if there
964          * is an error.
965          */
966
967         ret = driver_match_device(drv, dev);
968         if (ret == 0) {
969                 /* no match */
970                 return 0;
971         } else if (ret == -EPROBE_DEFER) {
972                 dev_dbg(dev, "Device match requests probe deferral\n");
973                 driver_deferred_probe_add(dev);
974         } else if (ret < 0) {
975                 dev_dbg(dev, "Bus failed to match device: %d", ret);
976                 return ret;
977         } /* ret > 0 means positive match */
978
979         if (driver_allows_async_probing(drv)) {
980                 /*
981                  * Instead of probing the device synchronously we will
982                  * probe it asynchronously to allow for more parallelism.
983                  *
984                  * We only take the device lock here in order to guarantee
985                  * that the dev->driver and async_driver fields are protected
986                  */
987                 dev_dbg(dev, "probing driver %s asynchronously\n", drv->name);
988                 device_lock(dev);
989                 if (!dev->driver) {
990                         get_device(dev);
991                         dev->p->async_driver = drv;
992                         async_schedule(__driver_attach_async_helper, dev);
993                 }
994                 device_unlock(dev);
995                 return 0;
996         }
997
998         device_driver_attach(drv, dev);
999
1000         return 0;
1001 }
1002
1003 /**
1004  * driver_attach - try to bind driver to devices.
1005  * @drv: driver.
1006  *
1007  * Walk the list of devices that the bus has on it and try to
1008  * match the driver with each one.  If driver_probe_device()
1009  * returns 0 and the @dev->driver is set, we've found a
1010  * compatible pair.
1011  */
1012 int driver_attach(struct device_driver *drv)
1013 {
1014         return bus_for_each_dev(drv->bus, NULL, drv, __driver_attach);
1015 }
1016 EXPORT_SYMBOL_GPL(driver_attach);
1017
1018 /*
1019  * __device_release_driver() must be called with @dev lock held.
1020  * When called for a USB interface, @dev->parent lock must be held as well.
1021  */
1022 static void __device_release_driver(struct device *dev, struct device *parent)
1023 {
1024         struct device_driver *drv;
1025
1026         drv = dev->driver;
1027         if (drv) {
1028                 while (device_links_busy(dev)) {
1029                         __device_driver_unlock(dev, parent);
1030
1031                         device_links_unbind_consumers(dev);
1032
1033                         __device_driver_lock(dev, parent);
1034                         /*
1035                          * A concurrent invocation of the same function might
1036                          * have released the driver successfully while this one
1037                          * was waiting, so check for that.
1038                          */
1039                         if (dev->driver != drv)
1040                                 return;
1041                 }
1042
1043                 pm_runtime_get_sync(dev);
1044                 pm_runtime_clean_up_links(dev);
1045
1046                 driver_sysfs_remove(dev);
1047
1048                 if (dev->bus)
1049                         blocking_notifier_call_chain(&dev->bus->p->bus_notifier,
1050                                                      BUS_NOTIFY_UNBIND_DRIVER,
1051                                                      dev);
1052
1053                 pm_runtime_put_sync(dev);
1054
1055                 if (dev->bus && dev->bus->remove)
1056                         dev->bus->remove(dev);
1057                 else if (drv->remove)
1058                         drv->remove(dev);
1059
1060                 device_links_driver_cleanup(dev);
1061                 arch_teardown_dma_ops(dev);
1062
1063                 devres_release_all(dev);
1064                 dev->driver = NULL;
1065                 dev_set_drvdata(dev, NULL);
1066                 if (dev->pm_domain && dev->pm_domain->dismiss)
1067                         dev->pm_domain->dismiss(dev);
1068                 pm_runtime_reinit(dev);
1069                 dev_pm_set_driver_flags(dev, 0);
1070
1071                 klist_remove(&dev->p->knode_driver);
1072                 device_pm_check_callbacks(dev);
1073                 if (dev->bus)
1074                         blocking_notifier_call_chain(&dev->bus->p->bus_notifier,
1075                                                      BUS_NOTIFY_UNBOUND_DRIVER,
1076                                                      dev);
1077
1078                 kobject_uevent(&dev->kobj, KOBJ_UNBIND);
1079         }
1080 }
1081
1082 void device_release_driver_internal(struct device *dev,
1083                                     struct device_driver *drv,
1084                                     struct device *parent)
1085 {
1086         __device_driver_lock(dev, parent);
1087
1088         if (!drv || drv == dev->driver)
1089                 __device_release_driver(dev, parent);
1090
1091         __device_driver_unlock(dev, parent);
1092 }
1093
1094 /**
1095  * device_release_driver - manually detach device from driver.
1096  * @dev: device.
1097  *
1098  * Manually detach device from driver.
1099  * When called for a USB interface, @dev->parent lock must be held.
1100  *
1101  * If this function is to be called with @dev->parent lock held, ensure that
1102  * the device's consumers are unbound in advance or that their locks can be
1103  * acquired under the @dev->parent lock.
1104  */
1105 void device_release_driver(struct device *dev)
1106 {
1107         /*
1108          * If anyone calls device_release_driver() recursively from
1109          * within their ->remove callback for the same device, they
1110          * will deadlock right here.
1111          */
1112         device_release_driver_internal(dev, NULL, NULL);
1113 }
1114 EXPORT_SYMBOL_GPL(device_release_driver);
1115
1116 /**
1117  * device_driver_detach - detach driver from a specific device
1118  * @dev: device to detach driver from
1119  *
1120  * Detach driver from device. Will acquire both @dev lock and @dev->parent
1121  * lock if needed.
1122  */
1123 void device_driver_detach(struct device *dev)
1124 {
1125         device_release_driver_internal(dev, NULL, dev->parent);
1126 }
1127
1128 /**
1129  * driver_detach - detach driver from all devices it controls.
1130  * @drv: driver.
1131  */
1132 void driver_detach(struct device_driver *drv)
1133 {
1134         struct device_private *dev_prv;
1135         struct device *dev;
1136
1137         if (driver_allows_async_probing(drv))
1138                 async_synchronize_full();
1139
1140         for (;;) {
1141                 spin_lock(&drv->p->klist_devices.k_lock);
1142                 if (list_empty(&drv->p->klist_devices.k_list)) {
1143                         spin_unlock(&drv->p->klist_devices.k_lock);
1144                         break;
1145                 }
1146                 dev_prv = list_entry(drv->p->klist_devices.k_list.prev,
1147                                      struct device_private,
1148                                      knode_driver.n_node);
1149                 dev = dev_prv->device;
1150                 get_device(dev);
1151                 spin_unlock(&drv->p->klist_devices.k_lock);
1152                 device_release_driver_internal(dev, drv, dev->parent);
1153                 put_device(dev);
1154         }
1155 }