drivers/base/power/main.c

   1 /*
   2  * drivers/base/power/main.c - Where the driver meets power management.
   3  *
   4  * Copyright (c) 2003 Patrick Mochel
   5  * Copyright (c) 2003 Open Source Development Lab
   6  *
   7  * This file is released under the GPLv2
   8  *
   9  *
  10  * The driver model core calls device_pm_add() when a device is registered.
  11  * This will initialize the embedded device_pm_info object in the device
  12  * and add it to the list of power-controlled devices. sysfs entries for
  13  * controlling device power management will also be added.
  14  *
  15  * A separate list is used for keeping track of power info, because the power
  16  * domain dependencies may differ from the ancestral dependencies that the
  17  * subsystem list maintains.
  18  */
  19
  20 #include <linux/device.h>
  21 #include <linux/kallsyms.h>
  22 #include <linux/export.h>
  23 #include <linux/mutex.h>
  24 #include <linux/pm.h>
  25 #include <linux/pm_runtime.h>
  26 #include <linux/pm-trace.h>
  27 #include <linux/pm_wakeirq.h>
  28 #include <linux/interrupt.h>
  29 #include <linux/sched.h>
  30 #include <linux/async.h>
  31 #include <linux/suspend.h>
  32 #include <trace/events/power.h>
  33 #include <linux/cpufreq.h>
  34 #include <linux/cpuidle.h>
  35 #include <linux/timer.h>
  36
  37 #include "../base.h"
  38 #include "power.h"
  39
  40 typedef int (*pm_callback_t)(struct device *);
  41
  42 /*
  43  * The entries in the dpm_list list are in a depth first order, simply
  44  * because children are guaranteed to be discovered after parents, and
  45  * are inserted at the back of the list on discovery.
  46  *
  47  * Since device_pm_add() may be called with a device lock held,
  48  * we must never try to acquire a device lock while holding
  49  * dpm_list_mutex.
  50  */
  51
  52 LIST_HEAD(dpm_list);
  53 static LIST_HEAD(dpm_prepared_list);
  54 static LIST_HEAD(dpm_suspended_list);
  55 static LIST_HEAD(dpm_late_early_list);
  56 static LIST_HEAD(dpm_noirq_list);
  57
  58 struct suspend_stats suspend_stats;
  59 static DEFINE_MUTEX(dpm_list_mtx);
  60 static pm_message_t pm_transition;
  61
  62 static int async_error;
  63
  64 static char *pm_verb(int event)
  65 {
  66         switch (event) {
  67         case PM_EVENT_SUSPEND:
  68                 return "suspend";
  69         case PM_EVENT_RESUME:
  70                 return "resume";
  71         case PM_EVENT_FREEZE:
  72                 return "freeze";
  73         case PM_EVENT_QUIESCE:
  74                 return "quiesce";
  75         case PM_EVENT_HIBERNATE:
  76                 return "hibernate";
  77         case PM_EVENT_THAW:
  78                 return "thaw";
  79         case PM_EVENT_RESTORE:
  80                 return "restore";
  81         case PM_EVENT_RECOVER:
  82                 return "recover";
  83         default:
  84                 return "(unknown PM event)";
  85         }
  86 }
  87
  88 /**
  89  * device_pm_sleep_init - Initialize system suspend-related device fields.
  90  * @dev: Device object being initialized.
  91  */
  92 void device_pm_sleep_init(struct device *dev)
  93 {
  94         dev->power.is_prepared = false;
  95         dev->power.is_suspended = false;
  96         dev->power.is_noirq_suspended = false;
  97         dev->power.is_late_suspended = false;
  98         init_completion(&dev->power.completion);
  99         complete_all(&dev->power.completion);
 100         dev->power.wakeup = NULL;
 101         INIT_LIST_HEAD(&dev->power.entry);
 102 }
 103
 104 /**
 105  * device_pm_lock - Lock the list of active devices used by the PM core.
 106  */
 107 void device_pm_lock(void)
 108 {
 109         mutex_lock(&dpm_list_mtx);
 110 }
 111
 112 /**
 113  * device_pm_unlock - Unlock the list of active devices used by the PM core.
 114  */
 115 void device_pm_unlock(void)
 116 {
 117         mutex_unlock(&dpm_list_mtx);
 118 }
 119
 120 /**
 121  * device_pm_add - Add a device to the PM core's list of active devices.
 122  * @dev: Device to add to the list.
 123  */
 124 void device_pm_add(struct device *dev)
 125 {
 126         pr_debug("PM: Adding info for %s:%s\n",
 127                  dev->bus ? dev->bus->name : "No Bus", dev_name(dev));
 128         device_pm_check_callbacks(dev);
 129         mutex_lock(&dpm_list_mtx);
 130         if (dev->parent && dev->parent->power.is_prepared)
 131                 dev_warn(dev, "parent %s should not be sleeping\n",
 132                         dev_name(dev->parent));
 133         list_add_tail(&dev->power.entry, &dpm_list);
 134         mutex_unlock(&dpm_list_mtx);
 135 }
 136
 137 /**
 138  * device_pm_remove - Remove a device from the PM core's list of active devices.
 139  * @dev: Device to be removed from the list.
 140  */
 141 void device_pm_remove(struct device *dev)
 142 {
 143         pr_debug("PM: Removing info for %s:%s\n",
 144                  dev->bus ? dev->bus->name : "No Bus", dev_name(dev));
 145         complete_all(&dev->power.completion);
 146         mutex_lock(&dpm_list_mtx);
 147         list_del_init(&dev->power.entry);
 148         mutex_unlock(&dpm_list_mtx);
 149         device_wakeup_disable(dev);
 150         pm_runtime_remove(dev);
 151         device_pm_check_callbacks(dev);
 152 }
 153
 154 /**
 155  * device_pm_move_before - Move device in the PM core's list of active devices.
 156  * @deva: Device to move in dpm_list.
 157  * @devb: Device @deva should come before.
 158  */
 159 void device_pm_move_before(struct device *deva, struct device *devb)
 160 {
 161         pr_debug("PM: Moving %s:%s before %s:%s\n",
 162                  deva->bus ? deva->bus->name : "No Bus", dev_name(deva),
 163                  devb->bus ? devb->bus->name : "No Bus", dev_name(devb));
 164         /* Delete deva from dpm_list and reinsert before devb. */
 165         list_move_tail(&deva->power.entry, &devb->power.entry);
 166 }
 167
 168 /**
 169  * device_pm_move_after - Move device in the PM core's list of active devices.
 170  * @deva: Device to move in dpm_list.
 171  * @devb: Device @deva should come after.
 172  */
 173 void device_pm_move_after(struct device *deva, struct device *devb)
 174 {
 175         pr_debug("PM: Moving %s:%s after %s:%s\n",
 176                  deva->bus ? deva->bus->name : "No Bus", dev_name(deva),
 177                  devb->bus ? devb->bus->name : "No Bus", dev_name(devb));
 178         /* Delete deva from dpm_list and reinsert after devb. */
 179         list_move(&deva->power.entry, &devb->power.entry);
 180 }
 181
 182 /**
 183  * device_pm_move_last - Move device to end of the PM core's list of devices.
 184  * @dev: Device to move in dpm_list.
 185  */
 186 void device_pm_move_last(struct device *dev)
 187 {
 188         pr_debug("PM: Moving %s:%s to end of list\n",
 189                  dev->bus ? dev->bus->name : "No Bus", dev_name(dev));
 190         list_move_tail(&dev->power.entry, &dpm_list);
 191 }
 192
 193 static ktime_t initcall_debug_start(struct device *dev)
 194 {
 195         ktime_t calltime = ktime_set(0, 0);
 196
 197         if (pm_print_times_enabled) {
 198                 pr_info("calling  %s+ @ %i, parent: %s\n",
 199                         dev_name(dev), task_pid_nr(current),
 200                         dev->parent ? dev_name(dev->parent) : "none");
 201                 calltime = ktime_get();
 202         }
 203
 204         return calltime;
 205 }
 206
 207 static void initcall_debug_report(struct device *dev, ktime_t calltime,
 208                                   int error, pm_message_t state, char *info)
 209 {
 210         ktime_t rettime;
 211         s64 nsecs;
 212
 213         rettime = ktime_get();
 214         nsecs = (s64) ktime_to_ns(ktime_sub(rettime, calltime));
 215
 216         if (pm_print_times_enabled) {
 217                 pr_info("call %s+ returned %d after %Ld usecs\n", dev_name(dev),
 218                         error, (unsigned long long)nsecs >> 10);
 219         }
 220 }
 221
 222 /**
 223  * dpm_wait - Wait for a PM operation to complete.
 224  * @dev: Device to wait for.
 225  * @async: If unset, wait only if the device's power.async_suspend flag is set.
 226  */
 227 static void dpm_wait(struct device *dev, bool async)
 228 {
 229         if (!dev)
 230                 return;
 231
 232         if (async || (pm_async_enabled && dev->power.async_suspend))
 233                 wait_for_completion(&dev->power.completion);
 234 }
 235
 236 static int dpm_wait_fn(struct device *dev, void *async_ptr)
 237 {
 238         dpm_wait(dev, *((bool *)async_ptr));
 239         return 0;
 240 }
 241
 242 static void dpm_wait_for_children(struct device *dev, bool async)
 243 {
 244        device_for_each_child(dev, &async, dpm_wait_fn);
 245 }
 246
 247 /**
 248  * pm_op - Return the PM operation appropriate for given PM event.
 249  * @ops: PM operations to choose from.
 250  * @state: PM transition of the system being carried out.
 251  */
 252 static pm_callback_t pm_op(const struct dev_pm_ops *ops, pm_message_t state)
 253 {
 254         switch (state.event) {
 255 #ifdef CONFIG_SUSPEND
 256         case PM_EVENT_SUSPEND:
 257                 return ops->suspend;
 258         case PM_EVENT_RESUME:
 259                 return ops->resume;
 260 #endif /* CONFIG_SUSPEND */
 261 #ifdef CONFIG_HIBERNATE_CALLBACKS
 262         case PM_EVENT_FREEZE:
 263         case PM_EVENT_QUIESCE:
 264                 return ops->freeze;
 265         case PM_EVENT_HIBERNATE:
 266                 return ops->poweroff;
 267         case PM_EVENT_THAW:
 268         case PM_EVENT_RECOVER:
 269                 return ops->thaw;
 270                 break;
 271         case PM_EVENT_RESTORE:
 272                 return ops->restore;
 273 #endif /* CONFIG_HIBERNATE_CALLBACKS */
 274         }
 275
 276         return NULL;
 277 }
 278
 279 /**
 280  * pm_late_early_op - Return the PM operation appropriate for given PM event.
 281  * @ops: PM operations to choose from.
 282  * @state: PM transition of the system being carried out.
 283  *
 284  * Runtime PM is disabled for @dev while this function is being executed.
 285  */
 286 static pm_callback_t pm_late_early_op(const struct dev_pm_ops *ops,
 287                                       pm_message_t state)
 288 {
 289         switch (state.event) {
 290 #ifdef CONFIG_SUSPEND
 291         case PM_EVENT_SUSPEND:
 292                 return ops->suspend_late;
 293         case PM_EVENT_RESUME:
 294                 return ops->resume_early;
 295 #endif /* CONFIG_SUSPEND */
 296 #ifdef CONFIG_HIBERNATE_CALLBACKS
 297         case PM_EVENT_FREEZE:
 298         case PM_EVENT_QUIESCE:
 299                 return ops->freeze_late;
 300         case PM_EVENT_HIBERNATE:
 301                 return ops->poweroff_late;
 302         case PM_EVENT_THAW:
 303         case PM_EVENT_RECOVER:
 304                 return ops->thaw_early;
 305         case PM_EVENT_RESTORE:
 306                 return ops->restore_early;
 307 #endif /* CONFIG_HIBERNATE_CALLBACKS */
 308         }
 309
 310         return NULL;
 311 }
 312
 313 /**
 314  * pm_noirq_op - Return the PM operation appropriate for given PM event.
 315  * @ops: PM operations to choose from.
 316  * @state: PM transition of the system being carried out.
 317  *
 318  * The driver of @dev will not receive interrupts while this function is being
 319  * executed.
 320  */
 321 static pm_callback_t pm_noirq_op(const struct dev_pm_ops *ops, pm_message_t state)
 322 {
 323         switch (state.event) {
 324 #ifdef CONFIG_SUSPEND
 325         case PM_EVENT_SUSPEND:
 326                 return ops->suspend_noirq;
 327         case PM_EVENT_RESUME:
 328                 return ops->resume_noirq;
 329 #endif /* CONFIG_SUSPEND */
 330 #ifdef CONFIG_HIBERNATE_CALLBACKS
 331         case PM_EVENT_FREEZE:
 332         case PM_EVENT_QUIESCE:
 333                 return ops->freeze_noirq;
 334         case PM_EVENT_HIBERNATE:
 335                 return ops->poweroff_noirq;
 336         case PM_EVENT_THAW:
 337         case PM_EVENT_RECOVER:
 338                 return ops->thaw_noirq;
 339         case PM_EVENT_RESTORE:
 340                 return ops->restore_noirq;
 341 #endif /* CONFIG_HIBERNATE_CALLBACKS */
 342         }
 343
 344         return NULL;
 345 }
 346
 347 static void pm_dev_dbg(struct device *dev, pm_message_t state, char *info)
 348 {
 349         dev_dbg(dev, "%s%s%s\n", info, pm_verb(state.event),
 350                 ((state.event & PM_EVENT_SLEEP) && device_may_wakeup(dev)) ?
 351                 ", may wakeup" : "");
 352 }
 353
 354 static void pm_dev_err(struct device *dev, pm_message_t state, char *info,
 355                         int error)
 356 {
 357         printk(KERN_ERR "PM: Device %s failed to %s%s: error %d\n",
 358                 dev_name(dev), pm_verb(state.event), info, error);
 359 }
 360
 361 static void dpm_show_time(ktime_t starttime, pm_message_t state, char *info)
 362 {
 363         ktime_t calltime;
 364         u64 usecs64;
 365         int usecs;
 366
 367         calltime = ktime_get();
 368         usecs64 = ktime_to_ns(ktime_sub(calltime, starttime));
 369         do_div(usecs64, NSEC_PER_USEC);
 370         usecs = usecs64;
 371         if (usecs == 0)
 372                 usecs = 1;
 373         pr_info("PM: %s%s%s of devices complete after %ld.%03ld msecs\n",
 374                 info ?: "", info ? " " : "", pm_verb(state.event),
 375                 usecs / USEC_PER_MSEC, usecs % USEC_PER_MSEC);
 376 }
 377
 378 static int dpm_run_callback(pm_callback_t cb, struct device *dev,
 379                             pm_message_t state, char *info)
 380 {
 381         ktime_t calltime;
 382         int error;
 383
 384         if (!cb)
 385                 return 0;
 386
 387         calltime = initcall_debug_start(dev);
 388
 389         pm_dev_dbg(dev, state, info);
 390         trace_device_pm_callback_start(dev, info, state.event);
 391         error = cb(dev);
 392         trace_device_pm_callback_end(dev, error);
 393         suspend_report_result(cb, error);
 394
 395         initcall_debug_report(dev, calltime, error, state, info);
 396
 397         return error;
 398 }
 399
 400 #ifdef CONFIG_DPM_WATCHDOG
 401 struct dpm_watchdog {
 402         struct device           *dev;
 403         struct task_struct      *tsk;
 404         struct timer_list       timer;
 405 };
 406
 407 #define DECLARE_DPM_WATCHDOG_ON_STACK(wd) \
 408         struct dpm_watchdog wd
 409
 410 /**
 411  * dpm_watchdog_handler - Driver suspend / resume watchdog handler.
 412  * @data: Watchdog object address.
 413  *
 414  * Called when a driver has timed out suspending or resuming.
 415  * There's not much we can do here to recover so panic() to
 416  * capture a crash-dump in pstore.
 417  */
 418 static void dpm_watchdog_handler(unsigned long data)
 419 {
 420         struct dpm_watchdog *wd = (void *)data;
 421
 422         dev_emerg(wd->dev, "**** DPM device timeout ****\n");
 423         show_stack(wd->tsk, NULL);
 424         panic("%s %s: unrecoverable failure\n",
 425                 dev_driver_string(wd->dev), dev_name(wd->dev));
 426 }
 427
 428 /**
 429  * dpm_watchdog_set - Enable pm watchdog for given device.
 430  * @wd: Watchdog. Must be allocated on the stack.
 431  * @dev: Device to handle.
 432  */
 433 static void dpm_watchdog_set(struct dpm_watchdog *wd, struct device *dev)
 434 {
 435         struct timer_list *timer = &wd->timer;
 436
 437         wd->dev = dev;
 438         wd->tsk = current;
 439
 440         init_timer_on_stack(timer);
 441         /* use same timeout value for both suspend and resume */
 442         timer->expires = jiffies + HZ * CONFIG_DPM_WATCHDOG_TIMEOUT;
 443         timer->function = dpm_watchdog_handler;
 444         timer->data = (unsigned long)wd;
 445         add_timer(timer);
 446 }
 447
 448 /**
 449  * dpm_watchdog_clear - Disable suspend/resume watchdog.
 450  * @wd: Watchdog to disable.
 451  */
 452 static void dpm_watchdog_clear(struct dpm_watchdog *wd)
 453 {
 454         struct timer_list *timer = &wd->timer;
 455
 456         del_timer_sync(timer);
 457         destroy_timer_on_stack(timer);
 458 }
 459 #else
 460 #define DECLARE_DPM_WATCHDOG_ON_STACK(wd)
 461 #define dpm_watchdog_set(x, y)
 462 #define dpm_watchdog_clear(x)
 463 #endif
 464
 465 /*------------------------- Resume routines -------------------------*/
 466
 467 /**
 468  * device_resume_noirq - Execute an "early resume" callback for given device.
 469  * @dev: Device to handle.
 470  * @state: PM transition of the system being carried out.
 471  * @async: If true, the device is being resumed asynchronously.
 472  *
 473  * The driver of @dev will not receive interrupts while this function is being
 474  * executed.
 475  */
 476 static int device_resume_noirq(struct device *dev, pm_message_t state, bool async)
 477 {
 478         pm_callback_t callback = NULL;
 479         char *info = NULL;
 480         int error = 0;
 481
 482         TRACE_DEVICE(dev);
 483         TRACE_RESUME(0);
 484
 485         if (dev->power.syscore || dev->power.direct_complete)
 486                 goto Out;
 487
 488         if (!dev->power.is_noirq_suspended)
 489                 goto Out;
 490
 491         dpm_wait(dev->parent, async);
 492
 493         if (dev->pm_domain) {
 494                 info = "noirq power domain ";
 495                 callback = pm_noirq_op(&dev->pm_domain->ops, state);
 496         } else if (dev->type && dev->type->pm) {
 497                 info = "noirq type ";
 498                 callback = pm_noirq_op(dev->type->pm, state);
 499         } else if (dev->class && dev->class->pm) {
 500                 info = "noirq class ";
 501                 callback = pm_noirq_op(dev->class->pm, state);
 502         } else if (dev->bus && dev->bus->pm) {
 503                 info = "noirq bus ";
 504                 callback = pm_noirq_op(dev->bus->pm, state);
 505         }
 506
 507         if (!callback && dev->driver && dev->driver->pm) {
 508                 info = "noirq driver ";
 509                 callback = pm_noirq_op(dev->driver->pm, state);
 510         }
 511
 512         error = dpm_run_callback(callback, dev, state, info);
 513         dev->power.is_noirq_suspended = false;
 514
 515  Out:
 516         complete_all(&dev->power.completion);
 517         TRACE_RESUME(error);
 518         return error;
 519 }
 520
 521 static bool is_async(struct device *dev)
 522 {
 523         return dev->power.async_suspend && pm_async_enabled
 524                 && !pm_trace_is_enabled();
 525 }
 526
 527 static void async_resume_noirq(void *data, async_cookie_t cookie)
 528 {
 529         struct device *dev = (struct device *)data;
 530         int error;
 531
 532         error = device_resume_noirq(dev, pm_transition, true);
 533         if (error)
 534                 pm_dev_err(dev, pm_transition, " async", error);
 535
 536         put_device(dev);
 537 }
 538
 539 /**
 540  * dpm_resume_noirq - Execute "noirq resume" callbacks for all devices.
 541  * @state: PM transition of the system being carried out.
 542  *
 543  * Call the "noirq" resume handlers for all devices in dpm_noirq_list and
 544  * enable device drivers to receive interrupts.
 545  */
 546 void dpm_resume_noirq(pm_message_t state)
 547 {
 548         struct device *dev;
 549         ktime_t starttime = ktime_get();
 550
 551         trace_suspend_resume(TPS("dpm_resume_noirq"), state.event, true);
 552         mutex_lock(&dpm_list_mtx);
 553         pm_transition = state;
 554
 555         /*
 556          * Advanced the async threads upfront,
 557          * in case the starting of async threads is
 558          * delayed by non-async resuming devices.
 559          */
 560         list_for_each_entry(dev, &dpm_noirq_list, power.entry) {
 561                 reinit_completion(&dev->power.completion);
 562                 if (is_async(dev)) {
 563                         get_device(dev);
 564                         async_schedule(async_resume_noirq, dev);
 565                 }
 566         }
 567
 568         while (!list_empty(&dpm_noirq_list)) {
 569                 dev = to_device(dpm_noirq_list.next);
 570                 get_device(dev);
 571                 list_move_tail(&dev->power.entry, &dpm_late_early_list);
 572                 mutex_unlock(&dpm_list_mtx);
 573
 574                 if (!is_async(dev)) {
 575                         int error;
 576
 577                         error = device_resume_noirq(dev, state, false);
 578                         if (error) {
 579                                 suspend_stats.failed_resume_noirq++;
 580                                 dpm_save_failed_step(SUSPEND_RESUME_NOIRQ);
 581                                 dpm_save_failed_dev(dev_name(dev));
 582                                 pm_dev_err(dev, state, " noirq", error);
 583                         }
 584                 }
 585
 586                 mutex_lock(&dpm_list_mtx);
 587                 put_device(dev);
 588         }
 589         mutex_unlock(&dpm_list_mtx);
 590         async_synchronize_full();
 591         dpm_show_time(starttime, state, "noirq");
 592         resume_device_irqs();
 593         device_wakeup_disarm_wake_irqs();
 594         cpuidle_resume();
 595         trace_suspend_resume(TPS("dpm_resume_noirq"), state.event, false);
 596 }
 597
 598 /**
 599  * device_resume_early - Execute an "early resume" callback for given device.
 600  * @dev: Device to handle.
 601  * @state: PM transition of the system being carried out.
 602  * @async: If true, the device is being resumed asynchronously.
 603  *
 604  * Runtime PM is disabled for @dev while this function is being executed.
 605  */
 606 static int device_resume_early(struct device *dev, pm_message_t state, bool async)
 607 {
 608         pm_callback_t callback = NULL;
 609         char *info = NULL;
 610         int error = 0;
 611
 612         TRACE_DEVICE(dev);
 613         TRACE_RESUME(0);
 614
 615         if (dev->power.syscore || dev->power.direct_complete)
 616                 goto Out;
 617
 618         if (!dev->power.is_late_suspended)
 619                 goto Out;
 620
 621         dpm_wait(dev->parent, async);
 622
 623         if (dev->pm_domain) {
 624                 info = "early power domain ";
 625                 callback = pm_late_early_op(&dev->pm_domain->ops, state);
 626         } else if (dev->type && dev->type->pm) {
 627                 info = "early type ";
 628                 callback = pm_late_early_op(dev->type->pm, state);
 629         } else if (dev->class && dev->class->pm) {
 630                 info = "early class ";
 631                 callback = pm_late_early_op(dev->class->pm, state);
 632         } else if (dev->bus && dev->bus->pm) {
 633                 info = "early bus ";
 634                 callback = pm_late_early_op(dev->bus->pm, state);
 635         }
 636
 637         if (!callback && dev->driver && dev->driver->pm) {
 638                 info = "early driver ";
 639                 callback = pm_late_early_op(dev->driver->pm, state);
 640         }
 641
 642         error = dpm_run_callback(callback, dev, state, info);
 643         dev->power.is_late_suspended = false;
 644
 645  Out:
 646         TRACE_RESUME(error);
 647
 648         pm_runtime_enable(dev);
 649         complete_all(&dev->power.completion);
 650         return error;
 651 }
 652
 653 static void async_resume_early(void *data, async_cookie_t cookie)
 654 {
 655         struct device *dev = (struct device *)data;
 656         int error;
 657
 658         error = device_resume_early(dev, pm_transition, true);
 659         if (error)
 660                 pm_dev_err(dev, pm_transition, " async", error);
 661
 662         put_device(dev);
 663 }
 664
 665 /**
 666  * dpm_resume_early - Execute "early resume" callbacks for all devices.
 667  * @state: PM transition of the system being carried out.
 668  */
 669 void dpm_resume_early(pm_message_t state)
 670 {
 671         struct device *dev;
 672         ktime_t starttime = ktime_get();
 673
 674         trace_suspend_resume(TPS("dpm_resume_early"), state.event, true);
 675         mutex_lock(&dpm_list_mtx);
 676         pm_transition = state;
 677
 678         /*
 679          * Advanced the async threads upfront,
 680          * in case the starting of async threads is
 681          * delayed by non-async resuming devices.
 682          */
 683         list_for_each_entry(dev, &dpm_late_early_list, power.entry) {
 684                 reinit_completion(&dev->power.completion);
 685                 if (is_async(dev)) {
 686                         get_device(dev);
 687                         async_schedule(async_resume_early, dev);
 688                 }
 689         }
 690
 691         while (!list_empty(&dpm_late_early_list)) {
 692                 dev = to_device(dpm_late_early_list.next);
 693                 get_device(dev);
 694                 list_move_tail(&dev->power.entry, &dpm_suspended_list);
 695                 mutex_unlock(&dpm_list_mtx);
 696
 697                 if (!is_async(dev)) {
 698                         int error;
 699
 700                         error = device_resume_early(dev, state, false);
 701                         if (error) {
 702                                 suspend_stats.failed_resume_early++;
 703                                 dpm_save_failed_step(SUSPEND_RESUME_EARLY);
 704                                 dpm_save_failed_dev(dev_name(dev));
 705                                 pm_dev_err(dev, state, " early", error);
 706                         }
 707                 }
 708                 mutex_lock(&dpm_list_mtx);
 709                 put_device(dev);
 710         }
 711         mutex_unlock(&dpm_list_mtx);
 712         async_synchronize_full();
 713         dpm_show_time(starttime, state, "early");
 714         trace_suspend_resume(TPS("dpm_resume_early"), state.event, false);
 715 }
 716
 717 /**
 718  * dpm_resume_start - Execute "noirq" and "early" device callbacks.
 719  * @state: PM transition of the system being carried out.
 720  */
 721 void dpm_resume_start(pm_message_t state)
 722 {
 723         dpm_resume_noirq(state);
 724         dpm_resume_early(state);
 725 }
 726 EXPORT_SYMBOL_GPL(dpm_resume_start);
 727
 728 /**
 729  * device_resume - Execute "resume" callbacks for given device.
 730  * @dev: Device to handle.
 731  * @state: PM transition of the system being carried out.
 732  * @async: If true, the device is being resumed asynchronously.
 733  */
 734 static int device_resume(struct device *dev, pm_message_t state, bool async)
 735 {
 736         pm_callback_t callback = NULL;
 737         char *info = NULL;
 738         int error = 0;
 739         DECLARE_DPM_WATCHDOG_ON_STACK(wd);
 740
 741         TRACE_DEVICE(dev);
 742         TRACE_RESUME(0);
 743
 744         if (dev->power.syscore)
 745                 goto Complete;
 746
 747         if (dev->power.direct_complete) {
 748                 /* Match the pm_runtime_disable() in __device_suspend(). */
 749                 pm_runtime_enable(dev);
 750                 goto Complete;
 751         }
 752
 753         dpm_wait(dev->parent, async);
 754         dpm_watchdog_set(&wd, dev);
 755         device_lock(dev);
 756
 757         /*
 758          * This is a fib.  But we'll allow new children to be added below
 759          * a resumed device, even if the device hasn't been completed yet.
 760          */
 761         dev->power.is_prepared = false;
 762
 763         if (!dev->power.is_suspended)
 764                 goto Unlock;
 765
 766         if (dev->pm_domain) {
 767                 info = "power domain ";
 768                 callback = pm_op(&dev->pm_domain->ops, state);
 769                 goto Driver;
 770         }
 771
 772         if (dev->type && dev->type->pm) {
 773                 info = "type ";
 774                 callback = pm_op(dev->type->pm, state);
 775                 goto Driver;
 776         }
 777
 778         if (dev->class) {
 779                 if (dev->class->pm) {
 780                         info = "class ";
 781                         callback = pm_op(dev->class->pm, state);
 782                         goto Driver;
 783                 } else if (dev->class->resume) {
 784                         info = "legacy class ";
 785                         callback = dev->class->resume;
 786                         goto End;
 787                 }
 788         }
 789
 790         if (dev->bus) {
 791                 if (dev->bus->pm) {
 792                         info = "bus ";
 793                         callback = pm_op(dev->bus->pm, state);
 794                 } else if (dev->bus->resume) {
 795                         info = "legacy bus ";
 796                         callback = dev->bus->resume;
 797                         goto End;
 798                 }
 799         }
 800
 801  Driver:
 802         if (!callback && dev->driver && dev->driver->pm) {
 803                 info = "driver ";
 804                 callback = pm_op(dev->driver->pm, state);
 805         }
 806
 807  End:
 808         error = dpm_run_callback(callback, dev, state, info);
 809         dev->power.is_suspended = false;
 810
 811  Unlock:
 812         device_unlock(dev);
 813         dpm_watchdog_clear(&wd);
 814
 815  Complete:
 816         complete_all(&dev->power.completion);
 817
 818         TRACE_RESUME(error);
 819
 820         return error;
 821 }
 822
 823 static void async_resume(void *data, async_cookie_t cookie)
 824 {
 825         struct device *dev = (struct device *)data;
 826         int error;
 827
 828         error = device_resume(dev, pm_transition, true);
 829         if (error)
 830                 pm_dev_err(dev, pm_transition, " async", error);
 831         put_device(dev);
 832 }
 833
 834 /**
 835  * dpm_resume - Execute "resume" callbacks for non-sysdev devices.
 836  * @state: PM transition of the system being carried out.
 837  *
 838  * Execute the appropriate "resume" callback for all devices whose status
 839  * indicates that they are suspended.
 840  */
 841 void dpm_resume(pm_message_t state)
 842 {
 843         struct device *dev;
 844         ktime_t starttime = ktime_get();
 845
 846         trace_suspend_resume(TPS("dpm_resume"), state.event, true);
 847         might_sleep();
 848
 849         mutex_lock(&dpm_list_mtx);
 850         pm_transition = state;
 851         async_error = 0;
 852
 853         list_for_each_entry(dev, &dpm_suspended_list, power.entry) {
 854                 reinit_completion(&dev->power.completion);
 855                 if (is_async(dev)) {
 856                         get_device(dev);
 857                         async_schedule(async_resume, dev);
 858                 }
 859         }
 860
 861         while (!list_empty(&dpm_suspended_list)) {
 862                 dev = to_device(dpm_suspended_list.next);
 863                 get_device(dev);
 864                 if (!is_async(dev)) {
 865                         int error;
 866
 867                         mutex_unlock(&dpm_list_mtx);
 868
 869                         error = device_resume(dev, state, false);
 870                         if (error) {
 871                                 suspend_stats.failed_resume++;
 872                                 dpm_save_failed_step(SUSPEND_RESUME);
 873                                 dpm_save_failed_dev(dev_name(dev));
 874                                 pm_dev_err(dev, state, "", error);
 875                         }
 876
 877                         mutex_lock(&dpm_list_mtx);
 878                 }
 879                 if (!list_empty(&dev->power.entry))
 880                         list_move_tail(&dev->power.entry, &dpm_prepared_list);
 881                 put_device(dev);
 882         }
 883         mutex_unlock(&dpm_list_mtx);
 884         async_synchronize_full();
 885         dpm_show_time(starttime, state, NULL);
 886
 887         cpufreq_resume();
 888         trace_suspend_resume(TPS("dpm_resume"), state.event, false);
 889 }
 890
 891 /**
 892  * device_complete - Complete a PM transition for given device.
 893  * @dev: Device to handle.
 894  * @state: PM transition of the system being carried out.
 895  */
 896 static void device_complete(struct device *dev, pm_message_t state)
 897 {
 898         void (*callback)(struct device *) = NULL;
 899         char *info = NULL;
 900
 901         if (dev->power.syscore)
 902                 return;
 903
 904         device_lock(dev);
 905
 906         if (dev->pm_domain) {
 907                 info = "completing power domain ";
 908                 callback = dev->pm_domain->ops.complete;
 909         } else if (dev->type && dev->type->pm) {
 910                 info = "completing type ";
 911                 callback = dev->type->pm->complete;
 912         } else if (dev->class && dev->class->pm) {
 913                 info = "completing class ";
 914                 callback = dev->class->pm->complete;
 915         } else if (dev->bus && dev->bus->pm) {
 916                 info = "completing bus ";
 917                 callback = dev->bus->pm->complete;
 918         }
 919
 920         if (!callback && dev->driver && dev->driver->pm) {
 921                 info = "completing driver ";
 922                 callback = dev->driver->pm->complete;
 923         }
 924
 925         if (callback) {
 926                 pm_dev_dbg(dev, state, info);
 927                 callback(dev);
 928         }
 929
 930         device_unlock(dev);
 931
 932         pm_runtime_put(dev);
 933 }
 934
 935 /**
 936  * dpm_complete - Complete a PM transition for all non-sysdev devices.
 937  * @state: PM transition of the system being carried out.
 938  *
 939  * Execute the ->complete() callbacks for all devices whose PM status is not
 940  * DPM_ON (this allows new devices to be registered).
 941  */
 942 void dpm_complete(pm_message_t state)
 943 {
 944         struct list_head list;
 945
 946         trace_suspend_resume(TPS("dpm_complete"), state.event, true);
 947         might_sleep();
 948
 949         INIT_LIST_HEAD(&list);
 950         mutex_lock(&dpm_list_mtx);
 951         while (!list_empty(&dpm_prepared_list)) {
 952                 struct device *dev = to_device(dpm_prepared_list.prev);
 953
 954                 get_device(dev);
 955                 dev->power.is_prepared = false;
 956                 list_move(&dev->power.entry, &list);
 957                 mutex_unlock(&dpm_list_mtx);
 958
 959                 trace_device_pm_callback_start(dev, "", state.event);
 960                 device_complete(dev, state);
 961                 trace_device_pm_callback_end(dev, 0);
 962
 963                 mutex_lock(&dpm_list_mtx);
 964                 put_device(dev);
 965         }
 966         list_splice(&list, &dpm_list);
 967         mutex_unlock(&dpm_list_mtx);
 968
 969         /* Allow device probing and trigger re-probing of deferred devices */
 970         device_unblock_probing();
 971         trace_suspend_resume(TPS("dpm_complete"), state.event, false);
 972 }
 973
 974 /**
 975  * dpm_resume_end - Execute "resume" callbacks and complete system transition.
 976  * @state: PM transition of the system being carried out.
 977  *
 978  * Execute "resume" callbacks for all devices and complete the PM transition of
 979  * the system.
 980  */
 981 void dpm_resume_end(pm_message_t state)
 982 {
 983         dpm_resume(state);
 984         dpm_complete(state);
 985 }
 986 EXPORT_SYMBOL_GPL(dpm_resume_end);
 987
 988
 989 /*------------------------- Suspend routines -------------------------*/
 990
 991 /**
 992  * resume_event - Return a "resume" message for given "suspend" sleep state.
 993  * @sleep_state: PM message representing a sleep state.
 994  *
 995  * Return a PM message representing the resume event corresponding to given
 996  * sleep state.
 997  */
 998 static pm_message_t resume_event(pm_message_t sleep_state)
 999 {
1000         switch (sleep_state.event) {
1001         case PM_EVENT_SUSPEND:
1002                 return PMSG_RESUME;
1003         case PM_EVENT_FREEZE:
1004         case PM_EVENT_QUIESCE:
1005                 return PMSG_RECOVER;
1006         case PM_EVENT_HIBERNATE:
1007                 return PMSG_RESTORE;
1008         }
1009         return PMSG_ON;
1010 }
1011
1012 /**
1013  * device_suspend_noirq - Execute a "late suspend" callback for given device.
1014  * @dev: Device to handle.
1015  * @state: PM transition of the system being carried out.
1016  * @async: If true, the device is being suspended asynchronously.
1017  *
1018  * The driver of @dev will not receive interrupts while this function is being
1019  * executed.
1020  */
1021 static int __device_suspend_noirq(struct device *dev, pm_message_t state, bool async)
1022 {
1023         pm_callback_t callback = NULL;
1024         char *info = NULL;
1025         int error = 0;
1026
1027         TRACE_DEVICE(dev);
1028         TRACE_SUSPEND(0);
1029
1030         dpm_wait_for_children(dev, async);
1031
1032         if (async_error)
1033                 goto Complete;
1034
1035         if (pm_wakeup_pending()) {
1036                 async_error = -EBUSY;
1037                 goto Complete;
1038         }
1039
1040         if (dev->power.syscore || dev->power.direct_complete)
1041                 goto Complete;
1042
1043         if (dev->pm_domain) {
1044                 info = "noirq power domain ";
1045                 callback = pm_noirq_op(&dev->pm_domain->ops, state);
1046         } else if (dev->type && dev->type->pm) {
1047                 info = "noirq type ";
1048                 callback = pm_noirq_op(dev->type->pm, state);
1049         } else if (dev->class && dev->class->pm) {
1050                 info = "noirq class ";
1051                 callback = pm_noirq_op(dev->class->pm, state);
1052         } else if (dev->bus && dev->bus->pm) {
1053                 info = "noirq bus ";
1054                 callback = pm_noirq_op(dev->bus->pm, state);
1055         }
1056
1057         if (!callback && dev->driver && dev->driver->pm) {
1058                 info = "noirq driver ";
1059                 callback = pm_noirq_op(dev->driver->pm, state);
1060         }
1061
1062         error = dpm_run_callback(callback, dev, state, info);
1063         if (!error)
1064                 dev->power.is_noirq_suspended = true;
1065         else
1066                 async_error = error;
1067
1068 Complete:
1069         complete_all(&dev->power.completion);
1070         TRACE_SUSPEND(error);
1071         return error;
1072 }
1073
1074 static void async_suspend_noirq(void *data, async_cookie_t cookie)
1075 {
1076         struct device *dev = (struct device *)data;
1077         int error;
1078
1079         error = __device_suspend_noirq(dev, pm_transition, true);
1080         if (error) {
1081                 dpm_save_failed_dev(dev_name(dev));
1082                 pm_dev_err(dev, pm_transition, " async", error);
1083         }
1084
1085         put_device(dev);
1086 }
1087
1088 static int device_suspend_noirq(struct device *dev)
1089 {
1090         reinit_completion(&dev->power.completion);
1091
1092         if (is_async(dev)) {
1093                 get_device(dev);
1094                 async_schedule(async_suspend_noirq, dev);
1095                 return 0;
1096         }
1097         return __device_suspend_noirq(dev, pm_transition, false);
1098 }
1099
1100 /**
1101  * dpm_suspend_noirq - Execute "noirq suspend" callbacks for all devices.
1102  * @state: PM transition of the system being carried out.
1103  *
1104  * Prevent device drivers from receiving interrupts and call the "noirq" suspend
1105  * handlers for all non-sysdev devices.
1106  */
1107 int dpm_suspend_noirq(pm_message_t state)
1108 {
1109         ktime_t starttime = ktime_get();
1110         int error = 0;
1111
1112         trace_suspend_resume(TPS("dpm_suspend_noirq"), state.event, true);
1113         cpuidle_pause();
1114         device_wakeup_arm_wake_irqs();
1115         suspend_device_irqs();
1116         mutex_lock(&dpm_list_mtx);
1117         pm_transition = state;
1118         async_error = 0;
1119
1120         while (!list_empty(&dpm_late_early_list)) {
1121                 struct device *dev = to_device(dpm_late_early_list.prev);
1122
1123                 get_device(dev);
1124                 mutex_unlock(&dpm_list_mtx);
1125
1126                 error = device_suspend_noirq(dev);
1127
1128                 mutex_lock(&dpm_list_mtx);
1129                 if (error) {
1130                         pm_dev_err(dev, state, " noirq", error);
1131                         dpm_save_failed_dev(dev_name(dev));
1132                         put_device(dev);
1133                         break;
1134                 }
1135                 if (!list_empty(&dev->power.entry))
1136                         list_move(&dev->power.entry, &dpm_noirq_list);
1137                 put_device(dev);
1138
1139                 if (async_error)
1140                         break;
1141         }
1142         mutex_unlock(&dpm_list_mtx);
1143         async_synchronize_full();
1144         if (!error)
1145                 error = async_error;
1146
1147         if (error) {
1148                 suspend_stats.failed_suspend_noirq++;
1149                 dpm_save_failed_step(SUSPEND_SUSPEND_NOIRQ);
1150                 dpm_resume_noirq(resume_event(state));
1151         } else {
1152                 dpm_show_time(starttime, state, "noirq");
1153         }
1154         trace_suspend_resume(TPS("dpm_suspend_noirq"), state.event, false);
1155         return error;
1156 }
1157
1158 /**
1159  * device_suspend_late - Execute a "late suspend" callback for given device.
1160  * @dev: Device to handle.
1161  * @state: PM transition of the system being carried out.
1162  * @async: If true, the device is being suspended asynchronously.
1163  *
1164  * Runtime PM is disabled for @dev while this function is being executed.
1165  */
1166 static int __device_suspend_late(struct device *dev, pm_message_t state, bool async)
1167 {
1168         pm_callback_t callback = NULL;
1169         char *info = NULL;
1170         int error = 0;
1171
1172         TRACE_DEVICE(dev);
1173         TRACE_SUSPEND(0);
1174
1175         __pm_runtime_disable(dev, false);
1176
1177         dpm_wait_for_children(dev, async);
1178
1179         if (async_error)
1180                 goto Complete;
1181
1182         if (pm_wakeup_pending()) {
1183                 async_error = -EBUSY;
1184                 goto Complete;
1185         }
1186
1187         if (dev->power.syscore || dev->power.direct_complete)
1188                 goto Complete;
1189
1190         if (dev->pm_domain) {
1191                 info = "late power domain ";
1192                 callback = pm_late_early_op(&dev->pm_domain->ops, state);
1193         } else if (dev->type && dev->type->pm) {
1194                 info = "late type ";
1195                 callback = pm_late_early_op(dev->type->pm, state);
1196         } else if (dev->class && dev->class->pm) {
1197                 info = "late class ";
1198                 callback = pm_late_early_op(dev->class->pm, state);
1199         } else if (dev->bus && dev->bus->pm) {
1200                 info = "late bus ";
1201                 callback = pm_late_early_op(dev->bus->pm, state);
1202         }
1203
1204         if (!callback && dev->driver && dev->driver->pm) {
1205                 info = "late driver ";
1206                 callback = pm_late_early_op(dev->driver->pm, state);
1207         }
1208
1209         error = dpm_run_callback(callback, dev, state, info);
1210         if (!error)
1211                 dev->power.is_late_suspended = true;
1212         else
1213                 async_error = error;
1214
1215 Complete:
1216         TRACE_SUSPEND(error);
1217         complete_all(&dev->power.completion);
1218         return error;
1219 }
1220
1221 static void async_suspend_late(void *data, async_cookie_t cookie)
1222 {
1223         struct device *dev = (struct device *)data;
1224         int error;
1225
1226         error = __device_suspend_late(dev, pm_transition, true);
1227         if (error) {
1228                 dpm_save_failed_dev(dev_name(dev));
1229                 pm_dev_err(dev, pm_transition, " async", error);
1230         }
1231         put_device(dev);
1232 }
1233
1234 static int device_suspend_late(struct device *dev)
1235 {
1236         reinit_completion(&dev->power.completion);
1237
1238         if (is_async(dev)) {
1239                 get_device(dev);
1240                 async_schedule(async_suspend_late, dev);
1241                 return 0;
1242         }
1243
1244         return __device_suspend_late(dev, pm_transition, false);
1245 }
1246
1247 /**
1248  * dpm_suspend_late - Execute "late suspend" callbacks for all devices.
1249  * @state: PM transition of the system being carried out.
1250  */
1251 int dpm_suspend_late(pm_message_t state)
1252 {
1253         ktime_t starttime = ktime_get();
1254         int error = 0;
1255
1256         trace_suspend_resume(TPS("dpm_suspend_late"), state.event, true);
1257         mutex_lock(&dpm_list_mtx);
1258         pm_transition = state;
1259         async_error = 0;
1260
1261         while (!list_empty(&dpm_suspended_list)) {
1262                 struct device *dev = to_device(dpm_suspended_list.prev);
1263
1264                 get_device(dev);
1265                 mutex_unlock(&dpm_list_mtx);
1266
1267                 error = device_suspend_late(dev);
1268
1269                 mutex_lock(&dpm_list_mtx);
1270                 if (!list_empty(&dev->power.entry))
1271                         list_move(&dev->power.entry, &dpm_late_early_list);
1272
1273                 if (error) {
1274                         pm_dev_err(dev, state, " late", error);
1275                         dpm_save_failed_dev(dev_name(dev));
1276                         put_device(dev);
1277                         break;
1278                 }
1279                 put_device(dev);
1280
1281                 if (async_error)
1282                         break;
1283         }
1284         mutex_unlock(&dpm_list_mtx);
1285         async_synchronize_full();
1286         if (!error)
1287                 error = async_error;
1288         if (error) {
1289                 suspend_stats.failed_suspend_late++;
1290                 dpm_save_failed_step(SUSPEND_SUSPEND_LATE);
1291                 dpm_resume_early(resume_event(state));
1292         } else {
1293                 dpm_show_time(starttime, state, "late");
1294         }
1295         trace_suspend_resume(TPS("dpm_suspend_late"), state.event, false);
1296         return error;
1297 }
1298
1299 /**
1300  * dpm_suspend_end - Execute "late" and "noirq" device suspend callbacks.
1301  * @state: PM transition of the system being carried out.
1302  */
1303 int dpm_suspend_end(pm_message_t state)
1304 {
1305         int error = dpm_suspend_late(state);
1306         if (error)
1307                 return error;
1308
1309         error = dpm_suspend_noirq(state);
1310         if (error) {
1311                 dpm_resume_early(resume_event(state));
1312                 return error;
1313         }
1314
1315         return 0;
1316 }
1317 EXPORT_SYMBOL_GPL(dpm_suspend_end);
1318
1319 /**
1320  * legacy_suspend - Execute a legacy (bus or class) suspend callback for device.
1321  * @dev: Device to suspend.
1322  * @state: PM transition of the system being carried out.
1323  * @cb: Suspend callback to execute.
1324  * @info: string description of caller.
1325  */
1326 static int legacy_suspend(struct device *dev, pm_message_t state,
1327                           int (*cb)(struct device *dev, pm_message_t state),
1328                           char *info)
1329 {
1330         int error;
1331         ktime_t calltime;
1332
1333         calltime = initcall_debug_start(dev);
1334
1335         trace_device_pm_callback_start(dev, info, state.event);
1336         error = cb(dev, state);
1337         trace_device_pm_callback_end(dev, error);
1338         suspend_report_result(cb, error);
1339
1340         initcall_debug_report(dev, calltime, error, state, info);
1341
1342         return error;
1343 }
1344
1345 /**
1346  * device_suspend - Execute "suspend" callbacks for given device.
1347  * @dev: Device to handle.
1348  * @state: PM transition of the system being carried out.
1349  * @async: If true, the device is being suspended asynchronously.
1350  */
1351 static int __device_suspend(struct device *dev, pm_message_t state, bool async)
1352 {
1353         pm_callback_t callback = NULL;
1354         char *info = NULL;
1355         int error = 0;
1356         DECLARE_DPM_WATCHDOG_ON_STACK(wd);
1357
1358         TRACE_DEVICE(dev);
1359         TRACE_SUSPEND(0);
1360
1361         dpm_wait_for_children(dev, async);
1362
1363         if (async_error)
1364                 goto Complete;
1365
1366         /*
1367          * If a device configured to wake up the system from sleep states
1368          * has been suspended at run time and there's a resume request pending
1369          * for it, this is equivalent to the device signaling wakeup, so the
1370          * system suspend operation should be aborted.
1371          */
1372         if (pm_runtime_barrier(dev) && device_may_wakeup(dev))
1373                 pm_wakeup_event(dev, 0);
1374
1375         if (pm_wakeup_pending()) {
1376                 async_error = -EBUSY;
1377                 goto Complete;
1378         }
1379
1380         if (dev->power.syscore)
1381                 goto Complete;
1382
1383         if (dev->power.direct_complete) {
1384                 if (pm_runtime_status_suspended(dev)) {
1385                         pm_runtime_disable(dev);
1386                         if (pm_runtime_status_suspended(dev))
1387                                 goto Complete;
1388
1389                         pm_runtime_enable(dev);
1390                 }
1391                 dev->power.direct_complete = false;
1392         }
1393
1394         dpm_watchdog_set(&wd, dev);
1395         device_lock(dev);
1396
1397         if (dev->pm_domain) {
1398                 info = "power domain ";
1399                 callback = pm_op(&dev->pm_domain->ops, state);
1400                 goto Run;
1401         }
1402
1403         if (dev->type && dev->type->pm) {
1404                 info = "type ";
1405                 callback = pm_op(dev->type->pm, state);
1406                 goto Run;
1407         }
1408
1409         if (dev->class) {
1410                 if (dev->class->pm) {
1411                         info = "class ";
1412                         callback = pm_op(dev->class->pm, state);
1413                         goto Run;
1414                 } else if (dev->class->suspend) {
1415                         pm_dev_dbg(dev, state, "legacy class ");
1416                         error = legacy_suspend(dev, state, dev->class->suspend,
1417                                                 "legacy class ");
1418                         goto End;
1419                 }
1420         }
1421
1422         if (dev->bus) {
1423                 if (dev->bus->pm) {
1424                         info = "bus ";
1425                         callback = pm_op(dev->bus->pm, state);
1426                 } else if (dev->bus->suspend) {
1427                         pm_dev_dbg(dev, state, "legacy bus ");
1428                         error = legacy_suspend(dev, state, dev->bus->suspend,
1429                                                 "legacy bus ");
1430                         goto End;
1431                 }
1432         }
1433
1434  Run:
1435         if (!callback && dev->driver && dev->driver->pm) {
1436                 info = "driver ";
1437                 callback = pm_op(dev->driver->pm, state);
1438         }
1439
1440         error = dpm_run_callback(callback, dev, state, info);
1441
1442  End:
1443         if (!error) {
1444                 struct device *parent = dev->parent;
1445
1446                 dev->power.is_suspended = true;
1447                 if (parent) {
1448                         spin_lock_irq(&parent->power.lock);
1449
1450                         dev->parent->power.direct_complete = false;
1451                         if (dev->power.wakeup_path
1452                             && !dev->parent->power.ignore_children)
1453                                 dev->parent->power.wakeup_path = true;
1454
1455                         spin_unlock_irq(&parent->power.lock);
1456                 }
1457         }
1458
1459         device_unlock(dev);
1460         dpm_watchdog_clear(&wd);
1461
1462  Complete:
1463         complete_all(&dev->power.completion);
1464         if (error)
1465                 async_error = error;
1466
1467         TRACE_SUSPEND(error);
1468         return error;
1469 }
1470
1471 static void async_suspend(void *data, async_cookie_t cookie)
1472 {
1473         struct device *dev = (struct device *)data;
1474         int error;
1475
1476         error = __device_suspend(dev, pm_transition, true);
1477         if (error) {
1478                 dpm_save_failed_dev(dev_name(dev));
1479                 pm_dev_err(dev, pm_transition, " async", error);
1480         }
1481
1482         put_device(dev);
1483 }
1484
1485 static int device_suspend(struct device *dev)
1486 {
1487         reinit_completion(&dev->power.completion);
1488
1489         if (is_async(dev)) {
1490                 get_device(dev);
1491                 async_schedule(async_suspend, dev);
1492                 return 0;
1493         }
1494
1495         return __device_suspend(dev, pm_transition, false);
1496 }
1497
1498 /**
1499  * dpm_suspend - Execute "suspend" callbacks for all non-sysdev devices.
1500  * @state: PM transition of the system being carried out.
1501  */
1502 int dpm_suspend(pm_message_t state)
1503 {
1504         ktime_t starttime = ktime_get();
1505         int error = 0;
1506
1507         trace_suspend_resume(TPS("dpm_suspend"), state.event, true);
1508         might_sleep();
1509
1510         cpufreq_suspend();
1511
1512         mutex_lock(&dpm_list_mtx);
1513         pm_transition = state;
1514         async_error = 0;
1515         while (!list_empty(&dpm_prepared_list)) {
1516                 struct device *dev = to_device(dpm_prepared_list.prev);
1517
1518                 get_device(dev);
1519                 mutex_unlock(&dpm_list_mtx);
1520
1521                 error = device_suspend(dev);
1522
1523                 mutex_lock(&dpm_list_mtx);
1524                 if (error) {
1525                         pm_dev_err(dev, state, "", error);
1526                         dpm_save_failed_dev(dev_name(dev));
1527                         put_device(dev);
1528                         break;
1529                 }
1530                 if (!list_empty(&dev->power.entry))
1531                         list_move(&dev->power.entry, &dpm_suspended_list);
1532                 put_device(dev);
1533                 if (async_error)
1534                         break;
1535         }
1536         mutex_unlock(&dpm_list_mtx);
1537         async_synchronize_full();
1538         if (!error)
1539                 error = async_error;
1540         if (error) {
1541                 suspend_stats.failed_suspend++;
1542                 dpm_save_failed_step(SUSPEND_SUSPEND);
1543         } else
1544                 dpm_show_time(starttime, state, NULL);
1545         trace_suspend_resume(TPS("dpm_suspend"), state.event, false);
1546         return error;
1547 }
1548
1549 /**
1550  * device_prepare - Prepare a device for system power transition.
1551  * @dev: Device to handle.
1552  * @state: PM transition of the system being carried out.
1553  *
1554  * Execute the ->prepare() callback(s) for given device.  No new children of the
1555  * device may be registered after this function has returned.
1556  */
1557 static int device_prepare(struct device *dev, pm_message_t state)
1558 {
1559         int (*callback)(struct device *) = NULL;
1560         int ret = 0;
1561
1562         if (dev->power.syscore)
1563                 return 0;
1564
1565         /*
1566          * If a device's parent goes into runtime suspend at the wrong time,
1567          * it won't be possible to resume the device.  To prevent this we
1568          * block runtime suspend here, during the prepare phase, and allow
1569          * it again during the complete phase.
1570          */
1571         pm_runtime_get_noresume(dev);
1572
1573         device_lock(dev);
1574
1575         dev->power.wakeup_path = device_may_wakeup(dev);
1576
1577         if (dev->power.no_pm_callbacks) {
1578                 ret = 1;        /* Let device go direct_complete */
1579                 goto unlock;
1580         }
1581
1582         if (dev->pm_domain)
1583                 callback = dev->pm_domain->ops.prepare;
1584         else if (dev->type && dev->type->pm)
1585                 callback = dev->type->pm->prepare;
1586         else if (dev->class && dev->class->pm)
1587                 callback = dev->class->pm->prepare;
1588         else if (dev->bus && dev->bus->pm)
1589                 callback = dev->bus->pm->prepare;
1590
1591         if (!callback && dev->driver && dev->driver->pm)
1592                 callback = dev->driver->pm->prepare;
1593
1594         if (callback)
1595                 ret = callback(dev);
1596
1597 unlock:
1598         device_unlock(dev);
1599
1600         if (ret < 0) {
1601                 suspend_report_result(callback, ret);
1602                 pm_runtime_put(dev);
1603                 return ret;
1604         }
1605         /*
1606          * A positive return value from ->prepare() means "this device appears
1607          * to be runtime-suspended and its state is fine, so if it really is
1608          * runtime-suspended, you can leave it in that state provided that you
1609          * will do the same thing with all of its descendants".  This only
1610          * applies to suspend transitions, however.
1611          */
1612         spin_lock_irq(&dev->power.lock);
1613         dev->power.direct_complete = ret > 0 && state.event == PM_EVENT_SUSPEND;
1614         spin_unlock_irq(&dev->power.lock);
1615         return 0;
1616 }
1617
1618 /**
1619  * dpm_prepare - Prepare all non-sysdev devices for a system PM transition.
1620  * @state: PM transition of the system being carried out.
1621  *
1622  * Execute the ->prepare() callback(s) for all devices.
1623  */
1624 int dpm_prepare(pm_message_t state)
1625 {
1626         int error = 0;
1627
1628         trace_suspend_resume(TPS("dpm_prepare"), state.event, true);
1629         might_sleep();
1630
1631         /*
1632          * Give a chance for the known devices to complete their probes, before
1633          * disable probing of devices. This sync point is important at least
1634          * at boot time + hibernation restore.
1635          */
1636         wait_for_device_probe();
1637         /*
1638          * It is unsafe if probing of devices will happen during suspend or
1639          * hibernation and system behavior will be unpredictable in this case.
1640          * So, let's prohibit device's probing here and defer their probes
1641          * instead. The normal behavior will be restored in dpm_complete().
1642          */
1643         device_block_probing();
1644
1645         mutex_lock(&dpm_list_mtx);
1646         while (!list_empty(&dpm_list)) {
1647                 struct device *dev = to_device(dpm_list.next);
1648
1649                 get_device(dev);
1650                 mutex_unlock(&dpm_list_mtx);
1651
1652                 trace_device_pm_callback_start(dev, "", state.event);
1653                 error = device_prepare(dev, state);
1654                 trace_device_pm_callback_end(dev, error);
1655
1656                 mutex_lock(&dpm_list_mtx);
1657                 if (error) {
1658                         if (error == -EAGAIN) {
1659                                 put_device(dev);
1660                                 error = 0;
1661                                 continue;
1662                         }
1663                         printk(KERN_INFO "PM: Device %s not prepared "
1664                                 "for power transition: code %d\n",
1665                                 dev_name(dev), error);
1666                         put_device(dev);
1667                         break;
1668                 }
1669                 dev->power.is_prepared = true;
1670                 if (!list_empty(&dev->power.entry))
1671                         list_move_tail(&dev->power.entry, &dpm_prepared_list);
1672                 put_device(dev);
1673         }
1674         mutex_unlock(&dpm_list_mtx);
1675         trace_suspend_resume(TPS("dpm_prepare"), state.event, false);
1676         return error;
1677 }
1678
1679 /**
1680  * dpm_suspend_start - Prepare devices for PM transition and suspend them.
1681  * @state: PM transition of the system being carried out.
1682  *
1683  * Prepare all non-sysdev devices for system PM transition and execute "suspend"
1684  * callbacks for them.
1685  */
1686 int dpm_suspend_start(pm_message_t state)
1687 {
1688         int error;
1689
1690         error = dpm_prepare(state);
1691         if (error) {
1692                 suspend_stats.failed_prepare++;
1693                 dpm_save_failed_step(SUSPEND_PREPARE);
1694         } else
1695                 error = dpm_suspend(state);
1696         return error;
1697 }
1698 EXPORT_SYMBOL_GPL(dpm_suspend_start);
1699
1700 void __suspend_report_result(const char *function, void *fn, int ret)
1701 {
1702         if (ret)
1703                 printk(KERN_ERR "%s(): %pF returns %d\n", function, fn, ret);
1704 }
1705 EXPORT_SYMBOL_GPL(__suspend_report_result);
1706
1707 /**
1708  * device_pm_wait_for_dev - Wait for suspend/resume of a device to complete.
1709  * @dev: Device to wait for.
1710  * @subordinate: Device that needs to wait for @dev.
1711  */
1712 int device_pm_wait_for_dev(struct device *subordinate, struct device *dev)
1713 {
1714         dpm_wait(dev, subordinate->power.async_suspend);
1715         return async_error;
1716 }
1717 EXPORT_SYMBOL_GPL(device_pm_wait_for_dev);
1718
1719 /**
1720  * dpm_for_each_dev - device iterator.
1721  * @data: data for the callback.
1722  * @fn: function to be called for each device.
1723  *
1724  * Iterate over devices in dpm_list, and call @fn for each device,
1725  * passing it @data.
1726  */
1727 void dpm_for_each_dev(void *data, void (*fn)(struct device *, void *))
1728 {
1729         struct device *dev;
1730
1731         if (!fn)
1732                 return;
1733
1734         device_pm_lock();
1735         list_for_each_entry(dev, &dpm_list, power.entry)
1736                 fn(dev, data);
1737         device_pm_unlock();
1738 }
1739 EXPORT_SYMBOL_GPL(dpm_for_each_dev);
1740
1741 static bool pm_ops_is_empty(const struct dev_pm_ops *ops)
1742 {
1743         if (!ops)
1744                 return true;
1745
1746         return !ops->prepare &&
1747                !ops->suspend &&
1748                !ops->suspend_late &&
1749                !ops->suspend_noirq &&
1750                !ops->resume_noirq &&
1751                !ops->resume_early &&
1752                !ops->resume &&
1753                !ops->complete;
1754 }
1755
1756 void device_pm_check_callbacks(struct device *dev)
1757 {
1758         spin_lock_irq(&dev->power.lock);
1759         dev->power.no_pm_callbacks =
1760                 (!dev->bus || pm_ops_is_empty(dev->bus->pm)) &&
1761                 (!dev->class || pm_ops_is_empty(dev->class->pm)) &&
1762                 (!dev->type || pm_ops_is_empty(dev->type->pm)) &&
1763                 (!dev->pm_domain || pm_ops_is_empty(&dev->pm_domain->ops)) &&
1764                 (!dev->driver || pm_ops_is_empty(dev->driver->pm));
1765         spin_unlock_irq(&dev->power.lock);
1766 }