drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *            (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
   7  *
   8  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   9  *      Added handling for CPU hotplug
  10  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  11  *      Fix handling for CPU hotplug -- affected CPUs
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License version 2 as
  15  * published by the Free Software Foundation.
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/cpu.h>
  21 #include <linux/cpufreq.h>
  22 #include <linux/delay.h>
  23 #include <linux/device.h>
  24 #include <linux/init.h>
  25 #include <linux/kernel_stat.h>
  26 #include <linux/module.h>
  27 #include <linux/mutex.h>
  28 #include <linux/slab.h>
  29 #include <linux/suspend.h>
  30 #include <linux/syscore_ops.h>
  31 #include <linux/tick.h>
  32 #include <trace/events/power.h>
  33
  34 static LIST_HEAD(cpufreq_policy_list);
  35
  36 static inline bool policy_is_inactive(struct cpufreq_policy *policy)
  37 {
  38         return cpumask_empty(policy->cpus);
  39 }
  40
  41 /* Macros to iterate over CPU policies */
  42 #define for_each_suitable_policy(__policy, __active)                     \
  43         list_for_each_entry(__policy, &cpufreq_policy_list, policy_list) \
  44                 if ((__active) == !policy_is_inactive(__policy))
  45
  46 #define for_each_active_policy(__policy)                \
  47         for_each_suitable_policy(__policy, true)
  48 #define for_each_inactive_policy(__policy)              \
  49         for_each_suitable_policy(__policy, false)
  50
  51 #define for_each_policy(__policy)                       \
  52         list_for_each_entry(__policy, &cpufreq_policy_list, policy_list)
  53
  54 /* Iterate over governors */
  55 static LIST_HEAD(cpufreq_governor_list);
  56 #define for_each_governor(__governor)                           \
  57         list_for_each_entry(__governor, &cpufreq_governor_list, governor_list)
  58
  59 /**
  60  * The "cpufreq driver" - the arch- or hardware-dependent low
  61  * level driver of CPUFreq support, and its spinlock. This lock
  62  * also protects the cpufreq_cpu_data array.
  63  */
  64 static struct cpufreq_driver *cpufreq_driver;
  65 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
  66 static DEFINE_RWLOCK(cpufreq_driver_lock);
  67
  68 /* Flag to suspend/resume CPUFreq governors */
  69 static bool cpufreq_suspended;
  70
  71 static inline bool has_target(void)
  72 {
  73         return cpufreq_driver->target_index || cpufreq_driver->target;
  74 }
  75
  76 /* internal prototypes */
  77 static unsigned int __cpufreq_get(struct cpufreq_policy *policy);
  78 static int cpufreq_init_governor(struct cpufreq_policy *policy);
  79 static void cpufreq_exit_governor(struct cpufreq_policy *policy);
  80 static int cpufreq_start_governor(struct cpufreq_policy *policy);
  81 static void cpufreq_stop_governor(struct cpufreq_policy *policy);
  82 static void cpufreq_governor_limits(struct cpufreq_policy *policy);
  83
  84 /**
  85  * Two notifier lists: the "policy" list is involved in the
  86  * validation process for a new CPU frequency policy; the
  87  * "transition" list for kernel code that needs to handle
  88  * changes to devices when the CPU clock speed changes.
  89  * The mutex locks both lists.
  90  */
  91 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
  92 static struct srcu_notifier_head cpufreq_transition_notifier_list;
  93
  94 static bool init_cpufreq_transition_notifier_list_called;
  95 static int __init init_cpufreq_transition_notifier_list(void)
  96 {
  97         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
  98         init_cpufreq_transition_notifier_list_called = true;
  99         return 0;
 100 }
 101 pure_initcall(init_cpufreq_transition_notifier_list);
 102
 103 static int off __read_mostly;
 104 static int cpufreq_disabled(void)
 105 {
 106         return off;
 107 }
 108 void disable_cpufreq(void)
 109 {
 110         off = 1;
 111 }
 112 static DEFINE_MUTEX(cpufreq_governor_mutex);
 113
 114 bool have_governor_per_policy(void)
 115 {
 116         return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY);
 117 }
 118 EXPORT_SYMBOL_GPL(have_governor_per_policy);
 119
 120 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
 121 {
 122         if (have_governor_per_policy())
 123                 return &policy->kobj;
 124         else
 125                 return cpufreq_global_kobject;
 126 }
 127 EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
 128
 129 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
 130 {
 131         u64 idle_time;
 132         u64 cur_wall_time;
 133         u64 busy_time;
 134
 135         cur_wall_time = jiffies64_to_nsecs(get_jiffies_64());
 136
 137         busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
 138         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
 139         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
 140         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
 141         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
 142         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];
 143
 144         idle_time = cur_wall_time - busy_time;
 145         if (wall)
 146                 *wall = div_u64(cur_wall_time, NSEC_PER_USEC);
 147
 148         return div_u64(idle_time, NSEC_PER_USEC);
 149 }
 150
 151 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
 152 {
 153         u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
 154
 155         if (idle_time == -1ULL)
 156                 return get_cpu_idle_time_jiffy(cpu, wall);
 157         else if (!io_busy)
 158                 idle_time += get_cpu_iowait_time_us(cpu, wall);
 159
 160         return idle_time;
 161 }
 162 EXPORT_SYMBOL_GPL(get_cpu_idle_time);
 163
 164 __weak void arch_set_freq_scale(struct cpumask *cpus, unsigned long cur_freq,
 165                 unsigned long max_freq)
 166 {
 167 }
 168 EXPORT_SYMBOL_GPL(arch_set_freq_scale);
 169
 170 /*
 171  * This is a generic cpufreq init() routine which can be used by cpufreq
 172  * drivers of SMP systems. It will do following:
 173  * - validate & show freq table passed
 174  * - set policies transition latency
 175  * - policy->cpus with all possible CPUs
 176  */
 177 int cpufreq_generic_init(struct cpufreq_policy *policy,
 178                 struct cpufreq_frequency_table *table,
 179                 unsigned int transition_latency)
 180 {
 181         int ret;
 182
 183         ret = cpufreq_table_validate_and_show(policy, table);
 184         if (ret) {
 185                 pr_err("%s: invalid frequency table: %d\n", __func__, ret);
 186                 return ret;
 187         }
 188
 189         policy->cpuinfo.transition_latency = transition_latency;
 190
 191         /*
 192          * The driver only supports the SMP configuration where all processors
 193          * share the clock and voltage and clock.
 194          */
 195         cpumask_setall(policy->cpus);
 196
 197         return 0;
 198 }
 199 EXPORT_SYMBOL_GPL(cpufreq_generic_init);
 200
 201 struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu)
 202 {
 203         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
 204
 205         return policy && cpumask_test_cpu(cpu, policy->cpus) ? policy : NULL;
 206 }
 207 EXPORT_SYMBOL_GPL(cpufreq_cpu_get_raw);
 208
 209 unsigned int cpufreq_generic_get(unsigned int cpu)
 210 {
 211         struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);
 212
 213         if (!policy || IS_ERR(policy->clk)) {
 214                 pr_err("%s: No %s associated to cpu: %d\n",
 215                        __func__, policy ? "clk" : "policy", cpu);
 216                 return 0;
 217         }
 218
 219         return clk_get_rate(policy->clk) / 1000;
 220 }
 221 EXPORT_SYMBOL_GPL(cpufreq_generic_get);
 222
 223 /**
 224  * cpufreq_cpu_get: returns policy for a cpu and marks it busy.
 225  *
 226  * @cpu: cpu to find policy for.
 227  *
 228  * This returns policy for 'cpu', returns NULL if it doesn't exist.
 229  * It also increments the kobject reference count to mark it busy and so would
 230  * require a corresponding call to cpufreq_cpu_put() to decrement it back.
 231  * If corresponding call cpufreq_cpu_put() isn't made, the policy wouldn't be
 232  * freed as that depends on the kobj count.
 233  *
 234  * Return: A valid policy on success, otherwise NULL on failure.
 235  */
 236 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 237 {
 238         struct cpufreq_policy *policy = NULL;
 239         unsigned long flags;
 240
 241         if (WARN_ON(cpu >= nr_cpu_ids))
 242                 return NULL;
 243
 244         /* get the cpufreq driver */
 245         read_lock_irqsave(&cpufreq_driver_lock, flags);
 246
 247         if (cpufreq_driver) {
 248                 /* get the CPU */
 249                 policy = cpufreq_cpu_get_raw(cpu);
 250                 if (policy)
 251                         kobject_get(&policy->kobj);
 252         }
 253
 254         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 255
 256         return policy;
 257 }
 258 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 259
 260 /**
 261  * cpufreq_cpu_put: Decrements the usage count of a policy
 262  *
 263  * @policy: policy earlier returned by cpufreq_cpu_get().
 264  *
 265  * This decrements the kobject reference count incremented earlier by calling
 266  * cpufreq_cpu_get().
 267  */
 268 void cpufreq_cpu_put(struct cpufreq_policy *policy)
 269 {
 270         kobject_put(&policy->kobj);
 271 }
 272 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 273
 274 /*********************************************************************
 275  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 276  *********************************************************************/
 277
 278 /**
 279  * adjust_jiffies - adjust the system "loops_per_jiffy"
 280  *
 281  * This function alters the system "loops_per_jiffy" for the clock
 282  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 283  * systems as each CPU might be scaled differently. So, use the arch
 284  * per-CPU loops_per_jiffy value wherever possible.
 285  */
 286 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 287 {
 288 #ifndef CONFIG_SMP
 289         static unsigned long l_p_j_ref;
 290         static unsigned int l_p_j_ref_freq;
 291
 292         if (ci->flags & CPUFREQ_CONST_LOOPS)
 293                 return;
 294
 295         if (!l_p_j_ref_freq) {
 296                 l_p_j_ref = loops_per_jiffy;
 297                 l_p_j_ref_freq = ci->old;
 298                 pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n",
 299                          l_p_j_ref, l_p_j_ref_freq);
 300         }
 301         if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) {
 302                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 303                                                                 ci->new);
 304                 pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n",
 305                          loops_per_jiffy, ci->new);
 306         }
 307 #endif
 308 }
 309
 310 static void __cpufreq_notify_transition(struct cpufreq_policy *policy,
 311                 struct cpufreq_freqs *freqs, unsigned int state)
 312 {
 313         BUG_ON(irqs_disabled());
 314
 315         if (cpufreq_disabled())
 316                 return;
 317
 318         freqs->flags = cpufreq_driver->flags;
 319         pr_debug("notification %u of frequency transition to %u kHz\n",
 320                  state, freqs->new);
 321
 322         switch (state) {
 323
 324         case CPUFREQ_PRECHANGE:
 325                 /* detect if the driver reported a value as "old frequency"
 326                  * which is not equal to what the cpufreq core thinks is
 327                  * "old frequency".
 328                  */
 329                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 330                         if ((policy) && (policy->cpu == freqs->cpu) &&
 331                             (policy->cur) && (policy->cur != freqs->old)) {
 332                                 pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n",
 333                                          freqs->old, policy->cur);
 334                                 freqs->old = policy->cur;
 335                         }
 336                 }
 337                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 338                                 CPUFREQ_PRECHANGE, freqs);
 339                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 340                 break;
 341
 342         case CPUFREQ_POSTCHANGE:
 343                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 344                 pr_debug("FREQ: %lu - CPU: %lu\n",
 345                          (unsigned long)freqs->new, (unsigned long)freqs->cpu);
 346                 trace_cpu_frequency(freqs->new, freqs->cpu);
 347                 cpufreq_stats_record_transition(policy, freqs->new);
 348                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 349                                 CPUFREQ_POSTCHANGE, freqs);
 350                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 351                         policy->cur = freqs->new;
 352                 break;
 353         }
 354 }
 355
 356 /**
 357  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 358  * on frequency transition.
 359  *
 360  * This function calls the transition notifiers and the "adjust_jiffies"
 361  * function. It is called twice on all CPU frequency changes that have
 362  * external effects.
 363  */
 364 static void cpufreq_notify_transition(struct cpufreq_policy *policy,
 365                 struct cpufreq_freqs *freqs, unsigned int state)
 366 {
 367         for_each_cpu(freqs->cpu, policy->cpus)
 368                 __cpufreq_notify_transition(policy, freqs, state);
 369 }
 370
 371 /* Do post notifications when there are chances that transition has failed */
 372 static void cpufreq_notify_post_transition(struct cpufreq_policy *policy,
 373                 struct cpufreq_freqs *freqs, int transition_failed)
 374 {
 375         cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
 376         if (!transition_failed)
 377                 return;
 378
 379         swap(freqs->old, freqs->new);
 380         cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
 381         cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
 382 }
 383
 384 void cpufreq_freq_transition_begin(struct cpufreq_policy *policy,
 385                 struct cpufreq_freqs *freqs)
 386 {
 387
 388         /*
 389          * Catch double invocations of _begin() which lead to self-deadlock.
 390          * ASYNC_NOTIFICATION drivers are left out because the cpufreq core
 391          * doesn't invoke _begin() on their behalf, and hence the chances of
 392          * double invocations are very low. Moreover, there are scenarios
 393          * where these checks can emit false-positive warnings in these
 394          * drivers; so we avoid that by skipping them altogether.
 395          */
 396         WARN_ON(!(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION)
 397                                 && current == policy->transition_task);
 398
 399 wait:
 400         wait_event(policy->transition_wait, !policy->transition_ongoing);
 401
 402         spin_lock(&policy->transition_lock);
 403
 404         if (unlikely(policy->transition_ongoing)) {
 405                 spin_unlock(&policy->transition_lock);
 406                 goto wait;
 407         }
 408
 409         policy->transition_ongoing = true;
 410         policy->transition_task = current;
 411
 412         spin_unlock(&policy->transition_lock);
 413
 414         cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
 415 }
 416 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin);
 417
 418 void cpufreq_freq_transition_end(struct cpufreq_policy *policy,
 419                 struct cpufreq_freqs *freqs, int transition_failed)
 420 {
 421         if (unlikely(WARN_ON(!policy->transition_ongoing)))
 422                 return;
 423
 424         cpufreq_notify_post_transition(policy, freqs, transition_failed);
 425
 426         policy->transition_ongoing = false;
 427         policy->transition_task = NULL;
 428
 429         wake_up(&policy->transition_wait);
 430 }
 431 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end);
 432
 433 /*
 434  * Fast frequency switching status count.  Positive means "enabled", negative
 435  * means "disabled" and 0 means "not decided yet".
 436  */
 437 static int cpufreq_fast_switch_count;
 438 static DEFINE_MUTEX(cpufreq_fast_switch_lock);
 439
 440 static void cpufreq_list_transition_notifiers(void)
 441 {
 442         struct notifier_block *nb;
 443
 444         pr_info("Registered transition notifiers:\n");
 445
 446         mutex_lock(&cpufreq_transition_notifier_list.mutex);
 447
 448         for (nb = cpufreq_transition_notifier_list.head; nb; nb = nb->next)
 449                 pr_info("%pF\n", nb->notifier_call);
 450
 451         mutex_unlock(&cpufreq_transition_notifier_list.mutex);
 452 }
 453
 454 /**
 455  * cpufreq_enable_fast_switch - Enable fast frequency switching for policy.
 456  * @policy: cpufreq policy to enable fast frequency switching for.
 457  *
 458  * Try to enable fast frequency switching for @policy.
 459  *
 460  * The attempt will fail if there is at least one transition notifier registered
 461  * at this point, as fast frequency switching is quite fundamentally at odds
 462  * with transition notifiers.  Thus if successful, it will make registration of
 463  * transition notifiers fail going forward.
 464  */
 465 void cpufreq_enable_fast_switch(struct cpufreq_policy *policy)
 466 {
 467         lockdep_assert_held(&policy->rwsem);
 468
 469         if (!policy->fast_switch_possible)
 470                 return;
 471
 472         mutex_lock(&cpufreq_fast_switch_lock);
 473         if (cpufreq_fast_switch_count >= 0) {
 474                 cpufreq_fast_switch_count++;
 475                 policy->fast_switch_enabled = true;
 476         } else {
 477                 pr_warn("CPU%u: Fast frequency switching not enabled\n",
 478                         policy->cpu);
 479                 cpufreq_list_transition_notifiers();
 480         }
 481         mutex_unlock(&cpufreq_fast_switch_lock);
 482 }
 483 EXPORT_SYMBOL_GPL(cpufreq_enable_fast_switch);
 484
 485 /**
 486  * cpufreq_disable_fast_switch - Disable fast frequency switching for policy.
 487  * @policy: cpufreq policy to disable fast frequency switching for.
 488  */
 489 void cpufreq_disable_fast_switch(struct cpufreq_policy *policy)
 490 {
 491         mutex_lock(&cpufreq_fast_switch_lock);
 492         if (policy->fast_switch_enabled) {
 493                 policy->fast_switch_enabled = false;
 494                 if (!WARN_ON(cpufreq_fast_switch_count <= 0))
 495                         cpufreq_fast_switch_count--;
 496         }
 497         mutex_unlock(&cpufreq_fast_switch_lock);
 498 }
 499 EXPORT_SYMBOL_GPL(cpufreq_disable_fast_switch);
 500
 501 /**
 502  * cpufreq_driver_resolve_freq - Map a target frequency to a driver-supported
 503  * one.
 504  * @target_freq: target frequency to resolve.
 505  *
 506  * The target to driver frequency mapping is cached in the policy.
 507  *
 508  * Return: Lowest driver-supported frequency greater than or equal to the
 509  * given target_freq, subject to policy (min/max) and driver limitations.
 510  */
 511 unsigned int cpufreq_driver_resolve_freq(struct cpufreq_policy *policy,
 512                                          unsigned int target_freq)
 513 {
 514         target_freq = clamp_val(target_freq, policy->min, policy->max);
 515         policy->cached_target_freq = target_freq;
 516
 517         if (cpufreq_driver->target_index) {
 518                 int idx;
 519
 520                 idx = cpufreq_frequency_table_target(policy, target_freq,
 521                                                      CPUFREQ_RELATION_L);
 522                 policy->cached_resolved_idx = idx;
 523                 return policy->freq_table[idx].frequency;
 524         }
 525
 526         if (cpufreq_driver->resolve_freq)
 527                 return cpufreq_driver->resolve_freq(policy, target_freq);
 528
 529         return target_freq;
 530 }
 531 EXPORT_SYMBOL_GPL(cpufreq_driver_resolve_freq);
 532
 533 unsigned int cpufreq_policy_transition_delay_us(struct cpufreq_policy *policy)
 534 {
 535         unsigned int latency;
 536
 537         if (policy->transition_delay_us)
 538                 return policy->transition_delay_us;
 539
 540         latency = policy->cpuinfo.transition_latency / NSEC_PER_USEC;
 541         if (latency) {
 542                 /*
 543                  * For platforms that can change the frequency very fast (< 10
 544                  * us), the above formula gives a decent transition delay. But
 545                  * for platforms where transition_latency is in milliseconds, it
 546                  * ends up giving unrealistic values.
 547                  *
 548                  * Cap the default transition delay to 10 ms, which seems to be
 549                  * a reasonable amount of time after which we should reevaluate
 550                  * the frequency.
 551                  */
 552                 return min(latency * LATENCY_MULTIPLIER, (unsigned int)10000);
 553         }
 554
 555         return LATENCY_MULTIPLIER;
 556 }
 557 EXPORT_SYMBOL_GPL(cpufreq_policy_transition_delay_us);
 558
 559 /*********************************************************************
 560  *                          SYSFS INTERFACE                          *
 561  *********************************************************************/
 562 static ssize_t show_boost(struct kobject *kobj,
 563                                  struct attribute *attr, char *buf)
 564 {
 565         return sprintf(buf, "%d\n", cpufreq_driver->boost_enabled);
 566 }
 567
 568 static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
 569                                   const char *buf, size_t count)
 570 {
 571         int ret, enable;
 572
 573         ret = sscanf(buf, "%d", &enable);
 574         if (ret != 1 || enable < 0 || enable > 1)
 575                 return -EINVAL;
 576
 577         if (cpufreq_boost_trigger_state(enable)) {
 578                 pr_err("%s: Cannot %s BOOST!\n",
 579                        __func__, enable ? "enable" : "disable");
 580                 return -EINVAL;
 581         }
 582
 583         pr_debug("%s: cpufreq BOOST %s\n",
 584                  __func__, enable ? "enabled" : "disabled");
 585
 586         return count;
 587 }
 588 define_one_global_rw(boost);
 589
 590 static struct cpufreq_governor *find_governor(const char *str_governor)
 591 {
 592         struct cpufreq_governor *t;
 593
 594         for_each_governor(t)
 595                 if (!strncasecmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 596                         return t;
 597
 598         return NULL;
 599 }
 600
 601 /**
 602  * cpufreq_parse_governor - parse a governor string
 603  */
 604 static int cpufreq_parse_governor(char *str_governor,
 605                                   struct cpufreq_policy *policy)
 606 {
 607         if (cpufreq_driver->setpolicy) {
 608                 if (!strncasecmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 609                         policy->policy = CPUFREQ_POLICY_PERFORMANCE;
 610                         return 0;
 611                 }
 612
 613                 if (!strncasecmp(str_governor, "powersave", CPUFREQ_NAME_LEN)) {
 614                         policy->policy = CPUFREQ_POLICY_POWERSAVE;
 615                         return 0;
 616                 }
 617         } else {
 618                 struct cpufreq_governor *t;
 619
 620                 mutex_lock(&cpufreq_governor_mutex);
 621
 622                 t = find_governor(str_governor);
 623                 if (!t) {
 624                         int ret;
 625
 626                         mutex_unlock(&cpufreq_governor_mutex);
 627
 628                         ret = request_module("cpufreq_%s", str_governor);
 629                         if (ret)
 630                                 return -EINVAL;
 631
 632                         mutex_lock(&cpufreq_governor_mutex);
 633
 634                         t = find_governor(str_governor);
 635                 }
 636
 637                 mutex_unlock(&cpufreq_governor_mutex);
 638
 639                 if (t) {
 640                         policy->governor = t;
 641                         return 0;
 642                 }
 643         }
 644
 645         return -EINVAL;
 646 }
 647
 648 /**
 649  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 650  * print out cpufreq information
 651  *
 652  * Write out information from cpufreq_driver->policy[cpu]; object must be
 653  * "unsigned int".
 654  */
 655
 656 #define show_one(file_name, object)                     \
 657 static ssize_t show_##file_name                         \
 658 (struct cpufreq_policy *policy, char *buf)              \
 659 {                                                       \
 660         return sprintf(buf, "%u\n", policy->object);    \
 661 }
 662
 663 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 664 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 665 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 666 show_one(scaling_min_freq, min);
 667 show_one(scaling_max_freq, max);
 668
 669 __weak unsigned int arch_freq_get_on_cpu(int cpu)
 670 {
 671         return 0;
 672 }
 673
 674 static ssize_t show_scaling_cur_freq(struct cpufreq_policy *policy, char *buf)
 675 {
 676         ssize_t ret;
 677         unsigned int freq;
 678
 679         freq = arch_freq_get_on_cpu(policy->cpu);
 680         if (freq)
 681                 ret = sprintf(buf, "%u\n", freq);
 682         else if (cpufreq_driver && cpufreq_driver->setpolicy &&
 683                         cpufreq_driver->get)
 684                 ret = sprintf(buf, "%u\n", cpufreq_driver->get(policy->cpu));
 685         else
 686                 ret = sprintf(buf, "%u\n", policy->cur);
 687         return ret;
 688 }
 689
 690 static int cpufreq_set_policy(struct cpufreq_policy *policy,
 691                                 struct cpufreq_policy *new_policy);
 692
 693 /**
 694  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 695  */
 696 #define store_one(file_name, object)                    \
 697 static ssize_t store_##file_name                                        \
 698 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 699 {                                                                       \
 700         int ret, temp;                                                  \
 701         struct cpufreq_policy new_policy;                               \
 702                                                                         \
 703         memcpy(&new_policy, policy, sizeof(*policy));                   \
 704                                                                         \
 705         ret = sscanf(buf, "%u", &new_policy.object);                    \
 706         if (ret != 1)                                                   \
 707                 return -EINVAL;                                         \
 708                                                                         \
 709         temp = new_policy.object;                                       \
 710         ret = cpufreq_set_policy(policy, &new_policy);          \
 711         if (!ret)                                                       \
 712                 policy->user_policy.object = temp;                      \
 713                                                                         \
 714         return ret ? ret : count;                                       \
 715 }
 716
 717 store_one(scaling_min_freq, min);
 718 store_one(scaling_max_freq, max);
 719
 720 /**
 721  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 722  */
 723 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 724                                         char *buf)
 725 {
 726         unsigned int cur_freq = __cpufreq_get(policy);
 727
 728         if (cur_freq)
 729                 return sprintf(buf, "%u\n", cur_freq);
 730
 731         return sprintf(buf, "<unknown>\n");
 732 }
 733
 734 /**
 735  * show_scaling_governor - show the current policy for the specified CPU
 736  */
 737 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 738 {
 739         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 740                 return sprintf(buf, "powersave\n");
 741         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 742                 return sprintf(buf, "performance\n");
 743         else if (policy->governor)
 744                 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
 745                                 policy->governor->name);
 746         return -EINVAL;
 747 }
 748
 749 /**
 750  * store_scaling_governor - store policy for the specified CPU
 751  */
 752 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 753                                         const char *buf, size_t count)
 754 {
 755         int ret;
 756         char    str_governor[16];
 757         struct cpufreq_policy new_policy;
 758
 759         memcpy(&new_policy, policy, sizeof(*policy));
 760
 761         ret = sscanf(buf, "%15s", str_governor);
 762         if (ret != 1)
 763                 return -EINVAL;
 764
 765         if (cpufreq_parse_governor(str_governor, &new_policy))
 766                 return -EINVAL;
 767
 768         ret = cpufreq_set_policy(policy, &new_policy);
 769         return ret ? ret : count;
 770 }
 771
 772 /**
 773  * show_scaling_driver - show the cpufreq driver currently loaded
 774  */
 775 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 776 {
 777         return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
 778 }
 779
 780 /**
 781  * show_scaling_available_governors - show the available CPUfreq governors
 782  */
 783 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 784                                                 char *buf)
 785 {
 786         ssize_t i = 0;
 787         struct cpufreq_governor *t;
 788
 789         if (!has_target()) {
 790                 i += sprintf(buf, "performance powersave");
 791                 goto out;
 792         }
 793
 794         for_each_governor(t) {
 795                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 796                     - (CPUFREQ_NAME_LEN + 2)))
 797                         goto out;
 798                 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
 799         }
 800 out:
 801         i += sprintf(&buf[i], "\n");
 802         return i;
 803 }
 804
 805 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
 806 {
 807         ssize_t i = 0;
 808         unsigned int cpu;
 809
 810         for_each_cpu(cpu, mask) {
 811                 if (i)
 812                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 813                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 814                 if (i >= (PAGE_SIZE - 5))
 815                         break;
 816         }
 817         i += sprintf(&buf[i], "\n");
 818         return i;
 819 }
 820 EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
 821
 822 /**
 823  * show_related_cpus - show the CPUs affected by each transition even if
 824  * hw coordination is in use
 825  */
 826 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 827 {
 828         return cpufreq_show_cpus(policy->related_cpus, buf);
 829 }
 830
 831 /**
 832  * show_affected_cpus - show the CPUs affected by each transition
 833  */
 834 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 835 {
 836         return cpufreq_show_cpus(policy->cpus, buf);
 837 }
 838
 839 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 840                                         const char *buf, size_t count)
 841 {
 842         unsigned int freq = 0;
 843         unsigned int ret;
 844
 845         if (!policy->governor || !policy->governor->store_setspeed)
 846                 return -EINVAL;
 847
 848         ret = sscanf(buf, "%u", &freq);
 849         if (ret != 1)
 850                 return -EINVAL;
 851
 852         policy->governor->store_setspeed(policy, freq);
 853
 854         return count;
 855 }
 856
 857 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 858 {
 859         if (!policy->governor || !policy->governor->show_setspeed)
 860                 return sprintf(buf, "<unsupported>\n");
 861
 862         return policy->governor->show_setspeed(policy, buf);
 863 }
 864
 865 /**
 866  * show_bios_limit - show the current cpufreq HW/BIOS limitation
 867  */
 868 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
 869 {
 870         unsigned int limit;
 871         int ret;
 872         if (cpufreq_driver->bios_limit) {
 873                 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
 874                 if (!ret)
 875                         return sprintf(buf, "%u\n", limit);
 876         }
 877         return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
 878 }
 879
 880 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
 881 cpufreq_freq_attr_ro(cpuinfo_min_freq);
 882 cpufreq_freq_attr_ro(cpuinfo_max_freq);
 883 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
 884 cpufreq_freq_attr_ro(scaling_available_governors);
 885 cpufreq_freq_attr_ro(scaling_driver);
 886 cpufreq_freq_attr_ro(scaling_cur_freq);
 887 cpufreq_freq_attr_ro(bios_limit);
 888 cpufreq_freq_attr_ro(related_cpus);
 889 cpufreq_freq_attr_ro(affected_cpus);
 890 cpufreq_freq_attr_rw(scaling_min_freq);
 891 cpufreq_freq_attr_rw(scaling_max_freq);
 892 cpufreq_freq_attr_rw(scaling_governor);
 893 cpufreq_freq_attr_rw(scaling_setspeed);
 894
 895 static struct attribute *default_attrs[] = {
 896         &cpuinfo_min_freq.attr,
 897         &cpuinfo_max_freq.attr,
 898         &cpuinfo_transition_latency.attr,
 899         &scaling_min_freq.attr,
 900         &scaling_max_freq.attr,
 901         &affected_cpus.attr,
 902         &related_cpus.attr,
 903         &scaling_governor.attr,
 904         &scaling_driver.attr,
 905         &scaling_available_governors.attr,
 906         &scaling_setspeed.attr,
 907         NULL
 908 };
 909
 910 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 911 #define to_attr(a) container_of(a, struct freq_attr, attr)
 912
 913 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 914 {
 915         struct cpufreq_policy *policy = to_policy(kobj);
 916         struct freq_attr *fattr = to_attr(attr);
 917         ssize_t ret;
 918
 919         down_read(&policy->rwsem);
 920         ret = fattr->show(policy, buf);
 921         up_read(&policy->rwsem);
 922
 923         return ret;
 924 }
 925
 926 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 927                      const char *buf, size_t count)
 928 {
 929         struct cpufreq_policy *policy = to_policy(kobj);
 930         struct freq_attr *fattr = to_attr(attr);
 931         ssize_t ret = -EINVAL;
 932
 933         cpus_read_lock();
 934
 935         if (cpu_online(policy->cpu)) {
 936                 down_write(&policy->rwsem);
 937                 ret = fattr->store(policy, buf, count);
 938                 up_write(&policy->rwsem);
 939         }
 940
 941         cpus_read_unlock();
 942
 943         return ret;
 944 }
 945
 946 static void cpufreq_sysfs_release(struct kobject *kobj)
 947 {
 948         struct cpufreq_policy *policy = to_policy(kobj);
 949         pr_debug("last reference is dropped\n");
 950         complete(&policy->kobj_unregister);
 951 }
 952
 953 static const struct sysfs_ops sysfs_ops = {
 954         .show   = show,
 955         .store  = store,
 956 };
 957
 958 static struct kobj_type ktype_cpufreq = {
 959         .sysfs_ops      = &sysfs_ops,
 960         .default_attrs  = default_attrs,
 961         .release        = cpufreq_sysfs_release,
 962 };
 963
 964 static void add_cpu_dev_symlink(struct cpufreq_policy *policy, unsigned int cpu)
 965 {
 966         struct device *dev = get_cpu_device(cpu);
 967
 968         if (!dev)
 969                 return;
 970
 971         if (cpumask_test_and_set_cpu(cpu, policy->real_cpus))
 972                 return;
 973
 974         dev_dbg(dev, "%s: Adding symlink\n", __func__);
 975         if (sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq"))
 976                 dev_err(dev, "cpufreq symlink creation failed\n");
 977 }
 978
 979 static void remove_cpu_dev_symlink(struct cpufreq_policy *policy,
 980                                    struct device *dev)
 981 {
 982         dev_dbg(dev, "%s: Removing symlink\n", __func__);
 983         sysfs_remove_link(&dev->kobj, "cpufreq");
 984 }
 985
 986 static int cpufreq_add_dev_interface(struct cpufreq_policy *policy)
 987 {
 988         struct freq_attr **drv_attr;
 989         int ret = 0;
 990
 991         /* set up files for this cpu device */
 992         drv_attr = cpufreq_driver->attr;
 993         while (drv_attr && *drv_attr) {
 994                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 995                 if (ret)
 996                         return ret;
 997                 drv_attr++;
 998         }
 999         if (cpufreq_driver->get) {
1000                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
1001                 if (ret)
1002                         return ret;
1003         }
1004
1005         ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
1006         if (ret)
1007                 return ret;
1008
1009         if (cpufreq_driver->bios_limit) {
1010                 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
1011                 if (ret)
1012                         return ret;
1013         }
1014
1015         return 0;
1016 }
1017
1018 __weak struct cpufreq_governor *cpufreq_default_governor(void)
1019 {
1020         return NULL;
1021 }
1022
1023 static int cpufreq_init_policy(struct cpufreq_policy *policy)
1024 {
1025         struct cpufreq_governor *gov = NULL;
1026         struct cpufreq_policy new_policy;
1027
1028         memcpy(&new_policy, policy, sizeof(*policy));
1029
1030         /* Update governor of new_policy to the governor used before hotplug */
1031         gov = find_governor(policy->last_governor);
1032         if (gov) {
1033                 pr_debug("Restoring governor %s for cpu %d\n",
1034                                 policy->governor->name, policy->cpu);
1035         } else {
1036                 gov = cpufreq_default_governor();
1037                 if (!gov)
1038                         return -ENODATA;
1039         }
1040
1041         new_policy.governor = gov;
1042
1043         /* Use the default policy if there is no last_policy. */
1044         if (cpufreq_driver->setpolicy) {
1045                 if (policy->last_policy)
1046                         new_policy.policy = policy->last_policy;
1047                 else
1048                         cpufreq_parse_governor(gov->name, &new_policy);
1049         }
1050         /* set default policy */
1051         return cpufreq_set_policy(policy, &new_policy);
1052 }
1053
1054 static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
1055 {
1056         int ret = 0;
1057
1058         /* Has this CPU been taken care of already? */
1059         if (cpumask_test_cpu(cpu, policy->cpus))
1060                 return 0;
1061
1062         down_write(&policy->rwsem);
1063         if (has_target())
1064                 cpufreq_stop_governor(policy);
1065
1066         cpumask_set_cpu(cpu, policy->cpus);
1067
1068         if (has_target()) {
1069                 ret = cpufreq_start_governor(policy);
1070                 if (ret)
1071                         pr_err("%s: Failed to start governor\n", __func__);
1072         }
1073         up_write(&policy->rwsem);
1074         return ret;
1075 }
1076
1077 static void handle_update(struct work_struct *work)
1078 {
1079         struct cpufreq_policy *policy =
1080                 container_of(work, struct cpufreq_policy, update);
1081         unsigned int cpu = policy->cpu;
1082         pr_debug("handle_update for cpu %u called\n", cpu);
1083         cpufreq_update_policy(cpu);
1084 }
1085
1086 static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu)
1087 {
1088         struct cpufreq_policy *policy;
1089         int ret;
1090
1091         policy = kzalloc(sizeof(*policy), GFP_KERNEL);
1092         if (!policy)
1093                 return NULL;
1094
1095         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
1096                 goto err_free_policy;
1097
1098         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
1099                 goto err_free_cpumask;
1100
1101         if (!zalloc_cpumask_var(&policy->real_cpus, GFP_KERNEL))
1102                 goto err_free_rcpumask;
1103
1104         ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
1105                                    cpufreq_global_kobject, "policy%u", cpu);
1106         if (ret) {
1107                 pr_err("%s: failed to init policy->kobj: %d\n", __func__, ret);
1108                 goto err_free_real_cpus;
1109         }
1110
1111         INIT_LIST_HEAD(&policy->policy_list);
1112         init_rwsem(&policy->rwsem);
1113         spin_lock_init(&policy->transition_lock);
1114         init_waitqueue_head(&policy->transition_wait);
1115         init_completion(&policy->kobj_unregister);
1116         INIT_WORK(&policy->update, handle_update);
1117
1118         policy->cpu = cpu;
1119         return policy;
1120
1121 err_free_real_cpus:
1122         free_cpumask_var(policy->real_cpus);
1123 err_free_rcpumask:
1124         free_cpumask_var(policy->related_cpus);
1125 err_free_cpumask:
1126         free_cpumask_var(policy->cpus);
1127 err_free_policy:
1128         kfree(policy);
1129
1130         return NULL;
1131 }
1132
1133 static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy)
1134 {
1135         struct kobject *kobj;
1136         struct completion *cmp;
1137
1138         down_write(&policy->rwsem);
1139         cpufreq_stats_free_table(policy);
1140         kobj = &policy->kobj;
1141         cmp = &policy->kobj_unregister;
1142         up_write(&policy->rwsem);
1143         kobject_put(kobj);
1144
1145         /*
1146          * We need to make sure that the underlying kobj is
1147          * actually not referenced anymore by anybody before we
1148          * proceed with unloading.
1149          */
1150         pr_debug("waiting for dropping of refcount\n");
1151         wait_for_completion(cmp);
1152         pr_debug("wait complete\n");
1153 }
1154
1155 static void cpufreq_policy_free(struct cpufreq_policy *policy)
1156 {
1157         unsigned long flags;
1158         int cpu;
1159
1160         /* Remove policy from list */
1161         write_lock_irqsave(&cpufreq_driver_lock, flags);
1162         list_del(&policy->policy_list);
1163
1164         for_each_cpu(cpu, policy->related_cpus)
1165                 per_cpu(cpufreq_cpu_data, cpu) = NULL;
1166         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1167
1168         cpufreq_policy_put_kobj(policy);
1169         free_cpumask_var(policy->real_cpus);
1170         free_cpumask_var(policy->related_cpus);
1171         free_cpumask_var(policy->cpus);
1172         kfree(policy);
1173 }
1174
1175 static int cpufreq_online(unsigned int cpu)
1176 {
1177         struct cpufreq_policy *policy;
1178         bool new_policy;
1179         unsigned long flags;
1180         unsigned int j;
1181         int ret;
1182
1183         pr_debug("%s: bringing CPU%u online\n", __func__, cpu);
1184
1185         /* Check if this CPU already has a policy to manage it */
1186         policy = per_cpu(cpufreq_cpu_data, cpu);
1187         if (policy) {
1188                 WARN_ON(!cpumask_test_cpu(cpu, policy->related_cpus));
1189                 if (!policy_is_inactive(policy))
1190                         return cpufreq_add_policy_cpu(policy, cpu);
1191
1192                 /* This is the only online CPU for the policy.  Start over. */
1193                 new_policy = false;
1194                 down_write(&policy->rwsem);
1195                 policy->cpu = cpu;
1196                 policy->governor = NULL;
1197                 up_write(&policy->rwsem);
1198         } else {
1199                 new_policy = true;
1200                 policy = cpufreq_policy_alloc(cpu);
1201                 if (!policy)
1202                         return -ENOMEM;
1203         }
1204
1205         cpumask_copy(policy->cpus, cpumask_of(cpu));
1206
1207         /* call driver. From then on the cpufreq must be able
1208          * to accept all calls to ->verify and ->setpolicy for this CPU
1209          */
1210         ret = cpufreq_driver->init(policy);
1211         if (ret) {
1212                 pr_debug("initialization failed\n");
1213                 goto out_free_policy;
1214         }
1215
1216         down_write(&policy->rwsem);
1217
1218         if (new_policy) {
1219                 /* related_cpus should at least include policy->cpus. */
1220                 cpumask_copy(policy->related_cpus, policy->cpus);
1221         }
1222
1223         /*
1224          * affected cpus must always be the one, which are online. We aren't
1225          * managing offline cpus here.
1226          */
1227         cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1228
1229         if (new_policy) {
1230                 policy->user_policy.min = policy->min;
1231                 policy->user_policy.max = policy->max;
1232
1233                 for_each_cpu(j, policy->related_cpus) {
1234                         per_cpu(cpufreq_cpu_data, j) = policy;
1235                         add_cpu_dev_symlink(policy, j);
1236                 }
1237         } else {
1238                 policy->min = policy->user_policy.min;
1239                 policy->max = policy->user_policy.max;
1240         }
1241
1242         if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
1243                 policy->cur = cpufreq_driver->get(policy->cpu);
1244                 if (!policy->cur) {
1245                         pr_err("%s: ->get() failed\n", __func__);
1246                         goto out_exit_policy;
1247                 }
1248         }
1249
1250         /*
1251          * Sometimes boot loaders set CPU frequency to a value outside of
1252          * frequency table present with cpufreq core. In such cases CPU might be
1253          * unstable if it has to run on that frequency for long duration of time
1254          * and so its better to set it to a frequency which is specified in
1255          * freq-table. This also makes cpufreq stats inconsistent as
1256          * cpufreq-stats would fail to register because current frequency of CPU
1257          * isn't found in freq-table.
1258          *
1259          * Because we don't want this change to effect boot process badly, we go
1260          * for the next freq which is >= policy->cur ('cur' must be set by now,
1261          * otherwise we will end up setting freq to lowest of the table as 'cur'
1262          * is initialized to zero).
1263          *
1264          * We are passing target-freq as "policy->cur - 1" otherwise
1265          * __cpufreq_driver_target() would simply fail, as policy->cur will be
1266          * equal to target-freq.
1267          */
1268         if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK)
1269             && has_target()) {
1270                 /* Are we running at unknown frequency ? */
1271                 ret = cpufreq_frequency_table_get_index(policy, policy->cur);
1272                 if (ret == -EINVAL) {
1273                         /* Warn user and fix it */
1274                         pr_warn("%s: CPU%d: Running at unlisted freq: %u KHz\n",
1275                                 __func__, policy->cpu, policy->cur);
1276                         ret = __cpufreq_driver_target(policy, policy->cur - 1,
1277                                 CPUFREQ_RELATION_L);
1278
1279                         /*
1280                          * Reaching here after boot in a few seconds may not
1281                          * mean that system will remain stable at "unknown"
1282                          * frequency for longer duration. Hence, a BUG_ON().
1283                          */
1284                         BUG_ON(ret);
1285                         pr_warn("%s: CPU%d: Unlisted initial frequency changed to: %u KHz\n",
1286                                 __func__, policy->cpu, policy->cur);
1287                 }
1288         }
1289
1290         if (new_policy) {
1291                 ret = cpufreq_add_dev_interface(policy);
1292                 if (ret)
1293                         goto out_exit_policy;
1294
1295                 cpufreq_stats_create_table(policy);
1296
1297                 write_lock_irqsave(&cpufreq_driver_lock, flags);
1298                 list_add(&policy->policy_list, &cpufreq_policy_list);
1299                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1300         }
1301
1302         ret = cpufreq_init_policy(policy);
1303         if (ret) {
1304                 pr_err("%s: Failed to initialize policy for cpu: %d (%d)\n",
1305                        __func__, cpu, ret);
1306                 /* cpufreq_policy_free() will notify based on this */
1307                 new_policy = false;
1308                 goto out_exit_policy;
1309         }
1310
1311         up_write(&policy->rwsem);
1312
1313         kobject_uevent(&policy->kobj, KOBJ_ADD);
1314
1315         /* Callback for handling stuff after policy is ready */
1316         if (cpufreq_driver->ready)
1317                 cpufreq_driver->ready(policy);
1318
1319         pr_debug("initialization complete\n");
1320
1321         return 0;
1322
1323 out_exit_policy:
1324         up_write(&policy->rwsem);
1325
1326         if (cpufreq_driver->exit)
1327                 cpufreq_driver->exit(policy);
1328
1329         for_each_cpu(j, policy->real_cpus)
1330                 remove_cpu_dev_symlink(policy, get_cpu_device(j));
1331
1332 out_free_policy:
1333         cpufreq_policy_free(policy);
1334         return ret;
1335 }
1336
1337 /**
1338  * cpufreq_add_dev - the cpufreq interface for a CPU device.
1339  * @dev: CPU device.
1340  * @sif: Subsystem interface structure pointer (not used)
1341  */
1342 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1343 {
1344         struct cpufreq_policy *policy;
1345         unsigned cpu = dev->id;
1346         int ret;
1347
1348         dev_dbg(dev, "%s: adding CPU%u\n", __func__, cpu);
1349
1350         if (cpu_online(cpu)) {
1351                 ret = cpufreq_online(cpu);
1352                 if (ret)
1353                         return ret;
1354         }
1355
1356         /* Create sysfs link on CPU registration */
1357         policy = per_cpu(cpufreq_cpu_data, cpu);
1358         if (policy)
1359                 add_cpu_dev_symlink(policy, cpu);
1360
1361         return 0;
1362 }
1363
1364 static int cpufreq_offline(unsigned int cpu)
1365 {
1366         struct cpufreq_policy *policy;
1367         int ret;
1368
1369         pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1370
1371         policy = cpufreq_cpu_get_raw(cpu);
1372         if (!policy) {
1373                 pr_debug("%s: No cpu_data found\n", __func__);
1374                 return 0;
1375         }
1376
1377         down_write(&policy->rwsem);
1378         if (has_target())
1379                 cpufreq_stop_governor(policy);
1380
1381         cpumask_clear_cpu(cpu, policy->cpus);
1382
1383         if (policy_is_inactive(policy)) {
1384                 if (has_target())
1385                         strncpy(policy->last_governor, policy->governor->name,
1386                                 CPUFREQ_NAME_LEN);
1387                 else
1388                         policy->last_policy = policy->policy;
1389         } else if (cpu == policy->cpu) {
1390                 /* Nominate new CPU */
1391                 policy->cpu = cpumask_any(policy->cpus);
1392         }
1393
1394         /* Start governor again for active policy */
1395         if (!policy_is_inactive(policy)) {
1396                 if (has_target()) {
1397                         ret = cpufreq_start_governor(policy);
1398                         if (ret)
1399                                 pr_err("%s: Failed to start governor\n", __func__);
1400                 }
1401
1402                 goto unlock;
1403         }
1404
1405         if (cpufreq_driver->stop_cpu)
1406                 cpufreq_driver->stop_cpu(policy);
1407
1408         if (has_target())
1409                 cpufreq_exit_governor(policy);
1410
1411         /*
1412          * Perform the ->exit() even during light-weight tear-down,
1413          * since this is a core component, and is essential for the
1414          * subsequent light-weight ->init() to succeed.
1415          */
1416         if (cpufreq_driver->exit) {
1417                 cpufreq_driver->exit(policy);
1418                 policy->freq_table = NULL;
1419         }
1420
1421 unlock:
1422         up_write(&policy->rwsem);
1423         return 0;
1424 }
1425
1426 /**
1427  * cpufreq_remove_dev - remove a CPU device
1428  *
1429  * Removes the cpufreq interface for a CPU device.
1430  */
1431 static void cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1432 {
1433         unsigned int cpu = dev->id;
1434         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1435
1436         if (!policy)
1437                 return;
1438
1439         if (cpu_online(cpu))
1440                 cpufreq_offline(cpu);
1441
1442         cpumask_clear_cpu(cpu, policy->real_cpus);
1443         remove_cpu_dev_symlink(policy, dev);
1444
1445         if (cpumask_empty(policy->real_cpus))
1446                 cpufreq_policy_free(policy);
1447 }
1448
1449 /**
1450  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're
1451  *      in deep trouble.
1452  *      @policy: policy managing CPUs
1453  *      @new_freq: CPU frequency the CPU actually runs at
1454  *
1455  *      We adjust to current frequency first, and need to clean up later.
1456  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1457  */
1458 static void cpufreq_out_of_sync(struct cpufreq_policy *policy,
1459                                 unsigned int new_freq)
1460 {
1461         struct cpufreq_freqs freqs;
1462
1463         pr_debug("Warning: CPU frequency out of sync: cpufreq and timing core thinks of %u, is %u kHz\n",
1464                  policy->cur, new_freq);
1465
1466         freqs.old = policy->cur;
1467         freqs.new = new_freq;
1468
1469         cpufreq_freq_transition_begin(policy, &freqs);
1470         cpufreq_freq_transition_end(policy, &freqs, 0);
1471 }
1472
1473 /**
1474  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1475  * @cpu: CPU number
1476  *
1477  * This is the last known freq, without actually getting it from the driver.
1478  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1479  */
1480 unsigned int cpufreq_quick_get(unsigned int cpu)
1481 {
1482         struct cpufreq_policy *policy;
1483         unsigned int ret_freq = 0;
1484         unsigned long flags;
1485
1486         read_lock_irqsave(&cpufreq_driver_lock, flags);
1487
1488         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get) {
1489                 ret_freq = cpufreq_driver->get(cpu);
1490                 read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1491                 return ret_freq;
1492         }
1493
1494         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1495
1496         policy = cpufreq_cpu_get(cpu);
1497         if (policy) {
1498                 ret_freq = policy->cur;
1499                 cpufreq_cpu_put(policy);
1500         }
1501
1502         return ret_freq;
1503 }
1504 EXPORT_SYMBOL(cpufreq_quick_get);
1505
1506 /**
1507  * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1508  * @cpu: CPU number
1509  *
1510  * Just return the max possible frequency for a given CPU.
1511  */
1512 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1513 {
1514         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1515         unsigned int ret_freq = 0;
1516
1517         if (policy) {
1518                 ret_freq = policy->max;
1519                 cpufreq_cpu_put(policy);
1520         }
1521
1522         return ret_freq;
1523 }
1524 EXPORT_SYMBOL(cpufreq_quick_get_max);
1525
1526 static unsigned int __cpufreq_get(struct cpufreq_policy *policy)
1527 {
1528         unsigned int ret_freq = 0;
1529
1530         if (!cpufreq_driver->get)
1531                 return ret_freq;
1532
1533         ret_freq = cpufreq_driver->get(policy->cpu);
1534
1535         /*
1536          * Updating inactive policies is invalid, so avoid doing that.  Also
1537          * if fast frequency switching is used with the given policy, the check
1538          * against policy->cur is pointless, so skip it in that case too.
1539          */
1540         if (unlikely(policy_is_inactive(policy)) || policy->fast_switch_enabled)
1541                 return ret_freq;
1542
1543         if (ret_freq && policy->cur &&
1544                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1545                 /* verify no discrepancy between actual and
1546                                         saved value exists */
1547                 if (unlikely(ret_freq != policy->cur)) {
1548                         cpufreq_out_of_sync(policy, ret_freq);
1549                         schedule_work(&policy->update);
1550                 }
1551         }
1552
1553         return ret_freq;
1554 }
1555
1556 /**
1557  * cpufreq_get - get the current CPU frequency (in kHz)
1558  * @cpu: CPU number
1559  *
1560  * Get the CPU current (static) CPU frequency
1561  */
1562 unsigned int cpufreq_get(unsigned int cpu)
1563 {
1564         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1565         unsigned int ret_freq = 0;
1566
1567         if (policy) {
1568                 down_read(&policy->rwsem);
1569
1570                 if (!policy_is_inactive(policy))
1571                         ret_freq = __cpufreq_get(policy);
1572
1573                 up_read(&policy->rwsem);
1574
1575                 cpufreq_cpu_put(policy);
1576         }
1577
1578         return ret_freq;
1579 }
1580 EXPORT_SYMBOL(cpufreq_get);
1581
1582 static unsigned int cpufreq_update_current_freq(struct cpufreq_policy *policy)
1583 {
1584         unsigned int new_freq;
1585
1586         new_freq = cpufreq_driver->get(policy->cpu);
1587         if (!new_freq)
1588                 return 0;
1589
1590         if (!policy->cur) {
1591                 pr_debug("cpufreq: Driver did not initialize current freq\n");
1592                 policy->cur = new_freq;
1593         } else if (policy->cur != new_freq && has_target()) {
1594                 cpufreq_out_of_sync(policy, new_freq);
1595         }
1596
1597         return new_freq;
1598 }
1599
1600 static struct subsys_interface cpufreq_interface = {
1601         .name           = "cpufreq",
1602         .subsys         = &cpu_subsys,
1603         .add_dev        = cpufreq_add_dev,
1604         .remove_dev     = cpufreq_remove_dev,
1605 };
1606
1607 /*
1608  * In case platform wants some specific frequency to be configured
1609  * during suspend..
1610  */
1611 int cpufreq_generic_suspend(struct cpufreq_policy *policy)
1612 {
1613         int ret;
1614
1615         if (!policy->suspend_freq) {
1616                 pr_debug("%s: suspend_freq not defined\n", __func__);
1617                 return 0;
1618         }
1619
1620         pr_debug("%s: Setting suspend-freq: %u\n", __func__,
1621                         policy->suspend_freq);
1622
1623         ret = __cpufreq_driver_target(policy, policy->suspend_freq,
1624                         CPUFREQ_RELATION_H);
1625         if (ret)
1626                 pr_err("%s: unable to set suspend-freq: %u. err: %d\n",
1627                                 __func__, policy->suspend_freq, ret);
1628
1629         return ret;
1630 }
1631 EXPORT_SYMBOL(cpufreq_generic_suspend);
1632
1633 /**
1634  * cpufreq_suspend() - Suspend CPUFreq governors
1635  *
1636  * Called during system wide Suspend/Hibernate cycles for suspending governors
1637  * as some platforms can't change frequency after this point in suspend cycle.
1638  * Because some of the devices (like: i2c, regulators, etc) they use for
1639  * changing frequency are suspended quickly after this point.
1640  */
1641 void cpufreq_suspend(void)
1642 {
1643         struct cpufreq_policy *policy;
1644
1645         if (!cpufreq_driver)
1646                 return;
1647
1648         if (!has_target() && !cpufreq_driver->suspend)
1649                 goto suspend;
1650
1651         pr_debug("%s: Suspending Governors\n", __func__);
1652
1653         for_each_active_policy(policy) {
1654                 if (has_target()) {
1655                         down_write(&policy->rwsem);
1656                         cpufreq_stop_governor(policy);
1657                         up_write(&policy->rwsem);
1658                 }
1659
1660                 if (cpufreq_driver->suspend && cpufreq_driver->suspend(policy))
1661                         pr_err("%s: Failed to suspend driver: %p\n", __func__,
1662                                 policy);
1663         }
1664
1665 suspend:
1666         cpufreq_suspended = true;
1667 }
1668
1669 /**
1670  * cpufreq_resume() - Resume CPUFreq governors
1671  *
1672  * Called during system wide Suspend/Hibernate cycle for resuming governors that
1673  * are suspended with cpufreq_suspend().
1674  */
1675 void cpufreq_resume(void)
1676 {
1677         struct cpufreq_policy *policy;
1678         int ret;
1679
1680         if (!cpufreq_driver)
1681                 return;
1682
1683         cpufreq_suspended = false;
1684
1685         if (!has_target() && !cpufreq_driver->resume)
1686                 return;
1687
1688         pr_debug("%s: Resuming Governors\n", __func__);
1689
1690         for_each_active_policy(policy) {
1691                 if (cpufreq_driver->resume && cpufreq_driver->resume(policy)) {
1692                         pr_err("%s: Failed to resume driver: %p\n", __func__,
1693                                 policy);
1694                 } else if (has_target()) {
1695                         down_write(&policy->rwsem);
1696                         ret = cpufreq_start_governor(policy);
1697                         up_write(&policy->rwsem);
1698
1699                         if (ret)
1700                                 pr_err("%s: Failed to start governor for policy: %p\n",
1701                                        __func__, policy);
1702                 }
1703         }
1704 }
1705
1706 /**
1707  *      cpufreq_get_current_driver - return current driver's name
1708  *
1709  *      Return the name string of the currently loaded cpufreq driver
1710  *      or NULL, if none.
1711  */
1712 const char *cpufreq_get_current_driver(void)
1713 {
1714         if (cpufreq_driver)
1715                 return cpufreq_driver->name;
1716
1717         return NULL;
1718 }
1719 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
1720
1721 /**
1722  *      cpufreq_get_driver_data - return current driver data
1723  *
1724  *      Return the private data of the currently loaded cpufreq
1725  *      driver, or NULL if no cpufreq driver is loaded.
1726  */
1727 void *cpufreq_get_driver_data(void)
1728 {
1729         if (cpufreq_driver)
1730                 return cpufreq_driver->driver_data;
1731
1732         return NULL;
1733 }
1734 EXPORT_SYMBOL_GPL(cpufreq_get_driver_data);
1735
1736 /*********************************************************************
1737  *                     NOTIFIER LISTS INTERFACE                      *
1738  *********************************************************************/
1739
1740 /**
1741  *      cpufreq_register_notifier - register a driver with cpufreq
1742  *      @nb: notifier function to register
1743  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1744  *
1745  *      Add a driver to one of two lists: either a list of drivers that
1746  *      are notified about clock rate changes (once before and once after
1747  *      the transition), or a list of drivers that are notified about
1748  *      changes in cpufreq policy.
1749  *
1750  *      This function may sleep, and has the same return conditions as
1751  *      blocking_notifier_chain_register.
1752  */
1753 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1754 {
1755         int ret;
1756
1757         if (cpufreq_disabled())
1758                 return -EINVAL;
1759
1760         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1761
1762         switch (list) {
1763         case CPUFREQ_TRANSITION_NOTIFIER:
1764                 mutex_lock(&cpufreq_fast_switch_lock);
1765
1766                 if (cpufreq_fast_switch_count > 0) {
1767                         mutex_unlock(&cpufreq_fast_switch_lock);
1768                         return -EBUSY;
1769                 }
1770                 ret = srcu_notifier_chain_register(
1771                                 &cpufreq_transition_notifier_list, nb);
1772                 if (!ret)
1773                         cpufreq_fast_switch_count--;
1774
1775                 mutex_unlock(&cpufreq_fast_switch_lock);
1776                 break;
1777         case CPUFREQ_POLICY_NOTIFIER:
1778                 ret = blocking_notifier_chain_register(
1779                                 &cpufreq_policy_notifier_list, nb);
1780                 break;
1781         default:
1782                 ret = -EINVAL;
1783         }
1784
1785         return ret;
1786 }
1787 EXPORT_SYMBOL(cpufreq_register_notifier);
1788
1789 /**
1790  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1791  *      @nb: notifier block to be unregistered
1792  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1793  *
1794  *      Remove a driver from the CPU frequency notifier list.
1795  *
1796  *      This function may sleep, and has the same return conditions as
1797  *      blocking_notifier_chain_unregister.
1798  */
1799 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1800 {
1801         int ret;
1802
1803         if (cpufreq_disabled())
1804                 return -EINVAL;
1805
1806         switch (list) {
1807         case CPUFREQ_TRANSITION_NOTIFIER:
1808                 mutex_lock(&cpufreq_fast_switch_lock);
1809
1810                 ret = srcu_notifier_chain_unregister(
1811                                 &cpufreq_transition_notifier_list, nb);
1812                 if (!ret && !WARN_ON(cpufreq_fast_switch_count >= 0))
1813                         cpufreq_fast_switch_count++;
1814
1815                 mutex_unlock(&cpufreq_fast_switch_lock);
1816                 break;
1817         case CPUFREQ_POLICY_NOTIFIER:
1818                 ret = blocking_notifier_chain_unregister(
1819                                 &cpufreq_policy_notifier_list, nb);
1820                 break;
1821         default:
1822                 ret = -EINVAL;
1823         }
1824
1825         return ret;
1826 }
1827 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1828
1829
1830 /*********************************************************************
1831  *                              GOVERNORS                            *
1832  *********************************************************************/
1833
1834 /**
1835  * cpufreq_driver_fast_switch - Carry out a fast CPU frequency switch.
1836  * @policy: cpufreq policy to switch the frequency for.
1837  * @target_freq: New frequency to set (may be approximate).
1838  *
1839  * Carry out a fast frequency switch without sleeping.
1840  *
1841  * The driver's ->fast_switch() callback invoked by this function must be
1842  * suitable for being called from within RCU-sched read-side critical sections
1843  * and it is expected to select the minimum available frequency greater than or
1844  * equal to @target_freq (CPUFREQ_RELATION_L).
1845  *
1846  * This function must not be called if policy->fast_switch_enabled is unset.
1847  *
1848  * Governors calling this function must guarantee that it will never be invoked
1849  * twice in parallel for the same policy and that it will never be called in
1850  * parallel with either ->target() or ->target_index() for the same policy.
1851  *
1852  * Returns the actual frequency set for the CPU.
1853  *
1854  * If 0 is returned by the driver's ->fast_switch() callback to indicate an
1855  * error condition, the hardware configuration must be preserved.
1856  */
1857 unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy,
1858                                         unsigned int target_freq)
1859 {
1860         target_freq = clamp_val(target_freq, policy->min, policy->max);
1861
1862         return cpufreq_driver->fast_switch(policy, target_freq);
1863 }
1864 EXPORT_SYMBOL_GPL(cpufreq_driver_fast_switch);
1865
1866 /* Must set freqs->new to intermediate frequency */
1867 static int __target_intermediate(struct cpufreq_policy *policy,
1868                                  struct cpufreq_freqs *freqs, int index)
1869 {
1870         int ret;
1871
1872         freqs->new = cpufreq_driver->get_intermediate(policy, index);
1873
1874         /* We don't need to switch to intermediate freq */
1875         if (!freqs->new)
1876                 return 0;
1877
1878         pr_debug("%s: cpu: %d, switching to intermediate freq: oldfreq: %u, intermediate freq: %u\n",
1879                  __func__, policy->cpu, freqs->old, freqs->new);
1880
1881         cpufreq_freq_transition_begin(policy, freqs);
1882         ret = cpufreq_driver->target_intermediate(policy, index);
1883         cpufreq_freq_transition_end(policy, freqs, ret);
1884
1885         if (ret)
1886                 pr_err("%s: Failed to change to intermediate frequency: %d\n",
1887                        __func__, ret);
1888
1889         return ret;
1890 }
1891
1892 static int __target_index(struct cpufreq_policy *policy, int index)
1893 {
1894         struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0};
1895         unsigned int intermediate_freq = 0;
1896         unsigned int newfreq = policy->freq_table[index].frequency;
1897         int retval = -EINVAL;
1898         bool notify;
1899
1900         if (newfreq == policy->cur)
1901                 return 0;
1902
1903         notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION);
1904         if (notify) {
1905                 /* Handle switching to intermediate frequency */
1906                 if (cpufreq_driver->get_intermediate) {
1907                         retval = __target_intermediate(policy, &freqs, index);
1908                         if (retval)
1909                                 return retval;
1910
1911                         intermediate_freq = freqs.new;
1912                         /* Set old freq to intermediate */
1913                         if (intermediate_freq)
1914                                 freqs.old = freqs.new;
1915                 }
1916
1917                 freqs.new = newfreq;
1918                 pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n",
1919                          __func__, policy->cpu, freqs.old, freqs.new);
1920
1921                 cpufreq_freq_transition_begin(policy, &freqs);
1922         }
1923
1924         retval = cpufreq_driver->target_index(policy, index);
1925         if (retval)
1926                 pr_err("%s: Failed to change cpu frequency: %d\n", __func__,
1927                        retval);
1928
1929         if (notify) {
1930                 cpufreq_freq_transition_end(policy, &freqs, retval);
1931
1932                 /*
1933                  * Failed after setting to intermediate freq? Driver should have
1934                  * reverted back to initial frequency and so should we. Check
1935                  * here for intermediate_freq instead of get_intermediate, in
1936                  * case we haven't switched to intermediate freq at all.
1937                  */
1938                 if (unlikely(retval && intermediate_freq)) {
1939                         freqs.old = intermediate_freq;
1940                         freqs.new = policy->restore_freq;
1941                         cpufreq_freq_transition_begin(policy, &freqs);
1942                         cpufreq_freq_transition_end(policy, &freqs, 0);
1943                 }
1944         }
1945
1946         return retval;
1947 }
1948
1949 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1950                             unsigned int target_freq,
1951                             unsigned int relation)
1952 {
1953         unsigned int old_target_freq = target_freq;
1954         int index;
1955
1956         if (cpufreq_disabled())
1957                 return -ENODEV;
1958
1959         /* Make sure that target_freq is within supported range */
1960         target_freq = clamp_val(target_freq, policy->min, policy->max);
1961
1962         pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
1963                  policy->cpu, target_freq, relation, old_target_freq);
1964
1965         /*
1966          * This might look like a redundant call as we are checking it again
1967          * after finding index. But it is left intentionally for cases where
1968          * exactly same freq is called again and so we can save on few function
1969          * calls.
1970          */
1971         if (target_freq == policy->cur)
1972                 return 0;
1973
1974         /* Save last value to restore later on errors */
1975         policy->restore_freq = policy->cur;
1976
1977         if (cpufreq_driver->target)
1978                 return cpufreq_driver->target(policy, target_freq, relation);
1979
1980         if (!cpufreq_driver->target_index)
1981                 return -EINVAL;
1982
1983         index = cpufreq_frequency_table_target(policy, target_freq, relation);
1984
1985         return __target_index(policy, index);
1986 }
1987 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1988
1989 int cpufreq_driver_target(struct cpufreq_policy *policy,
1990                           unsigned int target_freq,
1991                           unsigned int relation)
1992 {
1993         int ret = -EINVAL;
1994
1995         down_write(&policy->rwsem);
1996
1997         ret = __cpufreq_driver_target(policy, target_freq, relation);
1998
1999         up_write(&policy->rwsem);
2000
2001         return ret;
2002 }
2003 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
2004
2005 __weak struct cpufreq_governor *cpufreq_fallback_governor(void)
2006 {
2007         return NULL;
2008 }
2009
2010 static int cpufreq_init_governor(struct cpufreq_policy *policy)
2011 {
2012         int ret;
2013
2014         /* Don't start any governor operations if we are entering suspend */
2015         if (cpufreq_suspended)
2016                 return 0;
2017         /*
2018          * Governor might not be initiated here if ACPI _PPC changed
2019          * notification happened, so check it.
2020          */
2021         if (!policy->governor)
2022                 return -EINVAL;
2023
2024         /* Platform doesn't want dynamic frequency switching ? */
2025         if (policy->governor->dynamic_switching &&
2026             cpufreq_driver->flags & CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING) {
2027                 struct cpufreq_governor *gov = cpufreq_fallback_governor();
2028
2029                 if (gov) {
2030                         pr_warn("Can't use %s governor as dynamic switching is disallowed. Fallback to %s governor\n",
2031                                 policy->governor->name, gov->name);
2032                         policy->governor = gov;
2033                 } else {
2034                         return -EINVAL;
2035                 }
2036         }
2037
2038         if (!try_module_get(policy->governor->owner))
2039                 return -EINVAL;
2040
2041         pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2042
2043         if (policy->governor->init) {
2044                 ret = policy->governor->init(policy);
2045                 if (ret) {
2046                         module_put(policy->governor->owner);
2047                         return ret;
2048                 }
2049         }
2050
2051         return 0;
2052 }
2053
2054 static void cpufreq_exit_governor(struct cpufreq_policy *policy)
2055 {
2056         if (cpufreq_suspended || !policy->governor)
2057                 return;
2058
2059         pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2060
2061         if (policy->governor->exit)
2062                 policy->governor->exit(policy);
2063
2064         module_put(policy->governor->owner);
2065 }
2066
2067 static int cpufreq_start_governor(struct cpufreq_policy *policy)
2068 {
2069         int ret;
2070
2071         if (cpufreq_suspended)
2072                 return 0;
2073
2074         if (!policy->governor)
2075                 return -EINVAL;
2076
2077         pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2078
2079         if (cpufreq_driver->get && !cpufreq_driver->setpolicy)
2080                 cpufreq_update_current_freq(policy);
2081
2082         if (policy->governor->start) {
2083                 ret = policy->governor->start(policy);
2084                 if (ret)
2085                         return ret;
2086         }
2087
2088         if (policy->governor->limits)
2089                 policy->governor->limits(policy);
2090
2091         return 0;
2092 }
2093
2094 static void cpufreq_stop_governor(struct cpufreq_policy *policy)
2095 {
2096         if (cpufreq_suspended || !policy->governor)
2097                 return;
2098
2099         pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2100
2101         if (policy->governor->stop)
2102                 policy->governor->stop(policy);
2103 }
2104
2105 static void cpufreq_governor_limits(struct cpufreq_policy *policy)
2106 {
2107         if (cpufreq_suspended || !policy->governor)
2108                 return;
2109
2110         pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2111
2112         if (policy->governor->limits)
2113                 policy->governor->limits(policy);
2114 }
2115
2116 int cpufreq_register_governor(struct cpufreq_governor *governor)
2117 {
2118         int err;
2119
2120         if (!governor)
2121                 return -EINVAL;
2122
2123         if (cpufreq_disabled())
2124                 return -ENODEV;
2125
2126         mutex_lock(&cpufreq_governor_mutex);
2127
2128         err = -EBUSY;
2129         if (!find_governor(governor->name)) {
2130                 err = 0;
2131                 list_add(&governor->governor_list, &cpufreq_governor_list);
2132         }
2133
2134         mutex_unlock(&cpufreq_governor_mutex);
2135         return err;
2136 }
2137 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
2138
2139 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
2140 {
2141         struct cpufreq_policy *policy;
2142         unsigned long flags;
2143
2144         if (!governor)
2145                 return;
2146
2147         if (cpufreq_disabled())
2148                 return;
2149
2150         /* clear last_governor for all inactive policies */
2151         read_lock_irqsave(&cpufreq_driver_lock, flags);
2152         for_each_inactive_policy(policy) {
2153                 if (!strcmp(policy->last_governor, governor->name)) {
2154                         policy->governor = NULL;
2155                         strcpy(policy->last_governor, "\0");
2156                 }
2157         }
2158         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
2159
2160         mutex_lock(&cpufreq_governor_mutex);
2161         list_del(&governor->governor_list);
2162         mutex_unlock(&cpufreq_governor_mutex);
2163 }
2164 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
2165
2166
2167 /*********************************************************************
2168  *                          POLICY INTERFACE                         *
2169  *********************************************************************/
2170
2171 /**
2172  * cpufreq_get_policy - get the current cpufreq_policy
2173  * @policy: struct cpufreq_policy into which the current cpufreq_policy
2174  *      is written
2175  *
2176  * Reads the current cpufreq policy.
2177  */
2178 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
2179 {
2180         struct cpufreq_policy *cpu_policy;
2181         if (!policy)
2182                 return -EINVAL;
2183
2184         cpu_policy = cpufreq_cpu_get(cpu);
2185         if (!cpu_policy)
2186                 return -EINVAL;
2187
2188         memcpy(policy, cpu_policy, sizeof(*policy));
2189
2190         cpufreq_cpu_put(cpu_policy);
2191         return 0;
2192 }
2193 EXPORT_SYMBOL(cpufreq_get_policy);
2194
2195 /*
2196  * policy : current policy.
2197  * new_policy: policy to be set.
2198  */
2199 static int cpufreq_set_policy(struct cpufreq_policy *policy,
2200                                 struct cpufreq_policy *new_policy)
2201 {
2202         struct cpufreq_governor *old_gov;
2203         int ret;
2204
2205         pr_debug("setting new policy for CPU %u: %u - %u kHz\n",
2206                  new_policy->cpu, new_policy->min, new_policy->max);
2207
2208         memcpy(&new_policy->cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
2209
2210         /*
2211         * This check works well when we store new min/max freq attributes,
2212         * because new_policy is a copy of policy with one field updated.
2213         */
2214         if (new_policy->min > new_policy->max)
2215                 return -EINVAL;
2216
2217         /* verify the cpu speed can be set within this limit */
2218         ret = cpufreq_driver->verify(new_policy);
2219         if (ret)
2220                 return ret;
2221
2222         /* adjust if necessary - all reasons */
2223         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2224                         CPUFREQ_ADJUST, new_policy);
2225
2226         /*
2227          * verify the cpu speed can be set within this limit, which might be
2228          * different to the first one
2229          */
2230         ret = cpufreq_driver->verify(new_policy);
2231         if (ret)
2232                 return ret;
2233
2234         /* notification of the new policy */
2235         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2236                         CPUFREQ_NOTIFY, new_policy);
2237
2238         policy->min = new_policy->min;
2239         policy->max = new_policy->max;
2240
2241         policy->cached_target_freq = UINT_MAX;
2242
2243         pr_debug("new min and max freqs are %u - %u kHz\n",
2244                  policy->min, policy->max);
2245
2246         if (cpufreq_driver->setpolicy) {
2247                 policy->policy = new_policy->policy;
2248                 pr_debug("setting range\n");
2249                 return cpufreq_driver->setpolicy(new_policy);
2250         }
2251
2252         if (new_policy->governor == policy->governor) {
2253                 pr_debug("cpufreq: governor limits update\n");
2254                 cpufreq_governor_limits(policy);
2255                 return 0;
2256         }
2257
2258         pr_debug("governor switch\n");
2259
2260         /* save old, working values */
2261         old_gov = policy->governor;
2262         /* end old governor */
2263         if (old_gov) {
2264                 cpufreq_stop_governor(policy);
2265                 cpufreq_exit_governor(policy);
2266         }
2267
2268         /* start new governor */
2269         policy->governor = new_policy->governor;
2270         ret = cpufreq_init_governor(policy);
2271         if (!ret) {
2272                 ret = cpufreq_start_governor(policy);
2273                 if (!ret) {
2274                         pr_debug("cpufreq: governor change\n");
2275                         return 0;
2276                 }
2277                 cpufreq_exit_governor(policy);
2278         }
2279
2280         /* new governor failed, so re-start old one */
2281         pr_debug("starting governor %s failed\n", policy->governor->name);
2282         if (old_gov) {
2283                 policy->governor = old_gov;
2284                 if (cpufreq_init_governor(policy))
2285                         policy->governor = NULL;
2286                 else
2287                         cpufreq_start_governor(policy);
2288         }
2289
2290         return ret;
2291 }
2292
2293 /**
2294  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
2295  *      @cpu: CPU which shall be re-evaluated
2296  *
2297  *      Useful for policy notifiers which have different necessities
2298  *      at different times.
2299  */
2300 void cpufreq_update_policy(unsigned int cpu)
2301 {
2302         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
2303         struct cpufreq_policy new_policy;
2304
2305         if (!policy)
2306                 return;
2307
2308         down_write(&policy->rwsem);
2309
2310         if (policy_is_inactive(policy))
2311                 goto unlock;
2312
2313         pr_debug("updating policy for CPU %u\n", cpu);
2314         memcpy(&new_policy, policy, sizeof(*policy));
2315         new_policy.min = policy->user_policy.min;
2316         new_policy.max = policy->user_policy.max;
2317
2318         /*
2319          * BIOS might change freq behind our back
2320          * -> ask driver for current freq and notify governors about a change
2321          */
2322         if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
2323                 if (cpufreq_suspended)
2324                         goto unlock;
2325
2326                 new_policy.cur = cpufreq_update_current_freq(policy);
2327                 if (WARN_ON(!new_policy.cur))
2328                         goto unlock;
2329         }
2330
2331         cpufreq_set_policy(policy, &new_policy);
2332
2333 unlock:
2334         up_write(&policy->rwsem);
2335
2336         cpufreq_cpu_put(policy);
2337 }
2338 EXPORT_SYMBOL(cpufreq_update_policy);
2339
2340 /*********************************************************************
2341  *               BOOST                                               *
2342  *********************************************************************/
2343 static int cpufreq_boost_set_sw(int state)
2344 {
2345         struct cpufreq_policy *policy;
2346         int ret = -EINVAL;
2347
2348         for_each_active_policy(policy) {
2349                 if (!policy->freq_table)
2350                         continue;
2351
2352                 ret = cpufreq_frequency_table_cpuinfo(policy,
2353                                                       policy->freq_table);
2354                 if (ret) {
2355                         pr_err("%s: Policy frequency update failed\n",
2356                                __func__);
2357                         break;
2358                 }
2359
2360                 down_write(&policy->rwsem);
2361                 policy->user_policy.max = policy->max;
2362                 cpufreq_governor_limits(policy);
2363                 up_write(&policy->rwsem);
2364         }
2365
2366         return ret;
2367 }
2368
2369 int cpufreq_boost_trigger_state(int state)
2370 {
2371         unsigned long flags;
2372         int ret = 0;
2373
2374         if (cpufreq_driver->boost_enabled == state)
2375                 return 0;
2376
2377         write_lock_irqsave(&cpufreq_driver_lock, flags);
2378         cpufreq_driver->boost_enabled = state;
2379         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2380
2381         ret = cpufreq_driver->set_boost(state);
2382         if (ret) {
2383                 write_lock_irqsave(&cpufreq_driver_lock, flags);
2384                 cpufreq_driver->boost_enabled = !state;
2385                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2386
2387                 pr_err("%s: Cannot %s BOOST\n",
2388                        __func__, state ? "enable" : "disable");
2389         }
2390
2391         return ret;
2392 }
2393
2394 static bool cpufreq_boost_supported(void)
2395 {
2396         return likely(cpufreq_driver) && cpufreq_driver->set_boost;
2397 }
2398
2399 static int create_boost_sysfs_file(void)
2400 {
2401         int ret;
2402
2403         ret = sysfs_create_file(cpufreq_global_kobject, &boost.attr);
2404         if (ret)
2405                 pr_err("%s: cannot register global BOOST sysfs file\n",
2406                        __func__);
2407
2408         return ret;
2409 }
2410
2411 static void remove_boost_sysfs_file(void)
2412 {
2413         if (cpufreq_boost_supported())
2414                 sysfs_remove_file(cpufreq_global_kobject, &boost.attr);
2415 }
2416
2417 int cpufreq_enable_boost_support(void)
2418 {
2419         if (!cpufreq_driver)
2420                 return -EINVAL;
2421
2422         if (cpufreq_boost_supported())
2423                 return 0;
2424
2425         cpufreq_driver->set_boost = cpufreq_boost_set_sw;
2426
2427         /* This will get removed on driver unregister */
2428         return create_boost_sysfs_file();
2429 }
2430 EXPORT_SYMBOL_GPL(cpufreq_enable_boost_support);
2431
2432 int cpufreq_boost_enabled(void)
2433 {
2434         return cpufreq_driver->boost_enabled;
2435 }
2436 EXPORT_SYMBOL_GPL(cpufreq_boost_enabled);
2437
2438 /*********************************************************************
2439  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
2440  *********************************************************************/
2441 static enum cpuhp_state hp_online;
2442
2443 static int cpuhp_cpufreq_online(unsigned int cpu)
2444 {
2445         cpufreq_online(cpu);
2446
2447         return 0;
2448 }
2449
2450 static int cpuhp_cpufreq_offline(unsigned int cpu)
2451 {
2452         cpufreq_offline(cpu);
2453
2454         return 0;
2455 }
2456
2457 /**
2458  * cpufreq_register_driver - register a CPU Frequency driver
2459  * @driver_data: A struct cpufreq_driver containing the values#
2460  * submitted by the CPU Frequency driver.
2461  *
2462  * Registers a CPU Frequency driver to this core code. This code
2463  * returns zero on success, -EEXIST when another driver got here first
2464  * (and isn't unregistered in the meantime).
2465  *
2466  */
2467 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
2468 {
2469         unsigned long flags;
2470         int ret;
2471
2472         if (cpufreq_disabled())
2473                 return -ENODEV;
2474
2475         if (!driver_data || !driver_data->verify || !driver_data->init ||
2476             !(driver_data->setpolicy || driver_data->target_index ||
2477                     driver_data->target) ||
2478              (driver_data->setpolicy && (driver_data->target_index ||
2479                     driver_data->target)) ||
2480              (!!driver_data->get_intermediate != !!driver_data->target_intermediate))
2481                 return -EINVAL;
2482
2483         pr_debug("trying to register driver %s\n", driver_data->name);
2484
2485         /* Protect against concurrent CPU online/offline. */
2486         cpus_read_lock();
2487
2488         write_lock_irqsave(&cpufreq_driver_lock, flags);
2489         if (cpufreq_driver) {
2490                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2491                 ret = -EEXIST;
2492                 goto out;
2493         }
2494         cpufreq_driver = driver_data;
2495         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2496
2497         if (driver_data->setpolicy)
2498                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
2499
2500         if (cpufreq_boost_supported()) {
2501                 ret = create_boost_sysfs_file();
2502                 if (ret)
2503                         goto err_null_driver;
2504         }
2505
2506         ret = subsys_interface_register(&cpufreq_interface);
2507         if (ret)
2508                 goto err_boost_unreg;
2509
2510         if (!(cpufreq_driver->flags & CPUFREQ_STICKY) &&
2511             list_empty(&cpufreq_policy_list)) {
2512                 /* if all ->init() calls failed, unregister */
2513                 ret = -ENODEV;
2514                 pr_debug("%s: No CPU initialized for driver %s\n", __func__,
2515                          driver_data->name);
2516                 goto err_if_unreg;
2517         }
2518
2519         ret = cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ONLINE_DYN,
2520                                                    "cpufreq:online",
2521                                                    cpuhp_cpufreq_online,
2522                                                    cpuhp_cpufreq_offline);
2523         if (ret < 0)
2524                 goto err_if_unreg;
2525         hp_online = ret;
2526         ret = 0;
2527
2528         pr_debug("driver %s up and running\n", driver_data->name);
2529         goto out;
2530
2531 err_if_unreg:
2532         subsys_interface_unregister(&cpufreq_interface);
2533 err_boost_unreg:
2534         remove_boost_sysfs_file();
2535 err_null_driver:
2536         write_lock_irqsave(&cpufreq_driver_lock, flags);
2537         cpufreq_driver = NULL;
2538         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2539 out:
2540         cpus_read_unlock();
2541         return ret;
2542 }
2543 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
2544
2545 /**
2546  * cpufreq_unregister_driver - unregister the current CPUFreq driver
2547  *
2548  * Unregister the current CPUFreq driver. Only call this if you have
2549  * the right to do so, i.e. if you have succeeded in initialising before!
2550  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
2551  * currently not initialised.
2552  */
2553 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
2554 {
2555         unsigned long flags;
2556
2557         if (!cpufreq_driver || (driver != cpufreq_driver))
2558                 return -EINVAL;
2559
2560         pr_debug("unregistering driver %s\n", driver->name);
2561
2562         /* Protect against concurrent cpu hotplug */
2563         cpus_read_lock();
2564         subsys_interface_unregister(&cpufreq_interface);
2565         remove_boost_sysfs_file();
2566         cpuhp_remove_state_nocalls_cpuslocked(hp_online);
2567
2568         write_lock_irqsave(&cpufreq_driver_lock, flags);
2569
2570         cpufreq_driver = NULL;
2571
2572         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2573         cpus_read_unlock();
2574
2575         return 0;
2576 }
2577 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2578
2579 /*
2580  * Stop cpufreq at shutdown to make sure it isn't holding any locks
2581  * or mutexes when secondary CPUs are halted.
2582  */
2583 static struct syscore_ops cpufreq_syscore_ops = {
2584         .shutdown = cpufreq_suspend,
2585 };
2586
2587 struct kobject *cpufreq_global_kobject;
2588 EXPORT_SYMBOL(cpufreq_global_kobject);
2589
2590 static int __init cpufreq_core_init(void)
2591 {
2592         if (cpufreq_disabled())
2593                 return -ENODEV;
2594
2595         cpufreq_global_kobject = kobject_create_and_add("cpufreq", &cpu_subsys.dev_root->kobj);
2596         BUG_ON(!cpufreq_global_kobject);
2597
2598         register_syscore_ops(&cpufreq_syscore_ops);
2599
2600         return 0;
2601 }
2602 module_param(off, int, 0444);
2603 core_initcall(cpufreq_core_init);