drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *
   7  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   8  *      Added handling for CPU hotplug
   9  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  10  *      Fix handling for CPU hotplug -- affected CPUs
  11  *
  12  * This program is free software; you can redistribute it and/or modify
  13  * it under the terms of the GNU General Public License version 2 as
  14  * published by the Free Software Foundation.
  15  *
  16  */
  17
  18 #include <linux/kernel.h>
  19 #include <linux/module.h>
  20 #include <linux/init.h>
  21 #include <linux/notifier.h>
  22 #include <linux/cpufreq.h>
  23 #include <linux/delay.h>
  24 #include <linux/interrupt.h>
  25 #include <linux/spinlock.h>
  26 #include <linux/device.h>
  27 #include <linux/slab.h>
  28 #include <linux/cpu.h>
  29 #include <linux/completion.h>
  30 #include <linux/mutex.h>
  31
  32 #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_CORE, \
  33                                                 "cpufreq-core", msg)
  34
  35 /**
  36  * The "cpufreq driver" - the arch- or hardware-dependent low
  37  * level driver of CPUFreq support, and its spinlock. This lock
  38  * also protects the cpufreq_cpu_data array.
  39  */
  40 static struct cpufreq_driver *cpufreq_driver;
  41 static struct cpufreq_policy *cpufreq_cpu_data[NR_CPUS];
  42 #ifdef CONFIG_HOTPLUG_CPU
  43 /* This one keeps track of the previously set governor of a removed CPU */
  44 static struct cpufreq_governor *cpufreq_cpu_governor[NR_CPUS];
  45 #endif
  46 static DEFINE_SPINLOCK(cpufreq_driver_lock);
  47
  48 /*
  49  * cpu_policy_rwsem is a per CPU reader-writer semaphore designed to cure
  50  * all cpufreq/hotplug/workqueue/etc related lock issues.
  51  *
  52  * The rules for this semaphore:
  53  * - Any routine that wants to read from the policy structure will
  54  *   do a down_read on this semaphore.
  55  * - Any routine that will write to the policy structure and/or may take away
  56  *   the policy altogether (eg. CPU hotplug), will hold this lock in write
  57  *   mode before doing so.
  58  *
  59  * Additional rules:
  60  * - All holders of the lock should check to make sure that the CPU they
  61  *   are concerned with are online after they get the lock.
  62  * - Governor routines that can be called in cpufreq hotplug path should not
  63  *   take this sem as top level hotplug notifier handler takes this.
  64  */
  65 static DEFINE_PER_CPU(int, policy_cpu);
  66 static DEFINE_PER_CPU(struct rw_semaphore, cpu_policy_rwsem);
  67
  68 #define lock_policy_rwsem(mode, cpu)                                    \
  69 int lock_policy_rwsem_##mode                                            \
  70 (int cpu)                                                               \
  71 {                                                                       \
  72         int policy_cpu = per_cpu(policy_cpu, cpu);                      \
  73         BUG_ON(policy_cpu == -1);                                       \
  74         down_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));            \
  75         if (unlikely(!cpu_online(cpu))) {                               \
  76                 up_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));      \
  77                 return -1;                                              \
  78         }                                                               \
  79                                                                         \
  80         return 0;                                                       \
  81 }
  82
  83 lock_policy_rwsem(read, cpu);
  84 EXPORT_SYMBOL_GPL(lock_policy_rwsem_read);
  85
  86 lock_policy_rwsem(write, cpu);
  87 EXPORT_SYMBOL_GPL(lock_policy_rwsem_write);
  88
  89 void unlock_policy_rwsem_read(int cpu)
  90 {
  91         int policy_cpu = per_cpu(policy_cpu, cpu);
  92         BUG_ON(policy_cpu == -1);
  93         up_read(&per_cpu(cpu_policy_rwsem, policy_cpu));
  94 }
  95 EXPORT_SYMBOL_GPL(unlock_policy_rwsem_read);
  96
  97 void unlock_policy_rwsem_write(int cpu)
  98 {
  99         int policy_cpu = per_cpu(policy_cpu, cpu);
 100         BUG_ON(policy_cpu == -1);
 101         up_write(&per_cpu(cpu_policy_rwsem, policy_cpu));
 102 }
 103 EXPORT_SYMBOL_GPL(unlock_policy_rwsem_write);
 104
 105
 106 /* internal prototypes */
 107 static int __cpufreq_governor(struct cpufreq_policy *policy, unsigned int event);
 108 static unsigned int __cpufreq_get(unsigned int cpu);
 109 static void handle_update(struct work_struct *work);
 110
 111 /**
 112  * Two notifier lists: the "policy" list is involved in the
 113  * validation process for a new CPU frequency policy; the
 114  * "transition" list for kernel code that needs to handle
 115  * changes to devices when the CPU clock speed changes.
 116  * The mutex locks both lists.
 117  */
 118 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
 119 static struct srcu_notifier_head cpufreq_transition_notifier_list;
 120
 121 static int __init init_cpufreq_transition_notifier_list(void)
 122 {
 123         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
 124         return 0;
 125 }
 126 pure_initcall(init_cpufreq_transition_notifier_list);
 127
 128 static LIST_HEAD(cpufreq_governor_list);
 129 static DEFINE_MUTEX (cpufreq_governor_mutex);
 130
 131 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 132 {
 133         struct cpufreq_policy *data;
 134         unsigned long flags;
 135
 136         if (cpu >= NR_CPUS)
 137                 goto err_out;
 138
 139         /* get the cpufreq driver */
 140         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 141
 142         if (!cpufreq_driver)
 143                 goto err_out_unlock;
 144
 145         if (!try_module_get(cpufreq_driver->owner))
 146                 goto err_out_unlock;
 147
 148
 149         /* get the CPU */
 150         data = cpufreq_cpu_data[cpu];
 151
 152         if (!data)
 153                 goto err_out_put_module;
 154
 155         if (!kobject_get(&data->kobj))
 156                 goto err_out_put_module;
 157
 158         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 159         return data;
 160
 161 err_out_put_module:
 162         module_put(cpufreq_driver->owner);
 163 err_out_unlock:
 164         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 165 err_out:
 166         return NULL;
 167 }
 168 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 169
 170
 171 void cpufreq_cpu_put(struct cpufreq_policy *data)
 172 {
 173         kobject_put(&data->kobj);
 174         module_put(cpufreq_driver->owner);
 175 }
 176 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 177
 178
 179 /*********************************************************************
 180  *                     UNIFIED DEBUG HELPERS                         *
 181  *********************************************************************/
 182 #ifdef CONFIG_CPU_FREQ_DEBUG
 183
 184 /* what part(s) of the CPUfreq subsystem are debugged? */
 185 static unsigned int debug;
 186
 187 /* is the debug output ratelimit'ed using printk_ratelimit? User can
 188  * set or modify this value.
 189  */
 190 static unsigned int debug_ratelimit = 1;
 191
 192 /* is the printk_ratelimit'ing enabled? It's enabled after a successful
 193  * loading of a cpufreq driver, temporarily disabled when a new policy
 194  * is set, and disabled upon cpufreq driver removal
 195  */
 196 static unsigned int disable_ratelimit = 1;
 197 static DEFINE_SPINLOCK(disable_ratelimit_lock);
 198
 199 static void cpufreq_debug_enable_ratelimit(void)
 200 {
 201         unsigned long flags;
 202
 203         spin_lock_irqsave(&disable_ratelimit_lock, flags);
 204         if (disable_ratelimit)
 205                 disable_ratelimit--;
 206         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 207 }
 208
 209 static void cpufreq_debug_disable_ratelimit(void)
 210 {
 211         unsigned long flags;
 212
 213         spin_lock_irqsave(&disable_ratelimit_lock, flags);
 214         disable_ratelimit++;
 215         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 216 }
 217
 218 void cpufreq_debug_printk(unsigned int type, const char *prefix,
 219                                                         const char *fmt, ...)
 220 {
 221         char s[256];
 222         va_list args;
 223         unsigned int len;
 224         unsigned long flags;
 225
 226         WARN_ON(!prefix);
 227         if (type & debug) {
 228                 spin_lock_irqsave(&disable_ratelimit_lock, flags);
 229                 if (!disable_ratelimit && debug_ratelimit
 230                                         && !printk_ratelimit()) {
 231                         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 232                         return;
 233                 }
 234                 spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 235
 236                 len = snprintf(s, 256, KERN_DEBUG "%s: ", prefix);
 237
 238                 va_start(args, fmt);
 239                 len += vsnprintf(&s[len], (256 - len), fmt, args);
 240                 va_end(args);
 241
 242                 printk(s);
 243
 244                 WARN_ON(len < 5);
 245         }
 246 }
 247 EXPORT_SYMBOL(cpufreq_debug_printk);
 248
 249
 250 module_param(debug, uint, 0644);
 251 MODULE_PARM_DESC(debug, "CPUfreq debugging: add 1 to debug core,"
 252                         " 2 to debug drivers, and 4 to debug governors.");
 253
 254 module_param(debug_ratelimit, uint, 0644);
 255 MODULE_PARM_DESC(debug_ratelimit, "CPUfreq debugging:"
 256                                         " set to 0 to disable ratelimiting.");
 257
 258 #else /* !CONFIG_CPU_FREQ_DEBUG */
 259
 260 static inline void cpufreq_debug_enable_ratelimit(void) { return; }
 261 static inline void cpufreq_debug_disable_ratelimit(void) { return; }
 262
 263 #endif /* CONFIG_CPU_FREQ_DEBUG */
 264
 265
 266 /*********************************************************************
 267  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 268  *********************************************************************/
 269
 270 /**
 271  * adjust_jiffies - adjust the system "loops_per_jiffy"
 272  *
 273  * This function alters the system "loops_per_jiffy" for the clock
 274  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 275  * systems as each CPU might be scaled differently. So, use the arch
 276  * per-CPU loops_per_jiffy value wherever possible.
 277  */
 278 #ifndef CONFIG_SMP
 279 static unsigned long l_p_j_ref;
 280 static unsigned int  l_p_j_ref_freq;
 281
 282 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 283 {
 284         if (ci->flags & CPUFREQ_CONST_LOOPS)
 285                 return;
 286
 287         if (!l_p_j_ref_freq) {
 288                 l_p_j_ref = loops_per_jiffy;
 289                 l_p_j_ref_freq = ci->old;
 290                 dprintk("saving %lu as reference value for loops_per_jiffy; "
 291                         "freq is %u kHz\n", l_p_j_ref, l_p_j_ref_freq);
 292         }
 293         if ((val == CPUFREQ_PRECHANGE  && ci->old < ci->new) ||
 294             (val == CPUFREQ_POSTCHANGE && ci->old > ci->new) ||
 295             (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) {
 296                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 297                                                                 ci->new);
 298                 dprintk("scaling loops_per_jiffy to %lu "
 299                         "for frequency %u kHz\n", loops_per_jiffy, ci->new);
 300         }
 301 }
 302 #else
 303 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 304 {
 305         return;
 306 }
 307 #endif
 308
 309
 310 /**
 311  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 312  * on frequency transition.
 313  *
 314  * This function calls the transition notifiers and the "adjust_jiffies"
 315  * function. It is called twice on all CPU frequency changes that have
 316  * external effects.
 317  */
 318 void cpufreq_notify_transition(struct cpufreq_freqs *freqs, unsigned int state)
 319 {
 320         struct cpufreq_policy *policy;
 321
 322         BUG_ON(irqs_disabled());
 323
 324         freqs->flags = cpufreq_driver->flags;
 325         dprintk("notification %u of frequency transition to %u kHz\n",
 326                 state, freqs->new);
 327
 328         policy = cpufreq_cpu_data[freqs->cpu];
 329         switch (state) {
 330
 331         case CPUFREQ_PRECHANGE:
 332                 /* detect if the driver reported a value as "old frequency"
 333                  * which is not equal to what the cpufreq core thinks is
 334                  * "old frequency".
 335                  */
 336                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 337                         if ((policy) && (policy->cpu == freqs->cpu) &&
 338                             (policy->cur) && (policy->cur != freqs->old)) {
 339                                 dprintk("Warning: CPU frequency is"
 340                                         " %u, cpufreq assumed %u kHz.\n",
 341                                         freqs->old, policy->cur);
 342                                 freqs->old = policy->cur;
 343                         }
 344                 }
 345                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 346                                 CPUFREQ_PRECHANGE, freqs);
 347                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 348                 break;
 349
 350         case CPUFREQ_POSTCHANGE:
 351                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 352                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 353                                 CPUFREQ_POSTCHANGE, freqs);
 354                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 355                         policy->cur = freqs->new;
 356                 break;
 357         }
 358 }
 359 EXPORT_SYMBOL_GPL(cpufreq_notify_transition);
 360
 361
 362
 363 /*********************************************************************
 364  *                          SYSFS INTERFACE                          *
 365  *********************************************************************/
 366
 367 static struct cpufreq_governor *__find_governor(const char *str_governor)
 368 {
 369         struct cpufreq_governor *t;
 370
 371         list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 372                 if (!strnicmp(str_governor,t->name,CPUFREQ_NAME_LEN))
 373                         return t;
 374
 375         return NULL;
 376 }
 377
 378 /**
 379  * cpufreq_parse_governor - parse a governor string
 380  */
 381 static int cpufreq_parse_governor (char *str_governor, unsigned int *policy,
 382                                 struct cpufreq_governor **governor)
 383 {
 384         int err = -EINVAL;
 385
 386         if (!cpufreq_driver)
 387                 goto out;
 388
 389         if (cpufreq_driver->setpolicy) {
 390                 if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 391                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 392                         err = 0;
 393                 } else if (!strnicmp(str_governor, "powersave",
 394                                                 CPUFREQ_NAME_LEN)) {
 395                         *policy = CPUFREQ_POLICY_POWERSAVE;
 396                         err = 0;
 397                 }
 398         } else if (cpufreq_driver->target) {
 399                 struct cpufreq_governor *t;
 400
 401                 mutex_lock(&cpufreq_governor_mutex);
 402
 403                 t = __find_governor(str_governor);
 404
 405                 if (t == NULL) {
 406                         char *name = kasprintf(GFP_KERNEL, "cpufreq_%s",
 407                                                                 str_governor);
 408
 409                         if (name) {
 410                                 int ret;
 411
 412                                 mutex_unlock(&cpufreq_governor_mutex);
 413                                 ret = request_module(name);
 414                                 mutex_lock(&cpufreq_governor_mutex);
 415
 416                                 if (ret == 0)
 417                                         t = __find_governor(str_governor);
 418                         }
 419
 420                         kfree(name);
 421                 }
 422
 423                 if (t != NULL) {
 424                         *governor = t;
 425                         err = 0;
 426                 }
 427
 428                 mutex_unlock(&cpufreq_governor_mutex);
 429         }
 430   out:
 431         return err;
 432 }
 433
 434
 435 /* drivers/base/cpu.c */
 436 extern struct sysdev_class cpu_sysdev_class;
 437
 438
 439 /**
 440  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 441  * print out cpufreq information
 442  *
 443  * Write out information from cpufreq_driver->policy[cpu]; object must be
 444  * "unsigned int".
 445  */
 446
 447 #define show_one(file_name, object)                     \
 448 static ssize_t show_##file_name                         \
 449 (struct cpufreq_policy * policy, char *buf)             \
 450 {                                                       \
 451         return sprintf (buf, "%u\n", policy->object);   \
 452 }
 453
 454 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 455 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 456 show_one(scaling_min_freq, min);
 457 show_one(scaling_max_freq, max);
 458 show_one(scaling_cur_freq, cur);
 459
 460 static int __cpufreq_set_policy(struct cpufreq_policy *data,
 461                                 struct cpufreq_policy *policy);
 462
 463 /**
 464  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 465  */
 466 #define store_one(file_name, object)                    \
 467 static ssize_t store_##file_name                                        \
 468 (struct cpufreq_policy * policy, const char *buf, size_t count)         \
 469 {                                                                       \
 470         unsigned int ret = -EINVAL;                                     \
 471         struct cpufreq_policy new_policy;                               \
 472                                                                         \
 473         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 474         if (ret)                                                        \
 475                 return -EINVAL;                                         \
 476                                                                         \
 477         ret = sscanf (buf, "%u", &new_policy.object);                   \
 478         if (ret != 1)                                                   \
 479                 return -EINVAL;                                         \
 480                                                                         \
 481         ret = __cpufreq_set_policy(policy, &new_policy);                \
 482         policy->user_policy.object = policy->object;                    \
 483                                                                         \
 484         return ret ? ret : count;                                       \
 485 }
 486
 487 store_one(scaling_min_freq,min);
 488 store_one(scaling_max_freq,max);
 489
 490 /**
 491  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 492  */
 493 static ssize_t show_cpuinfo_cur_freq (struct cpufreq_policy * policy,
 494                                                         char *buf)
 495 {
 496         unsigned int cur_freq = __cpufreq_get(policy->cpu);
 497         if (!cur_freq)
 498                 return sprintf(buf, "<unknown>");
 499         return sprintf(buf, "%u\n", cur_freq);
 500 }
 501
 502
 503 /**
 504  * show_scaling_governor - show the current policy for the specified CPU
 505  */
 506 static ssize_t show_scaling_governor (struct cpufreq_policy * policy,
 507                                                         char *buf)
 508 {
 509         if(policy->policy == CPUFREQ_POLICY_POWERSAVE)
 510                 return sprintf(buf, "powersave\n");
 511         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 512                 return sprintf(buf, "performance\n");
 513         else if (policy->governor)
 514                 return scnprintf(buf, CPUFREQ_NAME_LEN, "%s\n", policy->governor->name);
 515         return -EINVAL;
 516 }
 517
 518
 519 /**
 520  * store_scaling_governor - store policy for the specified CPU
 521  */
 522 static ssize_t store_scaling_governor (struct cpufreq_policy * policy,
 523                                        const char *buf, size_t count)
 524 {
 525         unsigned int ret = -EINVAL;
 526         char    str_governor[16];
 527         struct cpufreq_policy new_policy;
 528
 529         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 530         if (ret)
 531                 return ret;
 532
 533         ret = sscanf (buf, "%15s", str_governor);
 534         if (ret != 1)
 535                 return -EINVAL;
 536
 537         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 538                                                 &new_policy.governor))
 539                 return -EINVAL;
 540
 541         /* Do not use cpufreq_set_policy here or the user_policy.max
 542            will be wrongly overridden */
 543         ret = __cpufreq_set_policy(policy, &new_policy);
 544
 545         policy->user_policy.policy = policy->policy;
 546         policy->user_policy.governor = policy->governor;
 547
 548         if (ret)
 549                 return ret;
 550         else
 551                 return count;
 552 }
 553
 554 /**
 555  * show_scaling_driver - show the cpufreq driver currently loaded
 556  */
 557 static ssize_t show_scaling_driver (struct cpufreq_policy * policy, char *buf)
 558 {
 559         return scnprintf(buf, CPUFREQ_NAME_LEN, "%s\n", cpufreq_driver->name);
 560 }
 561
 562 /**
 563  * show_scaling_available_governors - show the available CPUfreq governors
 564  */
 565 static ssize_t show_scaling_available_governors (struct cpufreq_policy *policy,
 566                                 char *buf)
 567 {
 568         ssize_t i = 0;
 569         struct cpufreq_governor *t;
 570
 571         if (!cpufreq_driver->target) {
 572                 i += sprintf(buf, "performance powersave");
 573                 goto out;
 574         }
 575
 576         list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 577                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char)) - (CPUFREQ_NAME_LEN + 2)))
 578                         goto out;
 579                 i += scnprintf(&buf[i], CPUFREQ_NAME_LEN, "%s ", t->name);
 580         }
 581 out:
 582         i += sprintf(&buf[i], "\n");
 583         return i;
 584 }
 585 /**
 586  * show_affected_cpus - show the CPUs affected by each transition
 587  */
 588 static ssize_t show_affected_cpus (struct cpufreq_policy * policy, char *buf)
 589 {
 590         ssize_t i = 0;
 591         unsigned int cpu;
 592
 593         for_each_cpu_mask(cpu, policy->cpus) {
 594                 if (i)
 595                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 596                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 597                 if (i >= (PAGE_SIZE - 5))
 598                     break;
 599         }
 600         i += sprintf(&buf[i], "\n");
 601         return i;
 602 }
 603
 604 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 605                 const char *buf, size_t count)
 606 {
 607         unsigned int freq = 0;
 608         unsigned int ret;
 609
 610         if (!policy->governor->store_setspeed)
 611                 return -EINVAL;
 612
 613         ret = sscanf(buf, "%u", &freq);
 614         if (ret != 1)
 615                 return -EINVAL;
 616
 617         policy->governor->store_setspeed(policy, freq);
 618
 619         return count;
 620 }
 621
 622 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 623 {
 624         if (!policy->governor->show_setspeed)
 625                 return sprintf(buf, "<unsupported>\n");
 626
 627         return policy->governor->show_setspeed(policy, buf);
 628 }
 629
 630 #define define_one_ro(_name) \
 631 static struct freq_attr _name = \
 632 __ATTR(_name, 0444, show_##_name, NULL)
 633
 634 #define define_one_ro0400(_name) \
 635 static struct freq_attr _name = \
 636 __ATTR(_name, 0400, show_##_name, NULL)
 637
 638 #define define_one_rw(_name) \
 639 static struct freq_attr _name = \
 640 __ATTR(_name, 0644, show_##_name, store_##_name)
 641
 642 define_one_ro0400(cpuinfo_cur_freq);
 643 define_one_ro(cpuinfo_min_freq);
 644 define_one_ro(cpuinfo_max_freq);
 645 define_one_ro(scaling_available_governors);
 646 define_one_ro(scaling_driver);
 647 define_one_ro(scaling_cur_freq);
 648 define_one_ro(affected_cpus);
 649 define_one_rw(scaling_min_freq);
 650 define_one_rw(scaling_max_freq);
 651 define_one_rw(scaling_governor);
 652 define_one_rw(scaling_setspeed);
 653
 654 static struct attribute * default_attrs[] = {
 655         &cpuinfo_min_freq.attr,
 656         &cpuinfo_max_freq.attr,
 657         &scaling_min_freq.attr,
 658         &scaling_max_freq.attr,
 659         &affected_cpus.attr,
 660         &scaling_governor.attr,
 661         &scaling_driver.attr,
 662         &scaling_available_governors.attr,
 663         &scaling_setspeed.attr,
 664         NULL
 665 };
 666
 667 #define to_policy(k) container_of(k,struct cpufreq_policy,kobj)
 668 #define to_attr(a) container_of(a,struct freq_attr,attr)
 669
 670 static ssize_t show(struct kobject * kobj, struct attribute * attr ,char * buf)
 671 {
 672         struct cpufreq_policy * policy = to_policy(kobj);
 673         struct freq_attr * fattr = to_attr(attr);
 674         ssize_t ret = -EINVAL;
 675         policy = cpufreq_cpu_get(policy->cpu);
 676         if (!policy)
 677                 goto no_policy;
 678
 679         if (lock_policy_rwsem_read(policy->cpu) < 0)
 680                 goto fail;
 681
 682         if (fattr->show)
 683                 ret = fattr->show(policy, buf);
 684         else
 685                 ret = -EIO;
 686
 687         unlock_policy_rwsem_read(policy->cpu);
 688 fail:
 689         cpufreq_cpu_put(policy);
 690 no_policy:
 691         return ret;
 692 }
 693
 694 static ssize_t store(struct kobject * kobj, struct attribute * attr,
 695                      const char * buf, size_t count)
 696 {
 697         struct cpufreq_policy * policy = to_policy(kobj);
 698         struct freq_attr * fattr = to_attr(attr);
 699         ssize_t ret = -EINVAL;
 700         policy = cpufreq_cpu_get(policy->cpu);
 701         if (!policy)
 702                 goto no_policy;
 703
 704         if (lock_policy_rwsem_write(policy->cpu) < 0)
 705                 goto fail;
 706
 707         if (fattr->store)
 708                 ret = fattr->store(policy, buf, count);
 709         else
 710                 ret = -EIO;
 711
 712         unlock_policy_rwsem_write(policy->cpu);
 713 fail:
 714         cpufreq_cpu_put(policy);
 715 no_policy:
 716         return ret;
 717 }
 718
 719 static void cpufreq_sysfs_release(struct kobject * kobj)
 720 {
 721         struct cpufreq_policy * policy = to_policy(kobj);
 722         dprintk("last reference is dropped\n");
 723         complete(&policy->kobj_unregister);
 724 }
 725
 726 static struct sysfs_ops sysfs_ops = {
 727         .show   = show,
 728         .store  = store,
 729 };
 730
 731 static struct kobj_type ktype_cpufreq = {
 732         .sysfs_ops      = &sysfs_ops,
 733         .default_attrs  = default_attrs,
 734         .release        = cpufreq_sysfs_release,
 735 };
 736
 737
 738 /**
 739  * cpufreq_add_dev - add a CPU device
 740  *
 741  * Adds the cpufreq interface for a CPU device.
 742  */
 743 static int cpufreq_add_dev (struct sys_device * sys_dev)
 744 {
 745         unsigned int cpu = sys_dev->id;
 746         int ret = 0;
 747         struct cpufreq_policy new_policy;
 748         struct cpufreq_policy *policy;
 749         struct freq_attr **drv_attr;
 750         struct sys_device *cpu_sys_dev;
 751         unsigned long flags;
 752         unsigned int j;
 753 #ifdef CONFIG_SMP
 754         struct cpufreq_policy *managed_policy;
 755 #endif
 756
 757         if (cpu_is_offline(cpu))
 758                 return 0;
 759
 760         cpufreq_debug_disable_ratelimit();
 761         dprintk("adding CPU %u\n", cpu);
 762
 763 #ifdef CONFIG_SMP
 764         /* check whether a different CPU already registered this
 765          * CPU because it is in the same boat. */
 766         policy = cpufreq_cpu_get(cpu);
 767         if (unlikely(policy)) {
 768                 cpufreq_cpu_put(policy);
 769                 cpufreq_debug_enable_ratelimit();
 770                 return 0;
 771         }
 772 #endif
 773
 774         if (!try_module_get(cpufreq_driver->owner)) {
 775                 ret = -EINVAL;
 776                 goto module_out;
 777         }
 778
 779         policy = kzalloc(sizeof(struct cpufreq_policy), GFP_KERNEL);
 780         if (!policy) {
 781                 ret = -ENOMEM;
 782                 goto nomem_out;
 783         }
 784
 785         policy->cpu = cpu;
 786         policy->cpus = cpumask_of_cpu(cpu);
 787
 788         /* Initially set CPU itself as the policy_cpu */
 789         per_cpu(policy_cpu, cpu) = cpu;
 790         lock_policy_rwsem_write(cpu);
 791
 792         init_completion(&policy->kobj_unregister);
 793         INIT_WORK(&policy->update, handle_update);
 794
 795         /* Set governor before ->init, so that driver could check it */
 796         policy->governor = CPUFREQ_DEFAULT_GOVERNOR;
 797         /* call driver. From then on the cpufreq must be able
 798          * to accept all calls to ->verify and ->setpolicy for this CPU
 799          */
 800         ret = cpufreq_driver->init(policy);
 801         if (ret) {
 802                 dprintk("initialization failed\n");
 803                 unlock_policy_rwsem_write(cpu);
 804                 goto err_out;
 805         }
 806         policy->user_policy.min = policy->cpuinfo.min_freq;
 807         policy->user_policy.max = policy->cpuinfo.max_freq;
 808
 809 #ifdef CONFIG_SMP
 810
 811 #ifdef CONFIG_HOTPLUG_CPU
 812         if (cpufreq_cpu_governor[cpu]){
 813                 policy->governor = cpufreq_cpu_governor[cpu];
 814                 dprintk("Restoring governor %s for cpu %d\n",
 815                        policy->governor->name, cpu);
 816         }
 817 #endif
 818
 819         for_each_cpu_mask(j, policy->cpus) {
 820                 if (cpu == j)
 821                         continue;
 822
 823                 /* check for existing affected CPUs.  They may not be aware
 824                  * of it due to CPU Hotplug.
 825                  */
 826                 managed_policy = cpufreq_cpu_get(j);
 827                 if (unlikely(managed_policy)) {
 828
 829                         /* Set proper policy_cpu */
 830                         unlock_policy_rwsem_write(cpu);
 831                         per_cpu(policy_cpu, cpu) = managed_policy->cpu;
 832
 833                         if (lock_policy_rwsem_write(cpu) < 0)
 834                                 goto err_out_driver_exit;
 835
 836                         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 837                         managed_policy->cpus = policy->cpus;
 838                         cpufreq_cpu_data[cpu] = managed_policy;
 839                         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 840
 841                         dprintk("CPU already managed, adding link\n");
 842                         ret = sysfs_create_link(&sys_dev->kobj,
 843                                                 &managed_policy->kobj,
 844                                                 "cpufreq");
 845                         if (ret) {
 846                                 unlock_policy_rwsem_write(cpu);
 847                                 goto err_out_driver_exit;
 848                         }
 849
 850                         cpufreq_debug_enable_ratelimit();
 851                         ret = 0;
 852                         unlock_policy_rwsem_write(cpu);
 853                         goto err_out_driver_exit; /* call driver->exit() */
 854                 }
 855         }
 856 #endif
 857         memcpy(&new_policy, policy, sizeof(struct cpufreq_policy));
 858
 859         /* prepare interface data */
 860         ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq, &sys_dev->kobj,
 861                                    "cpufreq");
 862         if (ret) {
 863                 unlock_policy_rwsem_write(cpu);
 864                 goto err_out_driver_exit;
 865         }
 866         /* set up files for this cpu device */
 867         drv_attr = cpufreq_driver->attr;
 868         while ((drv_attr) && (*drv_attr)) {
 869                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 870                 if (ret) {
 871                         unlock_policy_rwsem_write(cpu);
 872                         goto err_out_driver_exit;
 873                 }
 874                 drv_attr++;
 875         }
 876         if (cpufreq_driver->get){
 877                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 878                 if (ret) {
 879                         unlock_policy_rwsem_write(cpu);
 880                         goto err_out_driver_exit;
 881                 }
 882         }
 883         if (cpufreq_driver->target){
 884                 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 885                 if (ret) {
 886                         unlock_policy_rwsem_write(cpu);
 887                         goto err_out_driver_exit;
 888                 }
 889         }
 890
 891         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 892         for_each_cpu_mask(j, policy->cpus) {
 893                 cpufreq_cpu_data[j] = policy;
 894                 per_cpu(policy_cpu, j) = policy->cpu;
 895         }
 896         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 897
 898         /* symlink affected CPUs */
 899         for_each_cpu_mask(j, policy->cpus) {
 900                 if (j == cpu)
 901                         continue;
 902                 if (!cpu_online(j))
 903                         continue;
 904
 905                 dprintk("CPU %u already managed, adding link\n", j);
 906                 cpufreq_cpu_get(cpu);
 907                 cpu_sys_dev = get_cpu_sysdev(j);
 908                 ret = sysfs_create_link(&cpu_sys_dev->kobj, &policy->kobj,
 909                                         "cpufreq");
 910                 if (ret) {
 911                         unlock_policy_rwsem_write(cpu);
 912                         goto err_out_unregister;
 913                 }
 914         }
 915
 916         policy->governor = NULL; /* to assure that the starting sequence is
 917                                   * run in cpufreq_set_policy */
 918
 919         /* set default policy */
 920         ret = __cpufreq_set_policy(policy, &new_policy);
 921         policy->user_policy.policy = policy->policy;
 922         policy->user_policy.governor = policy->governor;
 923
 924         unlock_policy_rwsem_write(cpu);
 925
 926         if (ret) {
 927                 dprintk("setting policy failed\n");
 928                 goto err_out_unregister;
 929         }
 930
 931         kobject_uevent(&policy->kobj, KOBJ_ADD);
 932         module_put(cpufreq_driver->owner);
 933         dprintk("initialization complete\n");
 934         cpufreq_debug_enable_ratelimit();
 935
 936         return 0;
 937
 938
 939 err_out_unregister:
 940         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 941         for_each_cpu_mask(j, policy->cpus)
 942                 cpufreq_cpu_data[j] = NULL;
 943         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 944
 945         kobject_put(&policy->kobj);
 946         wait_for_completion(&policy->kobj_unregister);
 947
 948 err_out_driver_exit:
 949         if (cpufreq_driver->exit)
 950                 cpufreq_driver->exit(policy);
 951
 952 err_out:
 953         kfree(policy);
 954
 955 nomem_out:
 956         module_put(cpufreq_driver->owner);
 957 module_out:
 958         cpufreq_debug_enable_ratelimit();
 959         return ret;
 960 }
 961
 962
 963 /**
 964  * __cpufreq_remove_dev - remove a CPU device
 965  *
 966  * Removes the cpufreq interface for a CPU device.
 967  * Caller should already have policy_rwsem in write mode for this CPU.
 968  * This routine frees the rwsem before returning.
 969  */
 970 static int __cpufreq_remove_dev (struct sys_device * sys_dev)
 971 {
 972         unsigned int cpu = sys_dev->id;
 973         unsigned long flags;
 974         struct cpufreq_policy *data;
 975 #ifdef CONFIG_SMP
 976         struct sys_device *cpu_sys_dev;
 977         unsigned int j;
 978 #endif
 979
 980         cpufreq_debug_disable_ratelimit();
 981         dprintk("unregistering CPU %u\n", cpu);
 982
 983         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 984         data = cpufreq_cpu_data[cpu];
 985
 986         if (!data) {
 987                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 988                 cpufreq_debug_enable_ratelimit();
 989                 unlock_policy_rwsem_write(cpu);
 990                 return -EINVAL;
 991         }
 992         cpufreq_cpu_data[cpu] = NULL;
 993
 994
 995 #ifdef CONFIG_SMP
 996         /* if this isn't the CPU which is the parent of the kobj, we
 997          * only need to unlink, put and exit
 998          */
 999         if (unlikely(cpu != data->cpu)) {
1000                 dprintk("removing link\n");
1001                 cpu_clear(cpu, data->cpus);
1002                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1003                 sysfs_remove_link(&sys_dev->kobj, "cpufreq");
1004                 cpufreq_cpu_put(data);
1005                 cpufreq_debug_enable_ratelimit();
1006                 unlock_policy_rwsem_write(cpu);
1007                 return 0;
1008         }
1009 #endif
1010
1011 #ifdef CONFIG_SMP
1012
1013 #ifdef CONFIG_HOTPLUG_CPU
1014         cpufreq_cpu_governor[cpu] = data->governor;
1015 #endif
1016
1017         /* if we have other CPUs still registered, we need to unlink them,
1018          * or else wait_for_completion below will lock up. Clean the
1019          * cpufreq_cpu_data[] while holding the lock, and remove the sysfs
1020          * links afterwards.
1021          */
1022         if (unlikely(cpus_weight(data->cpus) > 1)) {
1023                 for_each_cpu_mask(j, data->cpus) {
1024                         if (j == cpu)
1025                                 continue;
1026                         cpufreq_cpu_data[j] = NULL;
1027                 }
1028         }
1029
1030         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1031
1032         if (unlikely(cpus_weight(data->cpus) > 1)) {
1033                 for_each_cpu_mask(j, data->cpus) {
1034                         if (j == cpu)
1035                                 continue;
1036                         dprintk("removing link for cpu %u\n", j);
1037 #ifdef CONFIG_HOTPLUG_CPU
1038                         cpufreq_cpu_governor[j] = data->governor;
1039 #endif
1040                         cpu_sys_dev = get_cpu_sysdev(j);
1041                         sysfs_remove_link(&cpu_sys_dev->kobj, "cpufreq");
1042                         cpufreq_cpu_put(data);
1043                 }
1044         }
1045 #else
1046         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1047 #endif
1048
1049         if (cpufreq_driver->target)
1050                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1051
1052         unlock_policy_rwsem_write(cpu);
1053
1054         kobject_put(&data->kobj);
1055
1056         /* we need to make sure that the underlying kobj is actually
1057          * not referenced anymore by anybody before we proceed with
1058          * unloading.
1059          */
1060         dprintk("waiting for dropping of refcount\n");
1061         wait_for_completion(&data->kobj_unregister);
1062         dprintk("wait complete\n");
1063
1064         if (cpufreq_driver->exit)
1065                 cpufreq_driver->exit(data);
1066
1067         kfree(data);
1068
1069         cpufreq_debug_enable_ratelimit();
1070         return 0;
1071 }
1072
1073
1074 static int cpufreq_remove_dev (struct sys_device * sys_dev)
1075 {
1076         unsigned int cpu = sys_dev->id;
1077         int retval;
1078
1079         if (cpu_is_offline(cpu))
1080                 return 0;
1081
1082         if (unlikely(lock_policy_rwsem_write(cpu)))
1083                 BUG();
1084
1085         retval = __cpufreq_remove_dev(sys_dev);
1086         return retval;
1087 }
1088
1089
1090 static void handle_update(struct work_struct *work)
1091 {
1092         struct cpufreq_policy *policy =
1093                 container_of(work, struct cpufreq_policy, update);
1094         unsigned int cpu = policy->cpu;
1095         dprintk("handle_update for cpu %u called\n", cpu);
1096         cpufreq_update_policy(cpu);
1097 }
1098
1099 /**
1100  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're in deep trouble.
1101  *      @cpu: cpu number
1102  *      @old_freq: CPU frequency the kernel thinks the CPU runs at
1103  *      @new_freq: CPU frequency the CPU actually runs at
1104  *
1105  *      We adjust to current frequency first, and need to clean up later. So either call
1106  *      to cpufreq_update_policy() or schedule handle_update()).
1107  */
1108 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1109                                 unsigned int new_freq)
1110 {
1111         struct cpufreq_freqs freqs;
1112
1113         dprintk("Warning: CPU frequency out of sync: cpufreq and timing "
1114                "core thinks of %u, is %u kHz.\n", old_freq, new_freq);
1115
1116         freqs.cpu = cpu;
1117         freqs.old = old_freq;
1118         freqs.new = new_freq;
1119         cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE);
1120         cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE);
1121 }
1122
1123
1124 /**
1125  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1126  * @cpu: CPU number
1127  *
1128  * This is the last known freq, without actually getting it from the driver.
1129  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1130  */
1131 unsigned int cpufreq_quick_get(unsigned int cpu)
1132 {
1133         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1134         unsigned int ret_freq = 0;
1135
1136         if (policy) {
1137                 ret_freq = policy->cur;
1138                 cpufreq_cpu_put(policy);
1139         }
1140
1141         return (ret_freq);
1142 }
1143 EXPORT_SYMBOL(cpufreq_quick_get);
1144
1145
1146 static unsigned int __cpufreq_get(unsigned int cpu)
1147 {
1148         struct cpufreq_policy *policy = cpufreq_cpu_data[cpu];
1149         unsigned int ret_freq = 0;
1150
1151         if (!cpufreq_driver->get)
1152                 return (ret_freq);
1153
1154         ret_freq = cpufreq_driver->get(cpu);
1155
1156         if (ret_freq && policy->cur &&
1157                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1158                 /* verify no discrepancy between actual and
1159                                         saved value exists */
1160                 if (unlikely(ret_freq != policy->cur)) {
1161                         cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1162                         schedule_work(&policy->update);
1163                 }
1164         }
1165
1166         return (ret_freq);
1167 }
1168
1169 /**
1170  * cpufreq_get - get the current CPU frequency (in kHz)
1171  * @cpu: CPU number
1172  *
1173  * Get the CPU current (static) CPU frequency
1174  */
1175 unsigned int cpufreq_get(unsigned int cpu)
1176 {
1177         unsigned int ret_freq = 0;
1178         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1179
1180         if (!policy)
1181                 goto out;
1182
1183         if (unlikely(lock_policy_rwsem_read(cpu)))
1184                 goto out_policy;
1185
1186         ret_freq = __cpufreq_get(cpu);
1187
1188         unlock_policy_rwsem_read(cpu);
1189
1190 out_policy:
1191         cpufreq_cpu_put(policy);
1192 out:
1193         return (ret_freq);
1194 }
1195 EXPORT_SYMBOL(cpufreq_get);
1196
1197
1198 /**
1199  *      cpufreq_suspend - let the low level driver prepare for suspend
1200  */
1201
1202 static int cpufreq_suspend(struct sys_device * sysdev, pm_message_t pmsg)
1203 {
1204         int cpu = sysdev->id;
1205         int ret = 0;
1206         unsigned int cur_freq = 0;
1207         struct cpufreq_policy *cpu_policy;
1208
1209         dprintk("suspending cpu %u\n", cpu);
1210
1211         if (!cpu_online(cpu))
1212                 return 0;
1213
1214         /* we may be lax here as interrupts are off. Nonetheless
1215          * we need to grab the correct cpu policy, as to check
1216          * whether we really run on this CPU.
1217          */
1218
1219         cpu_policy = cpufreq_cpu_get(cpu);
1220         if (!cpu_policy)
1221                 return -EINVAL;
1222
1223         /* only handle each CPU group once */
1224         if (unlikely(cpu_policy->cpu != cpu)) {
1225                 cpufreq_cpu_put(cpu_policy);
1226                 return 0;
1227         }
1228
1229         if (cpufreq_driver->suspend) {
1230                 ret = cpufreq_driver->suspend(cpu_policy, pmsg);
1231                 if (ret) {
1232                         printk(KERN_ERR "cpufreq: suspend failed in ->suspend "
1233                                         "step on CPU %u\n", cpu_policy->cpu);
1234                         cpufreq_cpu_put(cpu_policy);
1235                         return ret;
1236                 }
1237         }
1238
1239
1240         if (cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)
1241                 goto out;
1242
1243         if (cpufreq_driver->get)
1244                 cur_freq = cpufreq_driver->get(cpu_policy->cpu);
1245
1246         if (!cur_freq || !cpu_policy->cur) {
1247                 printk(KERN_ERR "cpufreq: suspend failed to assert current "
1248                        "frequency is what timing core thinks it is.\n");
1249                 goto out;
1250         }
1251
1252         if (unlikely(cur_freq != cpu_policy->cur)) {
1253                 struct cpufreq_freqs freqs;
1254
1255                 if (!(cpufreq_driver->flags & CPUFREQ_PM_NO_WARN))
1256                         dprintk("Warning: CPU frequency is %u, "
1257                                "cpufreq assumed %u kHz.\n",
1258                                cur_freq, cpu_policy->cur);
1259
1260                 freqs.cpu = cpu;
1261                 freqs.old = cpu_policy->cur;
1262                 freqs.new = cur_freq;
1263
1264                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
1265                                     CPUFREQ_SUSPENDCHANGE, &freqs);
1266                 adjust_jiffies(CPUFREQ_SUSPENDCHANGE, &freqs);
1267
1268                 cpu_policy->cur = cur_freq;
1269         }
1270
1271 out:
1272         cpufreq_cpu_put(cpu_policy);
1273         return 0;
1274 }
1275
1276 /**
1277  *      cpufreq_resume -  restore proper CPU frequency handling after resume
1278  *
1279  *      1.) resume CPUfreq hardware support (cpufreq_driver->resume())
1280  *      2.) if ->target and !CPUFREQ_CONST_LOOPS: verify we're in sync
1281  *      3.) schedule call cpufreq_update_policy() ASAP as interrupts are
1282  *          restored.
1283  */
1284 static int cpufreq_resume(struct sys_device * sysdev)
1285 {
1286         int cpu = sysdev->id;
1287         int ret = 0;
1288         struct cpufreq_policy *cpu_policy;
1289
1290         dprintk("resuming cpu %u\n", cpu);
1291
1292         if (!cpu_online(cpu))
1293                 return 0;
1294
1295         /* we may be lax here as interrupts are off. Nonetheless
1296          * we need to grab the correct cpu policy, as to check
1297          * whether we really run on this CPU.
1298          */
1299
1300         cpu_policy = cpufreq_cpu_get(cpu);
1301         if (!cpu_policy)
1302                 return -EINVAL;
1303
1304         /* only handle each CPU group once */
1305         if (unlikely(cpu_policy->cpu != cpu)) {
1306                 cpufreq_cpu_put(cpu_policy);
1307                 return 0;
1308         }
1309
1310         if (cpufreq_driver->resume) {
1311                 ret = cpufreq_driver->resume(cpu_policy);
1312                 if (ret) {
1313                         printk(KERN_ERR "cpufreq: resume failed in ->resume "
1314                                         "step on CPU %u\n", cpu_policy->cpu);
1315                         cpufreq_cpu_put(cpu_policy);
1316                         return ret;
1317                 }
1318         }
1319
1320         if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1321                 unsigned int cur_freq = 0;
1322
1323                 if (cpufreq_driver->get)
1324                         cur_freq = cpufreq_driver->get(cpu_policy->cpu);
1325
1326                 if (!cur_freq || !cpu_policy->cur) {
1327                         printk(KERN_ERR "cpufreq: resume failed to assert "
1328                                         "current frequency is what timing core "
1329                                         "thinks it is.\n");
1330                         goto out;
1331                 }
1332
1333                 if (unlikely(cur_freq != cpu_policy->cur)) {
1334                         struct cpufreq_freqs freqs;
1335
1336                         if (!(cpufreq_driver->flags & CPUFREQ_PM_NO_WARN))
1337                                 dprintk("Warning: CPU frequency "
1338                                        "is %u, cpufreq assumed %u kHz.\n",
1339                                        cur_freq, cpu_policy->cur);
1340
1341                         freqs.cpu = cpu;
1342                         freqs.old = cpu_policy->cur;
1343                         freqs.new = cur_freq;
1344
1345                         srcu_notifier_call_chain(
1346                                         &cpufreq_transition_notifier_list,
1347                                         CPUFREQ_RESUMECHANGE, &freqs);
1348                         adjust_jiffies(CPUFREQ_RESUMECHANGE, &freqs);
1349
1350                         cpu_policy->cur = cur_freq;
1351                 }
1352         }
1353
1354 out:
1355         schedule_work(&cpu_policy->update);
1356         cpufreq_cpu_put(cpu_policy);
1357         return ret;
1358 }
1359
1360 static struct sysdev_driver cpufreq_sysdev_driver = {
1361         .add            = cpufreq_add_dev,
1362         .remove         = cpufreq_remove_dev,
1363         .suspend        = cpufreq_suspend,
1364         .resume         = cpufreq_resume,
1365 };
1366
1367
1368 /*********************************************************************
1369  *                     NOTIFIER LISTS INTERFACE                      *
1370  *********************************************************************/
1371
1372 /**
1373  *      cpufreq_register_notifier - register a driver with cpufreq
1374  *      @nb: notifier function to register
1375  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1376  *
1377  *      Add a driver to one of two lists: either a list of drivers that
1378  *      are notified about clock rate changes (once before and once after
1379  *      the transition), or a list of drivers that are notified about
1380  *      changes in cpufreq policy.
1381  *
1382  *      This function may sleep, and has the same return conditions as
1383  *      blocking_notifier_chain_register.
1384  */
1385 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1386 {
1387         int ret;
1388
1389         switch (list) {
1390         case CPUFREQ_TRANSITION_NOTIFIER:
1391                 ret = srcu_notifier_chain_register(
1392                                 &cpufreq_transition_notifier_list, nb);
1393                 break;
1394         case CPUFREQ_POLICY_NOTIFIER:
1395                 ret = blocking_notifier_chain_register(
1396                                 &cpufreq_policy_notifier_list, nb);
1397                 break;
1398         default:
1399                 ret = -EINVAL;
1400         }
1401
1402         return ret;
1403 }
1404 EXPORT_SYMBOL(cpufreq_register_notifier);
1405
1406
1407 /**
1408  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1409  *      @nb: notifier block to be unregistered
1410  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1411  *
1412  *      Remove a driver from the CPU frequency notifier list.
1413  *
1414  *      This function may sleep, and has the same return conditions as
1415  *      blocking_notifier_chain_unregister.
1416  */
1417 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1418 {
1419         int ret;
1420
1421         switch (list) {
1422         case CPUFREQ_TRANSITION_NOTIFIER:
1423                 ret = srcu_notifier_chain_unregister(
1424                                 &cpufreq_transition_notifier_list, nb);
1425                 break;
1426         case CPUFREQ_POLICY_NOTIFIER:
1427                 ret = blocking_notifier_chain_unregister(
1428                                 &cpufreq_policy_notifier_list, nb);
1429                 break;
1430         default:
1431                 ret = -EINVAL;
1432         }
1433
1434         return ret;
1435 }
1436 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1437
1438
1439 /*********************************************************************
1440  *                              GOVERNORS                            *
1441  *********************************************************************/
1442
1443
1444 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1445                             unsigned int target_freq,
1446                             unsigned int relation)
1447 {
1448         int retval = -EINVAL;
1449
1450         dprintk("target for CPU %u: %u kHz, relation %u\n", policy->cpu,
1451                 target_freq, relation);
1452         if (cpu_online(policy->cpu) && cpufreq_driver->target)
1453                 retval = cpufreq_driver->target(policy, target_freq, relation);
1454
1455         return retval;
1456 }
1457 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1458
1459 int cpufreq_driver_target(struct cpufreq_policy *policy,
1460                           unsigned int target_freq,
1461                           unsigned int relation)
1462 {
1463         int ret;
1464
1465         policy = cpufreq_cpu_get(policy->cpu);
1466         if (!policy)
1467                 return -EINVAL;
1468
1469         if (unlikely(lock_policy_rwsem_write(policy->cpu)))
1470                 return -EINVAL;
1471
1472         ret = __cpufreq_driver_target(policy, target_freq, relation);
1473
1474         unlock_policy_rwsem_write(policy->cpu);
1475
1476         cpufreq_cpu_put(policy);
1477         return ret;
1478 }
1479 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1480
1481 int __cpufreq_driver_getavg(struct cpufreq_policy *policy)
1482 {
1483         int ret = 0;
1484
1485         policy = cpufreq_cpu_get(policy->cpu);
1486         if (!policy)
1487                 return -EINVAL;
1488
1489         if (cpu_online(policy->cpu) && cpufreq_driver->getavg)
1490                 ret = cpufreq_driver->getavg(policy->cpu);
1491
1492         cpufreq_cpu_put(policy);
1493         return ret;
1494 }
1495 EXPORT_SYMBOL_GPL(__cpufreq_driver_getavg);
1496
1497 /*
1498  * when "event" is CPUFREQ_GOV_LIMITS
1499  */
1500
1501 static int __cpufreq_governor(struct cpufreq_policy *policy,
1502                                         unsigned int event)
1503 {
1504         int ret;
1505
1506         /* Only must be defined when default governor is known to have latency
1507            restrictions, like e.g. conservative or ondemand.
1508            That this is the case is already ensured in Kconfig
1509         */
1510 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1511         struct cpufreq_governor *gov = &cpufreq_gov_performance;
1512 #else
1513         struct cpufreq_governor *gov = NULL;
1514 #endif
1515
1516         if (policy->governor->max_transition_latency &&
1517             policy->cpuinfo.transition_latency >
1518             policy->governor->max_transition_latency) {
1519                 if (!gov)
1520                         return -EINVAL;
1521                 else {
1522                         printk(KERN_WARNING "%s governor failed, too long"
1523                                " transition latency of HW, fallback"
1524                                " to %s governor\n",
1525                                policy->governor->name,
1526                                gov->name);
1527                         policy->governor = gov;
1528                 }
1529         }
1530
1531         if (!try_module_get(policy->governor->owner))
1532                 return -EINVAL;
1533
1534         dprintk("__cpufreq_governor for CPU %u, event %u\n",
1535                                                 policy->cpu, event);
1536         ret = policy->governor->governor(policy, event);
1537
1538         /* we keep one module reference alive for
1539                         each CPU governed by this CPU */
1540         if ((event != CPUFREQ_GOV_START) || ret)
1541                 module_put(policy->governor->owner);
1542         if ((event == CPUFREQ_GOV_STOP) && !ret)
1543                 module_put(policy->governor->owner);
1544
1545         return ret;
1546 }
1547
1548
1549 int cpufreq_register_governor(struct cpufreq_governor *governor)
1550 {
1551         int err;
1552
1553         if (!governor)
1554                 return -EINVAL;
1555
1556         mutex_lock(&cpufreq_governor_mutex);
1557
1558         err = -EBUSY;
1559         if (__find_governor(governor->name) == NULL) {
1560                 err = 0;
1561                 list_add(&governor->governor_list, &cpufreq_governor_list);
1562         }
1563
1564         mutex_unlock(&cpufreq_governor_mutex);
1565         return err;
1566 }
1567 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
1568
1569
1570 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
1571 {
1572         if (!governor)
1573                 return;
1574
1575         mutex_lock(&cpufreq_governor_mutex);
1576         list_del(&governor->governor_list);
1577         mutex_unlock(&cpufreq_governor_mutex);
1578         return;
1579 }
1580 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
1581
1582
1583
1584 /*********************************************************************
1585  *                          POLICY INTERFACE                         *
1586  *********************************************************************/
1587
1588 /**
1589  * cpufreq_get_policy - get the current cpufreq_policy
1590  * @policy: struct cpufreq_policy into which the current cpufreq_policy is written
1591  *
1592  * Reads the current cpufreq policy.
1593  */
1594 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
1595 {
1596         struct cpufreq_policy *cpu_policy;
1597         if (!policy)
1598                 return -EINVAL;
1599
1600         cpu_policy = cpufreq_cpu_get(cpu);
1601         if (!cpu_policy)
1602                 return -EINVAL;
1603
1604         memcpy(policy, cpu_policy, sizeof(struct cpufreq_policy));
1605
1606         cpufreq_cpu_put(cpu_policy);
1607         return 0;
1608 }
1609 EXPORT_SYMBOL(cpufreq_get_policy);
1610
1611
1612 /*
1613  * data   : current policy.
1614  * policy : policy to be set.
1615  */
1616 static int __cpufreq_set_policy(struct cpufreq_policy *data,
1617                                 struct cpufreq_policy *policy)
1618 {
1619         int ret = 0;
1620
1621         cpufreq_debug_disable_ratelimit();
1622         dprintk("setting new policy for CPU %u: %u - %u kHz\n", policy->cpu,
1623                 policy->min, policy->max);
1624
1625         memcpy(&policy->cpuinfo, &data->cpuinfo,
1626                                 sizeof(struct cpufreq_cpuinfo));
1627
1628         if (policy->min > data->max || policy->max < data->min) {
1629                 ret = -EINVAL;
1630                 goto error_out;
1631         }
1632
1633         /* verify the cpu speed can be set within this limit */
1634         ret = cpufreq_driver->verify(policy);
1635         if (ret)
1636                 goto error_out;
1637
1638         /* adjust if necessary - all reasons */
1639         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1640                         CPUFREQ_ADJUST, policy);
1641
1642         /* adjust if necessary - hardware incompatibility*/
1643         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1644                         CPUFREQ_INCOMPATIBLE, policy);
1645
1646         /* verify the cpu speed can be set within this limit,
1647            which might be different to the first one */
1648         ret = cpufreq_driver->verify(policy);
1649         if (ret)
1650                 goto error_out;
1651
1652         /* notification of the new policy */
1653         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1654                         CPUFREQ_NOTIFY, policy);
1655
1656         data->min = policy->min;
1657         data->max = policy->max;
1658
1659         dprintk("new min and max freqs are %u - %u kHz\n",
1660                                         data->min, data->max);
1661
1662         if (cpufreq_driver->setpolicy) {
1663                 data->policy = policy->policy;
1664                 dprintk("setting range\n");
1665                 ret = cpufreq_driver->setpolicy(policy);
1666         } else {
1667                 if (policy->governor != data->governor) {
1668                         /* save old, working values */
1669                         struct cpufreq_governor *old_gov = data->governor;
1670
1671                         dprintk("governor switch\n");
1672
1673                         /* end old governor */
1674                         if (data->governor)
1675                                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1676
1677                         /* start new governor */
1678                         data->governor = policy->governor;
1679                         if (__cpufreq_governor(data, CPUFREQ_GOV_START)) {
1680                                 /* new governor failed, so re-start old one */
1681                                 dprintk("starting governor %s failed\n",
1682                                                         data->governor->name);
1683                                 if (old_gov) {
1684                                         data->governor = old_gov;
1685                                         __cpufreq_governor(data,
1686                                                            CPUFREQ_GOV_START);
1687                                 }
1688                                 ret = -EINVAL;
1689                                 goto error_out;
1690                         }
1691                         /* might be a policy change, too, so fall through */
1692                 }
1693                 dprintk("governor: change or update limits\n");
1694                 __cpufreq_governor(data, CPUFREQ_GOV_LIMITS);
1695         }
1696
1697 error_out:
1698         cpufreq_debug_enable_ratelimit();
1699         return ret;
1700 }
1701
1702 /**
1703  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
1704  *      @cpu: CPU which shall be re-evaluated
1705  *
1706  *      Usefull for policy notifiers which have different necessities
1707  *      at different times.
1708  */
1709 int cpufreq_update_policy(unsigned int cpu)
1710 {
1711         struct cpufreq_policy *data = cpufreq_cpu_get(cpu);
1712         struct cpufreq_policy policy;
1713         int ret = 0;
1714
1715         if (!data)
1716                 return -ENODEV;
1717
1718         if (unlikely(lock_policy_rwsem_write(cpu)))
1719                 return -EINVAL;
1720
1721         dprintk("updating policy for CPU %u\n", cpu);
1722         memcpy(&policy, data, sizeof(struct cpufreq_policy));
1723         policy.min = data->user_policy.min;
1724         policy.max = data->user_policy.max;
1725         policy.policy = data->user_policy.policy;
1726         policy.governor = data->user_policy.governor;
1727
1728         /* BIOS might change freq behind our back
1729           -> ask driver for current freq and notify governors about a change */
1730         if (cpufreq_driver->get) {
1731                 policy.cur = cpufreq_driver->get(cpu);
1732                 if (!data->cur) {
1733                         dprintk("Driver did not initialize current freq");
1734                         data->cur = policy.cur;
1735                 } else {
1736                         if (data->cur != policy.cur)
1737                                 cpufreq_out_of_sync(cpu, data->cur,
1738                                                                 policy.cur);
1739                 }
1740         }
1741
1742         ret = __cpufreq_set_policy(data, &policy);
1743
1744         unlock_policy_rwsem_write(cpu);
1745
1746         cpufreq_cpu_put(data);
1747         return ret;
1748 }
1749 EXPORT_SYMBOL(cpufreq_update_policy);
1750
1751 static int __cpuinit cpufreq_cpu_callback(struct notifier_block *nfb,
1752                                         unsigned long action, void *hcpu)
1753 {
1754         unsigned int cpu = (unsigned long)hcpu;
1755         struct sys_device *sys_dev;
1756
1757         sys_dev = get_cpu_sysdev(cpu);
1758         if (sys_dev) {
1759                 switch (action) {
1760                 case CPU_ONLINE:
1761                 case CPU_ONLINE_FROZEN:
1762                         cpufreq_add_dev(sys_dev);
1763                         break;
1764                 case CPU_DOWN_PREPARE:
1765                 case CPU_DOWN_PREPARE_FROZEN:
1766                         if (unlikely(lock_policy_rwsem_write(cpu)))
1767                                 BUG();
1768
1769                         __cpufreq_remove_dev(sys_dev);
1770                         break;
1771                 case CPU_DOWN_FAILED:
1772                 case CPU_DOWN_FAILED_FROZEN:
1773                         cpufreq_add_dev(sys_dev);
1774                         break;
1775                 }
1776         }
1777         return NOTIFY_OK;
1778 }
1779
1780 static struct notifier_block __refdata cpufreq_cpu_notifier =
1781 {
1782     .notifier_call = cpufreq_cpu_callback,
1783 };
1784
1785 /*********************************************************************
1786  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
1787  *********************************************************************/
1788
1789 /**
1790  * cpufreq_register_driver - register a CPU Frequency driver
1791  * @driver_data: A struct cpufreq_driver containing the values#
1792  * submitted by the CPU Frequency driver.
1793  *
1794  *   Registers a CPU Frequency driver to this core code. This code
1795  * returns zero on success, -EBUSY when another driver got here first
1796  * (and isn't unregistered in the meantime).
1797  *
1798  */
1799 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
1800 {
1801         unsigned long flags;
1802         int ret;
1803
1804         if (!driver_data || !driver_data->verify || !driver_data->init ||
1805             ((!driver_data->setpolicy) && (!driver_data->target)))
1806                 return -EINVAL;
1807
1808         dprintk("trying to register driver %s\n", driver_data->name);
1809
1810         if (driver_data->setpolicy)
1811                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
1812
1813         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1814         if (cpufreq_driver) {
1815                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1816                 return -EBUSY;
1817         }
1818         cpufreq_driver = driver_data;
1819         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1820
1821         ret = sysdev_driver_register(&cpu_sysdev_class,&cpufreq_sysdev_driver);
1822
1823         if ((!ret) && !(cpufreq_driver->flags & CPUFREQ_STICKY)) {
1824                 int i;
1825                 ret = -ENODEV;
1826
1827                 /* check for at least one working CPU */
1828                 for (i=0; i<NR_CPUS; i++)
1829                         if (cpufreq_cpu_data[i])
1830                                 ret = 0;
1831
1832                 /* if all ->init() calls failed, unregister */
1833                 if (ret) {
1834                         dprintk("no CPU initialized for driver %s\n",
1835                                                         driver_data->name);
1836                         sysdev_driver_unregister(&cpu_sysdev_class,
1837                                                 &cpufreq_sysdev_driver);
1838
1839                         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1840                         cpufreq_driver = NULL;
1841                         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1842                 }
1843         }
1844
1845         if (!ret) {
1846                 register_hotcpu_notifier(&cpufreq_cpu_notifier);
1847                 dprintk("driver %s up and running\n", driver_data->name);
1848                 cpufreq_debug_enable_ratelimit();
1849         }
1850
1851         return (ret);
1852 }
1853 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
1854
1855
1856 /**
1857  * cpufreq_unregister_driver - unregister the current CPUFreq driver
1858  *
1859  *    Unregister the current CPUFreq driver. Only call this if you have
1860  * the right to do so, i.e. if you have succeeded in initialising before!
1861  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
1862  * currently not initialised.
1863  */
1864 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
1865 {
1866         unsigned long flags;
1867
1868         cpufreq_debug_disable_ratelimit();
1869
1870         if (!cpufreq_driver || (driver != cpufreq_driver)) {
1871                 cpufreq_debug_enable_ratelimit();
1872                 return -EINVAL;
1873         }
1874
1875         dprintk("unregistering driver %s\n", driver->name);
1876
1877         sysdev_driver_unregister(&cpu_sysdev_class, &cpufreq_sysdev_driver);
1878         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
1879
1880         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1881         cpufreq_driver = NULL;
1882         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1883
1884         return 0;
1885 }
1886 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
1887
1888 static int __init cpufreq_core_init(void)
1889 {
1890         int cpu;
1891
1892         for_each_possible_cpu(cpu) {
1893                 per_cpu(policy_cpu, cpu) = -1;
1894                 init_rwsem(&per_cpu(cpu_policy_rwsem, cpu));
1895         }
1896         return 0;
1897 }
1898
1899 core_initcall(cpufreq_core_init);