drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *
   7  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   8  *      Added handling for CPU hotplug
   9  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  10  *      Fix handling for CPU hotplug -- affected CPUs
  11  *
  12  * This program is free software; you can redistribute it and/or modify
  13  * it under the terms of the GNU General Public License version 2 as
  14  * published by the Free Software Foundation.
  15  *
  16  */
  17
  18 #include <linux/kernel.h>
  19 #include <linux/module.h>
  20 #include <linux/init.h>
  21 #include <linux/notifier.h>
  22 #include <linux/cpufreq.h>
  23 #include <linux/delay.h>
  24 #include <linux/interrupt.h>
  25 #include <linux/spinlock.h>
  26 #include <linux/device.h>
  27 #include <linux/slab.h>
  28 #include <linux/cpu.h>
  29 #include <linux/completion.h>
  30 #include <linux/mutex.h>
  31
  32 #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_CORE, \
  33                                                 "cpufreq-core", msg)
  34
  35 /**
  36  * The "cpufreq driver" - the arch- or hardware-dependent low
  37  * level driver of CPUFreq support, and its spinlock. This lock
  38  * also protects the cpufreq_cpu_data array.
  39  */
  40 static struct cpufreq_driver *cpufreq_driver;
  41 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
  42 #ifdef CONFIG_HOTPLUG_CPU
  43 /* This one keeps track of the previously set governor of a removed CPU */
  44 static DEFINE_PER_CPU(struct cpufreq_governor *, cpufreq_cpu_governor);
  45 #endif
  46 static DEFINE_SPINLOCK(cpufreq_driver_lock);
  47
  48 /*
  49  * cpu_policy_rwsem is a per CPU reader-writer semaphore designed to cure
  50  * all cpufreq/hotplug/workqueue/etc related lock issues.
  51  *
  52  * The rules for this semaphore:
  53  * - Any routine that wants to read from the policy structure will
  54  *   do a down_read on this semaphore.
  55  * - Any routine that will write to the policy structure and/or may take away
  56  *   the policy altogether (eg. CPU hotplug), will hold this lock in write
  57  *   mode before doing so.
  58  *
  59  * Additional rules:
  60  * - All holders of the lock should check to make sure that the CPU they
  61  *   are concerned with are online after they get the lock.
  62  * - Governor routines that can be called in cpufreq hotplug path should not
  63  *   take this sem as top level hotplug notifier handler takes this.
  64  */
  65 static DEFINE_PER_CPU(int, policy_cpu);
  66 static DEFINE_PER_CPU(struct rw_semaphore, cpu_policy_rwsem);
  67
  68 #define lock_policy_rwsem(mode, cpu)                                    \
  69 int lock_policy_rwsem_##mode                                            \
  70 (int cpu)                                                               \
  71 {                                                                       \
  72         int policy_cpu = per_cpu(policy_cpu, cpu);                      \
  73         BUG_ON(policy_cpu == -1);                                       \
  74         down_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));            \
  75         if (unlikely(!cpu_online(cpu))) {                               \
  76                 up_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));      \
  77                 return -1;                                              \
  78         }                                                               \
  79                                                                         \
  80         return 0;                                                       \
  81 }
  82
  83 lock_policy_rwsem(read, cpu);
  84 EXPORT_SYMBOL_GPL(lock_policy_rwsem_read);
  85
  86 lock_policy_rwsem(write, cpu);
  87 EXPORT_SYMBOL_GPL(lock_policy_rwsem_write);
  88
  89 void unlock_policy_rwsem_read(int cpu)
  90 {
  91         int policy_cpu = per_cpu(policy_cpu, cpu);
  92         BUG_ON(policy_cpu == -1);
  93         up_read(&per_cpu(cpu_policy_rwsem, policy_cpu));
  94 }
  95 EXPORT_SYMBOL_GPL(unlock_policy_rwsem_read);
  96
  97 void unlock_policy_rwsem_write(int cpu)
  98 {
  99         int policy_cpu = per_cpu(policy_cpu, cpu);
 100         BUG_ON(policy_cpu == -1);
 101         up_write(&per_cpu(cpu_policy_rwsem, policy_cpu));
 102 }
 103 EXPORT_SYMBOL_GPL(unlock_policy_rwsem_write);
 104
 105
 106 /* internal prototypes */
 107 static int __cpufreq_governor(struct cpufreq_policy *policy,
 108                 unsigned int event);
 109 static unsigned int __cpufreq_get(unsigned int cpu);
 110 static void handle_update(struct work_struct *work);
 111
 112 /**
 113  * Two notifier lists: the "policy" list is involved in the
 114  * validation process for a new CPU frequency policy; the
 115  * "transition" list for kernel code that needs to handle
 116  * changes to devices when the CPU clock speed changes.
 117  * The mutex locks both lists.
 118  */
 119 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
 120 static struct srcu_notifier_head cpufreq_transition_notifier_list;
 121
 122 static bool init_cpufreq_transition_notifier_list_called;
 123 static int __init init_cpufreq_transition_notifier_list(void)
 124 {
 125         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
 126         init_cpufreq_transition_notifier_list_called = true;
 127         return 0;
 128 }
 129 pure_initcall(init_cpufreq_transition_notifier_list);
 130
 131 static LIST_HEAD(cpufreq_governor_list);
 132 static DEFINE_MUTEX(cpufreq_governor_mutex);
 133
 134 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 135 {
 136         struct cpufreq_policy *data;
 137         unsigned long flags;
 138
 139         if (cpu >= nr_cpu_ids)
 140                 goto err_out;
 141
 142         /* get the cpufreq driver */
 143         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 144
 145         if (!cpufreq_driver)
 146                 goto err_out_unlock;
 147
 148         if (!try_module_get(cpufreq_driver->owner))
 149                 goto err_out_unlock;
 150
 151
 152         /* get the CPU */
 153         data = per_cpu(cpufreq_cpu_data, cpu);
 154
 155         if (!data)
 156                 goto err_out_put_module;
 157
 158         if (!kobject_get(&data->kobj))
 159                 goto err_out_put_module;
 160
 161         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 162         return data;
 163
 164 err_out_put_module:
 165         module_put(cpufreq_driver->owner);
 166 err_out_unlock:
 167         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 168 err_out:
 169         return NULL;
 170 }
 171 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 172
 173
 174 void cpufreq_cpu_put(struct cpufreq_policy *data)
 175 {
 176         kobject_put(&data->kobj);
 177         module_put(cpufreq_driver->owner);
 178 }
 179 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 180
 181
 182 /*********************************************************************
 183  *                     UNIFIED DEBUG HELPERS                         *
 184  *********************************************************************/
 185 #ifdef CONFIG_CPU_FREQ_DEBUG
 186
 187 /* what part(s) of the CPUfreq subsystem are debugged? */
 188 static unsigned int debug;
 189
 190 /* is the debug output ratelimit'ed using printk_ratelimit? User can
 191  * set or modify this value.
 192  */
 193 static unsigned int debug_ratelimit = 1;
 194
 195 /* is the printk_ratelimit'ing enabled? It's enabled after a successful
 196  * loading of a cpufreq driver, temporarily disabled when a new policy
 197  * is set, and disabled upon cpufreq driver removal
 198  */
 199 static unsigned int disable_ratelimit = 1;
 200 static DEFINE_SPINLOCK(disable_ratelimit_lock);
 201
 202 static void cpufreq_debug_enable_ratelimit(void)
 203 {
 204         unsigned long flags;
 205
 206         spin_lock_irqsave(&disable_ratelimit_lock, flags);
 207         if (disable_ratelimit)
 208                 disable_ratelimit--;
 209         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 210 }
 211
 212 static void cpufreq_debug_disable_ratelimit(void)
 213 {
 214         unsigned long flags;
 215
 216         spin_lock_irqsave(&disable_ratelimit_lock, flags);
 217         disable_ratelimit++;
 218         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 219 }
 220
 221 void cpufreq_debug_printk(unsigned int type, const char *prefix,
 222                         const char *fmt, ...)
 223 {
 224         char s[256];
 225         va_list args;
 226         unsigned int len;
 227         unsigned long flags;
 228
 229         WARN_ON(!prefix);
 230         if (type & debug) {
 231                 spin_lock_irqsave(&disable_ratelimit_lock, flags);
 232                 if (!disable_ratelimit && debug_ratelimit
 233                                         && !printk_ratelimit()) {
 234                         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 235                         return;
 236                 }
 237                 spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 238
 239                 len = snprintf(s, 256, KERN_DEBUG "%s: ", prefix);
 240
 241                 va_start(args, fmt);
 242                 len += vsnprintf(&s[len], (256 - len), fmt, args);
 243                 va_end(args);
 244
 245                 printk(s);
 246
 247                 WARN_ON(len < 5);
 248         }
 249 }
 250 EXPORT_SYMBOL(cpufreq_debug_printk);
 251
 252
 253 module_param(debug, uint, 0644);
 254 MODULE_PARM_DESC(debug, "CPUfreq debugging: add 1 to debug core,"
 255                         " 2 to debug drivers, and 4 to debug governors.");
 256
 257 module_param(debug_ratelimit, uint, 0644);
 258 MODULE_PARM_DESC(debug_ratelimit, "CPUfreq debugging:"
 259                                         " set to 0 to disable ratelimiting.");
 260
 261 #else /* !CONFIG_CPU_FREQ_DEBUG */
 262
 263 static inline void cpufreq_debug_enable_ratelimit(void) { return; }
 264 static inline void cpufreq_debug_disable_ratelimit(void) { return; }
 265
 266 #endif /* CONFIG_CPU_FREQ_DEBUG */
 267
 268
 269 /*********************************************************************
 270  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 271  *********************************************************************/
 272
 273 /**
 274  * adjust_jiffies - adjust the system "loops_per_jiffy"
 275  *
 276  * This function alters the system "loops_per_jiffy" for the clock
 277  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 278  * systems as each CPU might be scaled differently. So, use the arch
 279  * per-CPU loops_per_jiffy value wherever possible.
 280  */
 281 #ifndef CONFIG_SMP
 282 static unsigned long l_p_j_ref;
 283 static unsigned int  l_p_j_ref_freq;
 284
 285 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 286 {
 287         if (ci->flags & CPUFREQ_CONST_LOOPS)
 288                 return;
 289
 290         if (!l_p_j_ref_freq) {
 291                 l_p_j_ref = loops_per_jiffy;
 292                 l_p_j_ref_freq = ci->old;
 293                 dprintk("saving %lu as reference value for loops_per_jiffy; "
 294                         "freq is %u kHz\n", l_p_j_ref, l_p_j_ref_freq);
 295         }
 296         if ((val == CPUFREQ_PRECHANGE  && ci->old < ci->new) ||
 297             (val == CPUFREQ_POSTCHANGE && ci->old > ci->new) ||
 298             (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) {
 299                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 300                                                                 ci->new);
 301                 dprintk("scaling loops_per_jiffy to %lu "
 302                         "for frequency %u kHz\n", loops_per_jiffy, ci->new);
 303         }
 304 }
 305 #else
 306 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 307 {
 308         return;
 309 }
 310 #endif
 311
 312
 313 /**
 314  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 315  * on frequency transition.
 316  *
 317  * This function calls the transition notifiers and the "adjust_jiffies"
 318  * function. It is called twice on all CPU frequency changes that have
 319  * external effects.
 320  */
 321 void cpufreq_notify_transition(struct cpufreq_freqs *freqs, unsigned int state)
 322 {
 323         struct cpufreq_policy *policy;
 324
 325         BUG_ON(irqs_disabled());
 326
 327         freqs->flags = cpufreq_driver->flags;
 328         dprintk("notification %u of frequency transition to %u kHz\n",
 329                 state, freqs->new);
 330
 331         policy = per_cpu(cpufreq_cpu_data, freqs->cpu);
 332         switch (state) {
 333
 334         case CPUFREQ_PRECHANGE:
 335                 /* detect if the driver reported a value as "old frequency"
 336                  * which is not equal to what the cpufreq core thinks is
 337                  * "old frequency".
 338                  */
 339                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 340                         if ((policy) && (policy->cpu == freqs->cpu) &&
 341                             (policy->cur) && (policy->cur != freqs->old)) {
 342                                 dprintk("Warning: CPU frequency is"
 343                                         " %u, cpufreq assumed %u kHz.\n",
 344                                         freqs->old, policy->cur);
 345                                 freqs->old = policy->cur;
 346                         }
 347                 }
 348                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 349                                 CPUFREQ_PRECHANGE, freqs);
 350                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 351                 break;
 352
 353         case CPUFREQ_POSTCHANGE:
 354                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 355                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 356                                 CPUFREQ_POSTCHANGE, freqs);
 357                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 358                         policy->cur = freqs->new;
 359                 break;
 360         }
 361 }
 362 EXPORT_SYMBOL_GPL(cpufreq_notify_transition);
 363
 364
 365
 366 /*********************************************************************
 367  *                          SYSFS INTERFACE                          *
 368  *********************************************************************/
 369
 370 static struct cpufreq_governor *__find_governor(const char *str_governor)
 371 {
 372         struct cpufreq_governor *t;
 373
 374         list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 375                 if (!strnicmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 376                         return t;
 377
 378         return NULL;
 379 }
 380
 381 /**
 382  * cpufreq_parse_governor - parse a governor string
 383  */
 384 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 385                                 struct cpufreq_governor **governor)
 386 {
 387         int err = -EINVAL;
 388
 389         if (!cpufreq_driver)
 390                 goto out;
 391
 392         if (cpufreq_driver->setpolicy) {
 393                 if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 394                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 395                         err = 0;
 396                 } else if (!strnicmp(str_governor, "powersave",
 397                                                 CPUFREQ_NAME_LEN)) {
 398                         *policy = CPUFREQ_POLICY_POWERSAVE;
 399                         err = 0;
 400                 }
 401         } else if (cpufreq_driver->target) {
 402                 struct cpufreq_governor *t;
 403
 404                 mutex_lock(&cpufreq_governor_mutex);
 405
 406                 t = __find_governor(str_governor);
 407
 408                 if (t == NULL) {
 409                         char *name = kasprintf(GFP_KERNEL, "cpufreq_%s",
 410                                                                 str_governor);
 411
 412                         if (name) {
 413                                 int ret;
 414
 415                                 mutex_unlock(&cpufreq_governor_mutex);
 416                                 ret = request_module("%s", name);
 417                                 mutex_lock(&cpufreq_governor_mutex);
 418
 419                                 if (ret == 0)
 420                                         t = __find_governor(str_governor);
 421                         }
 422
 423                         kfree(name);
 424                 }
 425
 426                 if (t != NULL) {
 427                         *governor = t;
 428                         err = 0;
 429                 }
 430
 431                 mutex_unlock(&cpufreq_governor_mutex);
 432         }
 433 out:
 434         return err;
 435 }
 436
 437
 438 /**
 439  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 440  * print out cpufreq information
 441  *
 442  * Write out information from cpufreq_driver->policy[cpu]; object must be
 443  * "unsigned int".
 444  */
 445
 446 #define show_one(file_name, object)                     \
 447 static ssize_t show_##file_name                         \
 448 (struct cpufreq_policy *policy, char *buf)              \
 449 {                                                       \
 450         return sprintf(buf, "%u\n", policy->object);    \
 451 }
 452
 453 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 454 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 455 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 456 show_one(scaling_min_freq, min);
 457 show_one(scaling_max_freq, max);
 458 show_one(scaling_cur_freq, cur);
 459
 460 static int __cpufreq_set_policy(struct cpufreq_policy *data,
 461                                 struct cpufreq_policy *policy);
 462
 463 /**
 464  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 465  */
 466 #define store_one(file_name, object)                    \
 467 static ssize_t store_##file_name                                        \
 468 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 469 {                                                                       \
 470         unsigned int ret = -EINVAL;                                     \
 471         struct cpufreq_policy new_policy;                               \
 472                                                                         \
 473         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 474         if (ret)                                                        \
 475                 return -EINVAL;                                         \
 476                                                                         \
 477         ret = sscanf(buf, "%u", &new_policy.object);                    \
 478         if (ret != 1)                                                   \
 479                 return -EINVAL;                                         \
 480                                                                         \
 481         ret = __cpufreq_set_policy(policy, &new_policy);                \
 482         policy->user_policy.object = policy->object;                    \
 483                                                                         \
 484         return ret ? ret : count;                                       \
 485 }
 486
 487 store_one(scaling_min_freq, min);
 488 store_one(scaling_max_freq, max);
 489
 490 /**
 491  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 492  */
 493 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 494                                         char *buf)
 495 {
 496         unsigned int cur_freq = __cpufreq_get(policy->cpu);
 497         if (!cur_freq)
 498                 return sprintf(buf, "<unknown>");
 499         return sprintf(buf, "%u\n", cur_freq);
 500 }
 501
 502
 503 /**
 504  * show_scaling_governor - show the current policy for the specified CPU
 505  */
 506 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 507 {
 508         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 509                 return sprintf(buf, "powersave\n");
 510         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 511                 return sprintf(buf, "performance\n");
 512         else if (policy->governor)
 513                 return scnprintf(buf, CPUFREQ_NAME_LEN, "%s\n",
 514                                 policy->governor->name);
 515         return -EINVAL;
 516 }
 517
 518
 519 /**
 520  * store_scaling_governor - store policy for the specified CPU
 521  */
 522 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 523                                         const char *buf, size_t count)
 524 {
 525         unsigned int ret = -EINVAL;
 526         char    str_governor[16];
 527         struct cpufreq_policy new_policy;
 528
 529         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 530         if (ret)
 531                 return ret;
 532
 533         ret = sscanf(buf, "%15s", str_governor);
 534         if (ret != 1)
 535                 return -EINVAL;
 536
 537         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 538                                                 &new_policy.governor))
 539                 return -EINVAL;
 540
 541         /* Do not use cpufreq_set_policy here or the user_policy.max
 542            will be wrongly overridden */
 543         ret = __cpufreq_set_policy(policy, &new_policy);
 544
 545         policy->user_policy.policy = policy->policy;
 546         policy->user_policy.governor = policy->governor;
 547
 548         if (ret)
 549                 return ret;
 550         else
 551                 return count;
 552 }
 553
 554 /**
 555  * show_scaling_driver - show the cpufreq driver currently loaded
 556  */
 557 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 558 {
 559         return scnprintf(buf, CPUFREQ_NAME_LEN, "%s\n", cpufreq_driver->name);
 560 }
 561
 562 /**
 563  * show_scaling_available_governors - show the available CPUfreq governors
 564  */
 565 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 566                                                 char *buf)
 567 {
 568         ssize_t i = 0;
 569         struct cpufreq_governor *t;
 570
 571         if (!cpufreq_driver->target) {
 572                 i += sprintf(buf, "performance powersave");
 573                 goto out;
 574         }
 575
 576         list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 577                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 578                     - (CPUFREQ_NAME_LEN + 2)))
 579                         goto out;
 580                 i += scnprintf(&buf[i], CPUFREQ_NAME_LEN, "%s ", t->name);
 581         }
 582 out:
 583         i += sprintf(&buf[i], "\n");
 584         return i;
 585 }
 586
 587 static ssize_t show_cpus(const struct cpumask *mask, char *buf)
 588 {
 589         ssize_t i = 0;
 590         unsigned int cpu;
 591
 592         for_each_cpu(cpu, mask) {
 593                 if (i)
 594                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 595                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 596                 if (i >= (PAGE_SIZE - 5))
 597                         break;
 598         }
 599         i += sprintf(&buf[i], "\n");
 600         return i;
 601 }
 602
 603 /**
 604  * show_related_cpus - show the CPUs affected by each transition even if
 605  * hw coordination is in use
 606  */
 607 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 608 {
 609         if (cpumask_empty(policy->related_cpus))
 610                 return show_cpus(policy->cpus, buf);
 611         return show_cpus(policy->related_cpus, buf);
 612 }
 613
 614 /**
 615  * show_affected_cpus - show the CPUs affected by each transition
 616  */
 617 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 618 {
 619         return show_cpus(policy->cpus, buf);
 620 }
 621
 622 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 623                                         const char *buf, size_t count)
 624 {
 625         unsigned int freq = 0;
 626         unsigned int ret;
 627
 628         if (!policy->governor || !policy->governor->store_setspeed)
 629                 return -EINVAL;
 630
 631         ret = sscanf(buf, "%u", &freq);
 632         if (ret != 1)
 633                 return -EINVAL;
 634
 635         policy->governor->store_setspeed(policy, freq);
 636
 637         return count;
 638 }
 639
 640 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 641 {
 642         if (!policy->governor || !policy->governor->show_setspeed)
 643                 return sprintf(buf, "<unsupported>\n");
 644
 645         return policy->governor->show_setspeed(policy, buf);
 646 }
 647
 648 #define define_one_ro(_name) \
 649 static struct freq_attr _name = \
 650 __ATTR(_name, 0444, show_##_name, NULL)
 651
 652 #define define_one_ro0400(_name) \
 653 static struct freq_attr _name = \
 654 __ATTR(_name, 0400, show_##_name, NULL)
 655
 656 #define define_one_rw(_name) \
 657 static struct freq_attr _name = \
 658 __ATTR(_name, 0644, show_##_name, store_##_name)
 659
 660 define_one_ro0400(cpuinfo_cur_freq);
 661 define_one_ro(cpuinfo_min_freq);
 662 define_one_ro(cpuinfo_max_freq);
 663 define_one_ro(cpuinfo_transition_latency);
 664 define_one_ro(scaling_available_governors);
 665 define_one_ro(scaling_driver);
 666 define_one_ro(scaling_cur_freq);
 667 define_one_ro(related_cpus);
 668 define_one_ro(affected_cpus);
 669 define_one_rw(scaling_min_freq);
 670 define_one_rw(scaling_max_freq);
 671 define_one_rw(scaling_governor);
 672 define_one_rw(scaling_setspeed);
 673
 674 static struct attribute *default_attrs[] = {
 675         &cpuinfo_min_freq.attr,
 676         &cpuinfo_max_freq.attr,
 677         &cpuinfo_transition_latency.attr,
 678         &scaling_min_freq.attr,
 679         &scaling_max_freq.attr,
 680         &affected_cpus.attr,
 681         &related_cpus.attr,
 682         &scaling_governor.attr,
 683         &scaling_driver.attr,
 684         &scaling_available_governors.attr,
 685         &scaling_setspeed.attr,
 686         NULL
 687 };
 688
 689 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 690 #define to_attr(a) container_of(a, struct freq_attr, attr)
 691
 692 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 693 {
 694         struct cpufreq_policy *policy = to_policy(kobj);
 695         struct freq_attr *fattr = to_attr(attr);
 696         ssize_t ret = -EINVAL;
 697         policy = cpufreq_cpu_get(policy->cpu);
 698         if (!policy)
 699                 goto no_policy;
 700
 701         if (lock_policy_rwsem_read(policy->cpu) < 0)
 702                 goto fail;
 703
 704         if (fattr->show)
 705                 ret = fattr->show(policy, buf);
 706         else
 707                 ret = -EIO;
 708
 709         unlock_policy_rwsem_read(policy->cpu);
 710 fail:
 711         cpufreq_cpu_put(policy);
 712 no_policy:
 713         return ret;
 714 }
 715
 716 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 717                      const char *buf, size_t count)
 718 {
 719         struct cpufreq_policy *policy = to_policy(kobj);
 720         struct freq_attr *fattr = to_attr(attr);
 721         ssize_t ret = -EINVAL;
 722         policy = cpufreq_cpu_get(policy->cpu);
 723         if (!policy)
 724                 goto no_policy;
 725
 726         if (lock_policy_rwsem_write(policy->cpu) < 0)
 727                 goto fail;
 728
 729         if (fattr->store)
 730                 ret = fattr->store(policy, buf, count);
 731         else
 732                 ret = -EIO;
 733
 734         unlock_policy_rwsem_write(policy->cpu);
 735 fail:
 736         cpufreq_cpu_put(policy);
 737 no_policy:
 738         return ret;
 739 }
 740
 741 static void cpufreq_sysfs_release(struct kobject *kobj)
 742 {
 743         struct cpufreq_policy *policy = to_policy(kobj);
 744         dprintk("last reference is dropped\n");
 745         complete(&policy->kobj_unregister);
 746 }
 747
 748 static struct sysfs_ops sysfs_ops = {
 749         .show   = show,
 750         .store  = store,
 751 };
 752
 753 static struct kobj_type ktype_cpufreq = {
 754         .sysfs_ops      = &sysfs_ops,
 755         .default_attrs  = default_attrs,
 756         .release        = cpufreq_sysfs_release,
 757 };
 758
 759 /* symlink affected CPUs */
 760 int cpufreq_add_dev_symlink(unsigned int cpu, struct cpufreq_policy *policy)
 761 {
 762         unsigned int j;
 763         int ret = 0;
 764
 765         for_each_cpu(j, policy->cpus) {
 766                 struct cpufreq_policy *managed_policy;
 767                 struct sys_device *cpu_sys_dev;
 768
 769                 if (j == cpu)
 770                         continue;
 771                 if (!cpu_online(j))
 772                         continue;
 773
 774                 dprintk("CPU %u already managed, adding link\n", j);
 775                 managed_policy = cpufreq_cpu_get(cpu);
 776                 cpu_sys_dev = get_cpu_sysdev(j);
 777                 ret = sysfs_create_link(&cpu_sys_dev->kobj, &policy->kobj,
 778                                         "cpufreq");
 779                 if (ret) {
 780                         cpufreq_cpu_put(managed_policy);
 781                         return ret;
 782                 }
 783         }
 784         return ret;
 785 }
 786
 787 int cpufreq_add_dev_interface(unsigned int cpu, struct cpufreq_policy *policy,
 788                 struct sys_device *sys_dev)
 789 {
 790         struct freq_attr **drv_attr;
 791         unsigned long flags;
 792         int ret = 0;
 793         unsigned int j;
 794
 795         /* prepare interface data */
 796         ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
 797                                    &sys_dev->kobj, "cpufreq");
 798         if (ret)
 799                 return ret;
 800
 801         /* set up files for this cpu device */
 802         drv_attr = cpufreq_driver->attr;
 803         while ((drv_attr) && (*drv_attr)) {
 804                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 805                 if (ret)
 806                         goto err_out_kobj_put;
 807                 drv_attr++;
 808         }
 809         if (cpufreq_driver->get) {
 810                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 811                 if (ret)
 812                         goto err_out_kobj_put;
 813         }
 814         if (cpufreq_driver->target) {
 815                 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 816                 if (ret)
 817                         goto err_out_kobj_put;
 818         }
 819
 820         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 821         for_each_cpu(j, policy->cpus) {
 822         if (!cpu_online(j))
 823                 continue;
 824                 per_cpu(cpufreq_cpu_data, j) = policy;
 825                 per_cpu(policy_cpu, j) = policy->cpu;
 826         }
 827         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 828
 829         ret = cpufreq_add_dev_symlink(cpu, policy);
 830         return ret;
 831
 832 err_out_kobj_put:
 833         kobject_put(&policy->kobj);
 834         wait_for_completion(&policy->kobj_unregister);
 835         return ret;
 836 }
 837
 838
 839 /**
 840  * cpufreq_add_dev - add a CPU device
 841  *
 842  * Adds the cpufreq interface for a CPU device.
 843  *
 844  * The Oracle says: try running cpufreq registration/unregistration concurrently
 845  * with with cpu hotplugging and all hell will break loose. Tried to clean this
 846  * mess up, but more thorough testing is needed. - Mathieu
 847  */
 848 static int cpufreq_add_dev(struct sys_device *sys_dev)
 849 {
 850         unsigned int cpu = sys_dev->id;
 851         int ret = 0;
 852         struct cpufreq_policy new_policy;
 853         struct cpufreq_policy *policy;
 854         unsigned long flags;
 855         unsigned int j;
 856
 857         if (cpu_is_offline(cpu))
 858                 return 0;
 859
 860         cpufreq_debug_disable_ratelimit();
 861         dprintk("adding CPU %u\n", cpu);
 862
 863 #ifdef CONFIG_SMP
 864         /* check whether a different CPU already registered this
 865          * CPU because it is in the same boat. */
 866         policy = cpufreq_cpu_get(cpu);
 867         if (unlikely(policy)) {
 868                 cpufreq_cpu_put(policy);
 869                 cpufreq_debug_enable_ratelimit();
 870                 return 0;
 871         }
 872 #endif
 873
 874         if (!try_module_get(cpufreq_driver->owner)) {
 875                 ret = -EINVAL;
 876                 goto module_out;
 877         }
 878
 879         ret = -ENOMEM;
 880         policy = kzalloc(sizeof(struct cpufreq_policy), GFP_KERNEL);
 881         if (!policy)
 882                 goto nomem_out;
 883
 884         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
 885                 goto err_free_policy;
 886
 887         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
 888                 goto err_free_cpumask;
 889
 890         policy->cpu = cpu;
 891         cpumask_copy(policy->cpus, cpumask_of(cpu));
 892
 893         /* Initially set CPU itself as the policy_cpu */
 894         per_cpu(policy_cpu, cpu) = cpu;
 895         ret = (lock_policy_rwsem_write(cpu) < 0);
 896         WARN_ON(ret);
 897
 898         init_completion(&policy->kobj_unregister);
 899         INIT_WORK(&policy->update, handle_update);
 900
 901         /* Set governor before ->init, so that driver could check it */
 902         policy->governor = CPUFREQ_DEFAULT_GOVERNOR;
 903         /* call driver. From then on the cpufreq must be able
 904          * to accept all calls to ->verify and ->setpolicy for this CPU
 905          */
 906         ret = cpufreq_driver->init(policy);
 907         if (ret) {
 908                 dprintk("initialization failed\n");
 909                 goto err_unlock_policy;
 910         }
 911         policy->user_policy.min = policy->min;
 912         policy->user_policy.max = policy->max;
 913
 914         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
 915                                      CPUFREQ_START, policy);
 916
 917 #ifdef CONFIG_SMP
 918
 919 #ifdef CONFIG_HOTPLUG_CPU
 920         if (per_cpu(cpufreq_cpu_governor, cpu)) {
 921                 policy->governor = per_cpu(cpufreq_cpu_governor, cpu);
 922                 dprintk("Restoring governor %s for cpu %d\n",
 923                        policy->governor->name, cpu);
 924         }
 925 #endif
 926
 927         for_each_cpu(j, policy->cpus) {
 928                 struct cpufreq_policy *managed_policy;
 929
 930                 if (cpu == j)
 931                         continue;
 932
 933                 /* Check for existing affected CPUs.
 934                  * They may not be aware of it due to CPU Hotplug.
 935                  * cpufreq_cpu_put is called when the device is removed
 936                  * in __cpufreq_remove_dev()
 937                  */
 938                 managed_policy = cpufreq_cpu_get(j);
 939                 if (unlikely(managed_policy)) {
 940
 941                         /* Set proper policy_cpu */
 942                         unlock_policy_rwsem_write(cpu);
 943                         per_cpu(policy_cpu, cpu) = managed_policy->cpu;
 944
 945                         if (lock_policy_rwsem_write(cpu) < 0) {
 946                                 /* Should not go through policy unlock path */
 947                                 if (cpufreq_driver->exit)
 948                                         cpufreq_driver->exit(policy);
 949                                 ret = -EBUSY;
 950                                 cpufreq_cpu_put(managed_policy);
 951                                 goto err_free_cpumask;
 952                         }
 953
 954                         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 955                         cpumask_copy(managed_policy->cpus, policy->cpus);
 956                         per_cpu(cpufreq_cpu_data, cpu) = managed_policy;
 957                         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 958
 959                         dprintk("CPU already managed, adding link\n");
 960                         ret = sysfs_create_link(&sys_dev->kobj,
 961                                                 &managed_policy->kobj,
 962                                                 "cpufreq");
 963                         if (ret)
 964                                 cpufreq_cpu_put(managed_policy);
 965                         /*
 966                          * Success. We only needed to be added to the mask.
 967                          * Call driver->exit() because only the cpu parent of
 968                          * the kobj needed to call init().
 969                          */
 970                         goto out_driver_exit; /* call driver->exit() */
 971                 }
 972         }
 973 #endif
 974         memcpy(&new_policy, policy, sizeof(struct cpufreq_policy));
 975
 976         ret = cpufreq_add_dev_interface(cpu, policy, sys_dev);
 977         if (ret)
 978                 goto err_out_unregister;
 979
 980         policy->governor = NULL; /* to assure that the starting sequence is
 981                                   * run in cpufreq_set_policy */
 982
 983         /* set default policy */
 984         ret = __cpufreq_set_policy(policy, &new_policy);
 985         policy->user_policy.policy = policy->policy;
 986         policy->user_policy.governor = policy->governor;
 987
 988         if (ret) {
 989                 dprintk("setting policy failed\n");
 990                 goto err_out_unregister;
 991         }
 992
 993         unlock_policy_rwsem_write(cpu);
 994
 995         kobject_uevent(&policy->kobj, KOBJ_ADD);
 996         module_put(cpufreq_driver->owner);
 997         dprintk("initialization complete\n");
 998         cpufreq_debug_enable_ratelimit();
 999
1000         return 0;
1001
1002
1003 err_out_unregister:
1004         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1005         for_each_cpu(j, policy->cpus)
1006                 per_cpu(cpufreq_cpu_data, j) = NULL;
1007         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1008
1009         kobject_put(&policy->kobj);
1010         wait_for_completion(&policy->kobj_unregister);
1011
1012 out_driver_exit:
1013         if (cpufreq_driver->exit)
1014                 cpufreq_driver->exit(policy);
1015
1016 err_unlock_policy:
1017         unlock_policy_rwsem_write(cpu);
1018 err_free_cpumask:
1019         free_cpumask_var(policy->cpus);
1020 err_free_policy:
1021         kfree(policy);
1022 nomem_out:
1023         module_put(cpufreq_driver->owner);
1024 module_out:
1025         cpufreq_debug_enable_ratelimit();
1026         return ret;
1027 }
1028
1029
1030 /**
1031  * __cpufreq_remove_dev - remove a CPU device
1032  *
1033  * Removes the cpufreq interface for a CPU device.
1034  * Caller should already have policy_rwsem in write mode for this CPU.
1035  * This routine frees the rwsem before returning.
1036  */
1037 static int __cpufreq_remove_dev(struct sys_device *sys_dev)
1038 {
1039         unsigned int cpu = sys_dev->id;
1040         unsigned long flags;
1041         struct cpufreq_policy *data;
1042 #ifdef CONFIG_SMP
1043         struct sys_device *cpu_sys_dev;
1044         unsigned int j;
1045 #endif
1046
1047         cpufreq_debug_disable_ratelimit();
1048         dprintk("unregistering CPU %u\n", cpu);
1049
1050         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1051         data = per_cpu(cpufreq_cpu_data, cpu);
1052
1053         if (!data) {
1054                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1055                 cpufreq_debug_enable_ratelimit();
1056                 unlock_policy_rwsem_write(cpu);
1057                 return -EINVAL;
1058         }
1059         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1060
1061
1062 #ifdef CONFIG_SMP
1063         /* if this isn't the CPU which is the parent of the kobj, we
1064          * only need to unlink, put and exit
1065          */
1066         if (unlikely(cpu != data->cpu)) {
1067                 dprintk("removing link\n");
1068                 cpumask_clear_cpu(cpu, data->cpus);
1069                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1070                 sysfs_remove_link(&sys_dev->kobj, "cpufreq");
1071                 cpufreq_cpu_put(data);
1072                 cpufreq_debug_enable_ratelimit();
1073                 unlock_policy_rwsem_write(cpu);
1074                 return 0;
1075         }
1076 #endif
1077
1078 #ifdef CONFIG_SMP
1079
1080 #ifdef CONFIG_HOTPLUG_CPU
1081         per_cpu(cpufreq_cpu_governor, cpu) = data->governor;
1082 #endif
1083
1084         /* if we have other CPUs still registered, we need to unlink them,
1085          * or else wait_for_completion below will lock up. Clean the
1086          * per_cpu(cpufreq_cpu_data) while holding the lock, and remove
1087          * the sysfs links afterwards.
1088          */
1089         if (unlikely(cpumask_weight(data->cpus) > 1)) {
1090                 for_each_cpu(j, data->cpus) {
1091                         if (j == cpu)
1092                                 continue;
1093                         per_cpu(cpufreq_cpu_data, j) = NULL;
1094                 }
1095         }
1096
1097         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1098
1099         if (unlikely(cpumask_weight(data->cpus) > 1)) {
1100                 for_each_cpu(j, data->cpus) {
1101                         if (j == cpu)
1102                                 continue;
1103                         dprintk("removing link for cpu %u\n", j);
1104 #ifdef CONFIG_HOTPLUG_CPU
1105                         per_cpu(cpufreq_cpu_governor, j) = data->governor;
1106 #endif
1107                         cpu_sys_dev = get_cpu_sysdev(j);
1108                         sysfs_remove_link(&cpu_sys_dev->kobj, "cpufreq");
1109                         cpufreq_cpu_put(data);
1110                 }
1111         }
1112 #else
1113         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1114 #endif
1115
1116         if (cpufreq_driver->target)
1117                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1118
1119         kobject_put(&data->kobj);
1120
1121         /* we need to make sure that the underlying kobj is actually
1122          * not referenced anymore by anybody before we proceed with
1123          * unloading.
1124          */
1125         dprintk("waiting for dropping of refcount\n");
1126         wait_for_completion(&data->kobj_unregister);
1127         dprintk("wait complete\n");
1128
1129         if (cpufreq_driver->exit)
1130                 cpufreq_driver->exit(data);
1131
1132         unlock_policy_rwsem_write(cpu);
1133
1134         free_cpumask_var(data->related_cpus);
1135         free_cpumask_var(data->cpus);
1136         kfree(data);
1137         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1138
1139         cpufreq_debug_enable_ratelimit();
1140         return 0;
1141 }
1142
1143
1144 static int cpufreq_remove_dev(struct sys_device *sys_dev)
1145 {
1146         unsigned int cpu = sys_dev->id;
1147         int retval;
1148
1149         if (cpu_is_offline(cpu))
1150                 return 0;
1151
1152         if (unlikely(lock_policy_rwsem_write(cpu)))
1153                 BUG();
1154
1155         retval = __cpufreq_remove_dev(sys_dev);
1156         return retval;
1157 }
1158
1159
1160 static void handle_update(struct work_struct *work)
1161 {
1162         struct cpufreq_policy *policy =
1163                 container_of(work, struct cpufreq_policy, update);
1164         unsigned int cpu = policy->cpu;
1165         dprintk("handle_update for cpu %u called\n", cpu);
1166         cpufreq_update_policy(cpu);
1167 }
1168
1169 /**
1170  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're in deep trouble.
1171  *      @cpu: cpu number
1172  *      @old_freq: CPU frequency the kernel thinks the CPU runs at
1173  *      @new_freq: CPU frequency the CPU actually runs at
1174  *
1175  *      We adjust to current frequency first, and need to clean up later.
1176  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1177  */
1178 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1179                                 unsigned int new_freq)
1180 {
1181         struct cpufreq_freqs freqs;
1182
1183         dprintk("Warning: CPU frequency out of sync: cpufreq and timing "
1184                "core thinks of %u, is %u kHz.\n", old_freq, new_freq);
1185
1186         freqs.cpu = cpu;
1187         freqs.old = old_freq;
1188         freqs.new = new_freq;
1189         cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE);
1190         cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE);
1191 }
1192
1193
1194 /**
1195  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1196  * @cpu: CPU number
1197  *
1198  * This is the last known freq, without actually getting it from the driver.
1199  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1200  */
1201 unsigned int cpufreq_quick_get(unsigned int cpu)
1202 {
1203         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1204         unsigned int ret_freq = 0;
1205
1206         if (policy) {
1207                 ret_freq = policy->cur;
1208                 cpufreq_cpu_put(policy);
1209         }
1210
1211         return ret_freq;
1212 }
1213 EXPORT_SYMBOL(cpufreq_quick_get);
1214
1215
1216 static unsigned int __cpufreq_get(unsigned int cpu)
1217 {
1218         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1219         unsigned int ret_freq = 0;
1220
1221         if (!cpufreq_driver->get)
1222                 return ret_freq;
1223
1224         ret_freq = cpufreq_driver->get(cpu);
1225
1226         if (ret_freq && policy->cur &&
1227                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1228                 /* verify no discrepancy between actual and
1229                                         saved value exists */
1230                 if (unlikely(ret_freq != policy->cur)) {
1231                         cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1232                         schedule_work(&policy->update);
1233                 }
1234         }
1235
1236         return ret_freq;
1237 }
1238
1239 /**
1240  * cpufreq_get - get the current CPU frequency (in kHz)
1241  * @cpu: CPU number
1242  *
1243  * Get the CPU current (static) CPU frequency
1244  */
1245 unsigned int cpufreq_get(unsigned int cpu)
1246 {
1247         unsigned int ret_freq = 0;
1248         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1249
1250         if (!policy)
1251                 goto out;
1252
1253         if (unlikely(lock_policy_rwsem_read(cpu)))
1254                 goto out_policy;
1255
1256         ret_freq = __cpufreq_get(cpu);
1257
1258         unlock_policy_rwsem_read(cpu);
1259
1260 out_policy:
1261         cpufreq_cpu_put(policy);
1262 out:
1263         return ret_freq;
1264 }
1265 EXPORT_SYMBOL(cpufreq_get);
1266
1267
1268 /**
1269  *      cpufreq_suspend - let the low level driver prepare for suspend
1270  */
1271
1272 static int cpufreq_suspend(struct sys_device *sysdev, pm_message_t pmsg)
1273 {
1274         int ret = 0;
1275
1276         int cpu = sysdev->id;
1277         struct cpufreq_policy *cpu_policy;
1278
1279         dprintk("suspending cpu %u\n", cpu);
1280
1281         if (!cpu_online(cpu))
1282                 return 0;
1283
1284         /* we may be lax here as interrupts are off. Nonetheless
1285          * we need to grab the correct cpu policy, as to check
1286          * whether we really run on this CPU.
1287          */
1288
1289         cpu_policy = cpufreq_cpu_get(cpu);
1290         if (!cpu_policy)
1291                 return -EINVAL;
1292
1293         /* only handle each CPU group once */
1294         if (unlikely(cpu_policy->cpu != cpu))
1295                 goto out;
1296
1297         if (cpufreq_driver->suspend) {
1298                 ret = cpufreq_driver->suspend(cpu_policy, pmsg);
1299                 if (ret)
1300                         printk(KERN_ERR "cpufreq: suspend failed in ->suspend "
1301                                         "step on CPU %u\n", cpu_policy->cpu);
1302         }
1303
1304 out:
1305         cpufreq_cpu_put(cpu_policy);
1306         return ret;
1307 }
1308
1309 /**
1310  *      cpufreq_resume -  restore proper CPU frequency handling after resume
1311  *
1312  *      1.) resume CPUfreq hardware support (cpufreq_driver->resume())
1313  *      2.) schedule call cpufreq_update_policy() ASAP as interrupts are
1314  *          restored. It will verify that the current freq is in sync with
1315  *          what we believe it to be. This is a bit later than when it
1316  *          should be, but nonethteless it's better than calling
1317  *          cpufreq_driver->get() here which might re-enable interrupts...
1318  */
1319 static int cpufreq_resume(struct sys_device *sysdev)
1320 {
1321         int ret = 0;
1322
1323         int cpu = sysdev->id;
1324         struct cpufreq_policy *cpu_policy;
1325
1326         dprintk("resuming cpu %u\n", cpu);
1327
1328         if (!cpu_online(cpu))
1329                 return 0;
1330
1331         /* we may be lax here as interrupts are off. Nonetheless
1332          * we need to grab the correct cpu policy, as to check
1333          * whether we really run on this CPU.
1334          */
1335
1336         cpu_policy = cpufreq_cpu_get(cpu);
1337         if (!cpu_policy)
1338                 return -EINVAL;
1339
1340         /* only handle each CPU group once */
1341         if (unlikely(cpu_policy->cpu != cpu))
1342                 goto fail;
1343
1344         if (cpufreq_driver->resume) {
1345                 ret = cpufreq_driver->resume(cpu_policy);
1346                 if (ret) {
1347                         printk(KERN_ERR "cpufreq: resume failed in ->resume "
1348                                         "step on CPU %u\n", cpu_policy->cpu);
1349                         goto fail;
1350                 }
1351         }
1352
1353         schedule_work(&cpu_policy->update);
1354
1355 fail:
1356         cpufreq_cpu_put(cpu_policy);
1357         return ret;
1358 }
1359
1360 static struct sysdev_driver cpufreq_sysdev_driver = {
1361         .add            = cpufreq_add_dev,
1362         .remove         = cpufreq_remove_dev,
1363         .suspend        = cpufreq_suspend,
1364         .resume         = cpufreq_resume,
1365 };
1366
1367
1368 /*********************************************************************
1369  *                     NOTIFIER LISTS INTERFACE                      *
1370  *********************************************************************/
1371
1372 /**
1373  *      cpufreq_register_notifier - register a driver with cpufreq
1374  *      @nb: notifier function to register
1375  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1376  *
1377  *      Add a driver to one of two lists: either a list of drivers that
1378  *      are notified about clock rate changes (once before and once after
1379  *      the transition), or a list of drivers that are notified about
1380  *      changes in cpufreq policy.
1381  *
1382  *      This function may sleep, and has the same return conditions as
1383  *      blocking_notifier_chain_register.
1384  */
1385 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1386 {
1387         int ret;
1388
1389         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1390
1391         switch (list) {
1392         case CPUFREQ_TRANSITION_NOTIFIER:
1393                 ret = srcu_notifier_chain_register(
1394                                 &cpufreq_transition_notifier_list, nb);
1395                 break;
1396         case CPUFREQ_POLICY_NOTIFIER:
1397                 ret = blocking_notifier_chain_register(
1398                                 &cpufreq_policy_notifier_list, nb);
1399                 break;
1400         default:
1401                 ret = -EINVAL;
1402         }
1403
1404         return ret;
1405 }
1406 EXPORT_SYMBOL(cpufreq_register_notifier);
1407
1408
1409 /**
1410  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1411  *      @nb: notifier block to be unregistered
1412  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1413  *
1414  *      Remove a driver from the CPU frequency notifier list.
1415  *
1416  *      This function may sleep, and has the same return conditions as
1417  *      blocking_notifier_chain_unregister.
1418  */
1419 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1420 {
1421         int ret;
1422
1423         switch (list) {
1424         case CPUFREQ_TRANSITION_NOTIFIER:
1425                 ret = srcu_notifier_chain_unregister(
1426                                 &cpufreq_transition_notifier_list, nb);
1427                 break;
1428         case CPUFREQ_POLICY_NOTIFIER:
1429                 ret = blocking_notifier_chain_unregister(
1430                                 &cpufreq_policy_notifier_list, nb);
1431                 break;
1432         default:
1433                 ret = -EINVAL;
1434         }
1435
1436         return ret;
1437 }
1438 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1439
1440
1441 /*********************************************************************
1442  *                              GOVERNORS                            *
1443  *********************************************************************/
1444
1445
1446 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1447                             unsigned int target_freq,
1448                             unsigned int relation)
1449 {
1450         int retval = -EINVAL;
1451
1452         dprintk("target for CPU %u: %u kHz, relation %u\n", policy->cpu,
1453                 target_freq, relation);
1454         if (cpu_online(policy->cpu) && cpufreq_driver->target)
1455                 retval = cpufreq_driver->target(policy, target_freq, relation);
1456
1457         return retval;
1458 }
1459 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1460
1461 int cpufreq_driver_target(struct cpufreq_policy *policy,
1462                           unsigned int target_freq,
1463                           unsigned int relation)
1464 {
1465         int ret = -EINVAL;
1466
1467         policy = cpufreq_cpu_get(policy->cpu);
1468         if (!policy)
1469                 goto no_policy;
1470
1471         if (unlikely(lock_policy_rwsem_write(policy->cpu)))
1472                 goto fail;
1473
1474         ret = __cpufreq_driver_target(policy, target_freq, relation);
1475
1476         unlock_policy_rwsem_write(policy->cpu);
1477
1478 fail:
1479         cpufreq_cpu_put(policy);
1480 no_policy:
1481         return ret;
1482 }
1483 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1484
1485 int __cpufreq_driver_getavg(struct cpufreq_policy *policy, unsigned int cpu)
1486 {
1487         int ret = 0;
1488
1489         policy = cpufreq_cpu_get(policy->cpu);
1490         if (!policy)
1491                 return -EINVAL;
1492
1493         if (cpu_online(cpu) && cpufreq_driver->getavg)
1494                 ret = cpufreq_driver->getavg(policy, cpu);
1495
1496         cpufreq_cpu_put(policy);
1497         return ret;
1498 }
1499 EXPORT_SYMBOL_GPL(__cpufreq_driver_getavg);
1500
1501 /*
1502  * when "event" is CPUFREQ_GOV_LIMITS
1503  */
1504
1505 static int __cpufreq_governor(struct cpufreq_policy *policy,
1506                                         unsigned int event)
1507 {
1508         int ret;
1509
1510         /* Only must be defined when default governor is known to have latency
1511            restrictions, like e.g. conservative or ondemand.
1512            That this is the case is already ensured in Kconfig
1513         */
1514 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1515         struct cpufreq_governor *gov = &cpufreq_gov_performance;
1516 #else
1517         struct cpufreq_governor *gov = NULL;
1518 #endif
1519
1520         if (policy->governor->max_transition_latency &&
1521             policy->cpuinfo.transition_latency >
1522             policy->governor->max_transition_latency) {
1523                 if (!gov)
1524                         return -EINVAL;
1525                 else {
1526                         printk(KERN_WARNING "%s governor failed, too long"
1527                                " transition latency of HW, fallback"
1528                                " to %s governor\n",
1529                                policy->governor->name,
1530                                gov->name);
1531                         policy->governor = gov;
1532                 }
1533         }
1534
1535         if (!try_module_get(policy->governor->owner))
1536                 return -EINVAL;
1537
1538         dprintk("__cpufreq_governor for CPU %u, event %u\n",
1539                                                 policy->cpu, event);
1540         ret = policy->governor->governor(policy, event);
1541
1542         /* we keep one module reference alive for
1543                         each CPU governed by this CPU */
1544         if ((event != CPUFREQ_GOV_START) || ret)
1545                 module_put(policy->governor->owner);
1546         if ((event == CPUFREQ_GOV_STOP) && !ret)
1547                 module_put(policy->governor->owner);
1548
1549         return ret;
1550 }
1551
1552
1553 int cpufreq_register_governor(struct cpufreq_governor *governor)
1554 {
1555         int err;
1556
1557         if (!governor)
1558                 return -EINVAL;
1559
1560         mutex_lock(&cpufreq_governor_mutex);
1561
1562         err = -EBUSY;
1563         if (__find_governor(governor->name) == NULL) {
1564                 err = 0;
1565                 list_add(&governor->governor_list, &cpufreq_governor_list);
1566         }
1567
1568         mutex_unlock(&cpufreq_governor_mutex);
1569         return err;
1570 }
1571 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
1572
1573
1574 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
1575 {
1576         if (!governor)
1577                 return;
1578
1579         mutex_lock(&cpufreq_governor_mutex);
1580         list_del(&governor->governor_list);
1581         mutex_unlock(&cpufreq_governor_mutex);
1582         return;
1583 }
1584 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
1585
1586
1587
1588 /*********************************************************************
1589  *                          POLICY INTERFACE                         *
1590  *********************************************************************/
1591
1592 /**
1593  * cpufreq_get_policy - get the current cpufreq_policy
1594  * @policy: struct cpufreq_policy into which the current cpufreq_policy
1595  *      is written
1596  *
1597  * Reads the current cpufreq policy.
1598  */
1599 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
1600 {
1601         struct cpufreq_policy *cpu_policy;
1602         if (!policy)
1603                 return -EINVAL;
1604
1605         cpu_policy = cpufreq_cpu_get(cpu);
1606         if (!cpu_policy)
1607                 return -EINVAL;
1608
1609         memcpy(policy, cpu_policy, sizeof(struct cpufreq_policy));
1610
1611         cpufreq_cpu_put(cpu_policy);
1612         return 0;
1613 }
1614 EXPORT_SYMBOL(cpufreq_get_policy);
1615
1616
1617 /*
1618  * data   : current policy.
1619  * policy : policy to be set.
1620  */
1621 static int __cpufreq_set_policy(struct cpufreq_policy *data,
1622                                 struct cpufreq_policy *policy)
1623 {
1624         int ret = 0;
1625
1626         cpufreq_debug_disable_ratelimit();
1627         dprintk("setting new policy for CPU %u: %u - %u kHz\n", policy->cpu,
1628                 policy->min, policy->max);
1629
1630         memcpy(&policy->cpuinfo, &data->cpuinfo,
1631                                 sizeof(struct cpufreq_cpuinfo));
1632
1633         if (policy->min > data->max || policy->max < data->min) {
1634                 ret = -EINVAL;
1635                 goto error_out;
1636         }
1637
1638         /* verify the cpu speed can be set within this limit */
1639         ret = cpufreq_driver->verify(policy);
1640         if (ret)
1641                 goto error_out;
1642
1643         /* adjust if necessary - all reasons */
1644         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1645                         CPUFREQ_ADJUST, policy);
1646
1647         /* adjust if necessary - hardware incompatibility*/
1648         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1649                         CPUFREQ_INCOMPATIBLE, policy);
1650
1651         /* verify the cpu speed can be set within this limit,
1652            which might be different to the first one */
1653         ret = cpufreq_driver->verify(policy);
1654         if (ret)
1655                 goto error_out;
1656
1657         /* notification of the new policy */
1658         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1659                         CPUFREQ_NOTIFY, policy);
1660
1661         data->min = policy->min;
1662         data->max = policy->max;
1663
1664         dprintk("new min and max freqs are %u - %u kHz\n",
1665                                         data->min, data->max);
1666
1667         if (cpufreq_driver->setpolicy) {
1668                 data->policy = policy->policy;
1669                 dprintk("setting range\n");
1670                 ret = cpufreq_driver->setpolicy(policy);
1671         } else {
1672                 if (policy->governor != data->governor) {
1673                         /* save old, working values */
1674                         struct cpufreq_governor *old_gov = data->governor;
1675
1676                         dprintk("governor switch\n");
1677
1678                         /* end old governor */
1679                         if (data->governor)
1680                                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1681
1682                         /* start new governor */
1683                         data->governor = policy->governor;
1684                         if (__cpufreq_governor(data, CPUFREQ_GOV_START)) {
1685                                 /* new governor failed, so re-start old one */
1686                                 dprintk("starting governor %s failed\n",
1687                                                         data->governor->name);
1688                                 if (old_gov) {
1689                                         data->governor = old_gov;
1690                                         __cpufreq_governor(data,
1691                                                            CPUFREQ_GOV_START);
1692                                 }
1693                                 ret = -EINVAL;
1694                                 goto error_out;
1695                         }
1696                         /* might be a policy change, too, so fall through */
1697                 }
1698                 dprintk("governor: change or update limits\n");
1699                 __cpufreq_governor(data, CPUFREQ_GOV_LIMITS);
1700         }
1701
1702 error_out:
1703         cpufreq_debug_enable_ratelimit();
1704         return ret;
1705 }
1706
1707 /**
1708  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
1709  *      @cpu: CPU which shall be re-evaluated
1710  *
1711  *      Usefull for policy notifiers which have different necessities
1712  *      at different times.
1713  */
1714 int cpufreq_update_policy(unsigned int cpu)
1715 {
1716         struct cpufreq_policy *data = cpufreq_cpu_get(cpu);
1717         struct cpufreq_policy policy;
1718         int ret;
1719
1720         if (!data) {
1721                 ret = -ENODEV;
1722                 goto no_policy;
1723         }
1724
1725         if (unlikely(lock_policy_rwsem_write(cpu))) {
1726                 ret = -EINVAL;
1727                 goto fail;
1728         }
1729
1730         dprintk("updating policy for CPU %u\n", cpu);
1731         memcpy(&policy, data, sizeof(struct cpufreq_policy));
1732         policy.min = data->user_policy.min;
1733         policy.max = data->user_policy.max;
1734         policy.policy = data->user_policy.policy;
1735         policy.governor = data->user_policy.governor;
1736
1737         /* BIOS might change freq behind our back
1738           -> ask driver for current freq and notify governors about a change */
1739         if (cpufreq_driver->get) {
1740                 policy.cur = cpufreq_driver->get(cpu);
1741                 if (!data->cur) {
1742                         dprintk("Driver did not initialize current freq");
1743                         data->cur = policy.cur;
1744                 } else {
1745                         if (data->cur != policy.cur)
1746                                 cpufreq_out_of_sync(cpu, data->cur,
1747                                                                 policy.cur);
1748                 }
1749         }
1750
1751         ret = __cpufreq_set_policy(data, &policy);
1752
1753         unlock_policy_rwsem_write(cpu);
1754
1755 fail:
1756         cpufreq_cpu_put(data);
1757 no_policy:
1758         return ret;
1759 }
1760 EXPORT_SYMBOL(cpufreq_update_policy);
1761
1762 static int __cpuinit cpufreq_cpu_callback(struct notifier_block *nfb,
1763                                         unsigned long action, void *hcpu)
1764 {
1765         unsigned int cpu = (unsigned long)hcpu;
1766         struct sys_device *sys_dev;
1767
1768         sys_dev = get_cpu_sysdev(cpu);
1769         if (sys_dev) {
1770                 switch (action) {
1771                 case CPU_ONLINE:
1772                 case CPU_ONLINE_FROZEN:
1773                         cpufreq_add_dev(sys_dev);
1774                         break;
1775                 case CPU_DOWN_PREPARE:
1776                 case CPU_DOWN_PREPARE_FROZEN:
1777                         if (unlikely(lock_policy_rwsem_write(cpu)))
1778                                 BUG();
1779
1780                         __cpufreq_remove_dev(sys_dev);
1781                         break;
1782                 case CPU_DOWN_FAILED:
1783                 case CPU_DOWN_FAILED_FROZEN:
1784                         cpufreq_add_dev(sys_dev);
1785                         break;
1786                 }
1787         }
1788         return NOTIFY_OK;
1789 }
1790
1791 static struct notifier_block __refdata cpufreq_cpu_notifier =
1792 {
1793     .notifier_call = cpufreq_cpu_callback,
1794 };
1795
1796 /*********************************************************************
1797  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
1798  *********************************************************************/
1799
1800 /**
1801  * cpufreq_register_driver - register a CPU Frequency driver
1802  * @driver_data: A struct cpufreq_driver containing the values#
1803  * submitted by the CPU Frequency driver.
1804  *
1805  *   Registers a CPU Frequency driver to this core code. This code
1806  * returns zero on success, -EBUSY when another driver got here first
1807  * (and isn't unregistered in the meantime).
1808  *
1809  */
1810 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
1811 {
1812         unsigned long flags;
1813         int ret;
1814
1815         if (!driver_data || !driver_data->verify || !driver_data->init ||
1816             ((!driver_data->setpolicy) && (!driver_data->target)))
1817                 return -EINVAL;
1818
1819         dprintk("trying to register driver %s\n", driver_data->name);
1820
1821         if (driver_data->setpolicy)
1822                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
1823
1824         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1825         if (cpufreq_driver) {
1826                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1827                 return -EBUSY;
1828         }
1829         cpufreq_driver = driver_data;
1830         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1831
1832         ret = sysdev_driver_register(&cpu_sysdev_class,
1833                                         &cpufreq_sysdev_driver);
1834
1835         if ((!ret) && !(cpufreq_driver->flags & CPUFREQ_STICKY)) {
1836                 int i;
1837                 ret = -ENODEV;
1838
1839                 /* check for at least one working CPU */
1840                 for (i = 0; i < nr_cpu_ids; i++)
1841                         if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
1842                                 ret = 0;
1843                                 break;
1844                         }
1845
1846                 /* if all ->init() calls failed, unregister */
1847                 if (ret) {
1848                         dprintk("no CPU initialized for driver %s\n",
1849                                                         driver_data->name);
1850                         sysdev_driver_unregister(&cpu_sysdev_class,
1851                                                 &cpufreq_sysdev_driver);
1852
1853                         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1854                         cpufreq_driver = NULL;
1855                         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1856                 }
1857         }
1858
1859         if (!ret) {
1860                 register_hotcpu_notifier(&cpufreq_cpu_notifier);
1861                 dprintk("driver %s up and running\n", driver_data->name);
1862                 cpufreq_debug_enable_ratelimit();
1863         }
1864
1865         return ret;
1866 }
1867 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
1868
1869
1870 /**
1871  * cpufreq_unregister_driver - unregister the current CPUFreq driver
1872  *
1873  *    Unregister the current CPUFreq driver. Only call this if you have
1874  * the right to do so, i.e. if you have succeeded in initialising before!
1875  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
1876  * currently not initialised.
1877  */
1878 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
1879 {
1880         unsigned long flags;
1881
1882         cpufreq_debug_disable_ratelimit();
1883
1884         if (!cpufreq_driver || (driver != cpufreq_driver)) {
1885                 cpufreq_debug_enable_ratelimit();
1886                 return -EINVAL;
1887         }
1888
1889         dprintk("unregistering driver %s\n", driver->name);
1890
1891         sysdev_driver_unregister(&cpu_sysdev_class, &cpufreq_sysdev_driver);
1892         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
1893
1894         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1895         cpufreq_driver = NULL;
1896         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1897
1898         return 0;
1899 }
1900 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
1901
1902 static int __init cpufreq_core_init(void)
1903 {
1904         int cpu;
1905
1906         for_each_possible_cpu(cpu) {
1907                 per_cpu(policy_cpu, cpu) = -1;
1908                 init_rwsem(&per_cpu(cpu_policy_rwsem, cpu));
1909         }
1910         return 0;
1911 }
1912
1913 core_initcall(cpufreq_core_init);