drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *            (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
   7  *
   8  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   9  *      Added handling for CPU hotplug
  10  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  11  *      Fix handling for CPU hotplug -- affected CPUs
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License version 2 as
  15  * published by the Free Software Foundation.
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <asm/cputime.h>
  21 #include <linux/kernel.h>
  22 #include <linux/kernel_stat.h>
  23 #include <linux/module.h>
  24 #include <linux/init.h>
  25 #include <linux/notifier.h>
  26 #include <linux/cpufreq.h>
  27 #include <linux/delay.h>
  28 #include <linux/interrupt.h>
  29 #include <linux/spinlock.h>
  30 #include <linux/tick.h>
  31 #include <linux/device.h>
  32 #include <linux/slab.h>
  33 #include <linux/cpu.h>
  34 #include <linux/completion.h>
  35 #include <linux/mutex.h>
  36 #include <linux/syscore_ops.h>
  37
  38 #include <trace/events/power.h>
  39
  40 /**
  41  * The "cpufreq driver" - the arch- or hardware-dependent low
  42  * level driver of CPUFreq support, and its spinlock. This lock
  43  * also protects the cpufreq_cpu_data array.
  44  */
  45 static struct cpufreq_driver *cpufreq_driver;
  46 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
  47 static DEFINE_RWLOCK(cpufreq_driver_lock);
  48 static DEFINE_MUTEX(cpufreq_governor_lock);
  49
  50 #ifdef CONFIG_HOTPLUG_CPU
  51 /* This one keeps track of the previously set governor of a removed CPU */
  52 static DEFINE_PER_CPU(char[CPUFREQ_NAME_LEN], cpufreq_cpu_governor);
  53 #endif
  54
  55 /*
  56  * cpu_policy_rwsem is a per CPU reader-writer semaphore designed to cure
  57  * all cpufreq/hotplug/workqueue/etc related lock issues.
  58  *
  59  * The rules for this semaphore:
  60  * - Any routine that wants to read from the policy structure will
  61  *   do a down_read on this semaphore.
  62  * - Any routine that will write to the policy structure and/or may take away
  63  *   the policy altogether (eg. CPU hotplug), will hold this lock in write
  64  *   mode before doing so.
  65  *
  66  * Additional rules:
  67  * - Governor routines that can be called in cpufreq hotplug path should not
  68  *   take this sem as top level hotplug notifier handler takes this.
  69  * - Lock should not be held across
  70  *     __cpufreq_governor(data, CPUFREQ_GOV_STOP);
  71  */
  72 static DEFINE_PER_CPU(int, cpufreq_policy_cpu);
  73 static DEFINE_PER_CPU(struct rw_semaphore, cpu_policy_rwsem);
  74
  75 #define lock_policy_rwsem(mode, cpu)                                    \
  76 static int lock_policy_rwsem_##mode(int cpu)                            \
  77 {                                                                       \
  78         int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu);              \
  79         BUG_ON(policy_cpu == -1);                                       \
  80         down_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));            \
  81                                                                         \
  82         return 0;                                                       \
  83 }
  84
  85 lock_policy_rwsem(read, cpu);
  86 lock_policy_rwsem(write, cpu);
  87
  88 #define unlock_policy_rwsem(mode, cpu)                                  \
  89 static void unlock_policy_rwsem_##mode(int cpu)                         \
  90 {                                                                       \
  91         int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu);              \
  92         BUG_ON(policy_cpu == -1);                                       \
  93         up_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));              \
  94 }
  95
  96 unlock_policy_rwsem(read, cpu);
  97 unlock_policy_rwsem(write, cpu);
  98
  99 /* internal prototypes */
 100 static int __cpufreq_governor(struct cpufreq_policy *policy,
 101                 unsigned int event);
 102 static unsigned int __cpufreq_get(unsigned int cpu);
 103 static void handle_update(struct work_struct *work);
 104
 105 /**
 106  * Two notifier lists: the "policy" list is involved in the
 107  * validation process for a new CPU frequency policy; the
 108  * "transition" list for kernel code that needs to handle
 109  * changes to devices when the CPU clock speed changes.
 110  * The mutex locks both lists.
 111  */
 112 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
 113 static struct srcu_notifier_head cpufreq_transition_notifier_list;
 114
 115 static bool init_cpufreq_transition_notifier_list_called;
 116 static int __init init_cpufreq_transition_notifier_list(void)
 117 {
 118         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
 119         init_cpufreq_transition_notifier_list_called = true;
 120         return 0;
 121 }
 122 pure_initcall(init_cpufreq_transition_notifier_list);
 123
 124 static int off __read_mostly;
 125 static int cpufreq_disabled(void)
 126 {
 127         return off;
 128 }
 129 void disable_cpufreq(void)
 130 {
 131         off = 1;
 132 }
 133 static LIST_HEAD(cpufreq_governor_list);
 134 static DEFINE_MUTEX(cpufreq_governor_mutex);
 135
 136 bool have_governor_per_policy(void)
 137 {
 138         return cpufreq_driver->have_governor_per_policy;
 139 }
 140 EXPORT_SYMBOL_GPL(have_governor_per_policy);
 141
 142 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
 143 {
 144         if (have_governor_per_policy())
 145                 return &policy->kobj;
 146         else
 147                 return cpufreq_global_kobject;
 148 }
 149 EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
 150
 151 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
 152 {
 153         u64 idle_time;
 154         u64 cur_wall_time;
 155         u64 busy_time;
 156
 157         cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());
 158
 159         busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
 160         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
 161         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
 162         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
 163         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
 164         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];
 165
 166         idle_time = cur_wall_time - busy_time;
 167         if (wall)
 168                 *wall = cputime_to_usecs(cur_wall_time);
 169
 170         return cputime_to_usecs(idle_time);
 171 }
 172
 173 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
 174 {
 175         u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
 176
 177         if (idle_time == -1ULL)
 178                 return get_cpu_idle_time_jiffy(cpu, wall);
 179         else if (!io_busy)
 180                 idle_time += get_cpu_iowait_time_us(cpu, wall);
 181
 182         return idle_time;
 183 }
 184 EXPORT_SYMBOL_GPL(get_cpu_idle_time);
 185
 186 static struct cpufreq_policy *__cpufreq_cpu_get(unsigned int cpu, bool sysfs)
 187 {
 188         struct cpufreq_policy *data;
 189         unsigned long flags;
 190
 191         if (cpu >= nr_cpu_ids)
 192                 goto err_out;
 193
 194         /* get the cpufreq driver */
 195         read_lock_irqsave(&cpufreq_driver_lock, flags);
 196
 197         if (!cpufreq_driver)
 198                 goto err_out_unlock;
 199
 200         if (!try_module_get(cpufreq_driver->owner))
 201                 goto err_out_unlock;
 202
 203         /* get the CPU */
 204         data = per_cpu(cpufreq_cpu_data, cpu);
 205
 206         if (!data)
 207                 goto err_out_put_module;
 208
 209         if (!sysfs && !kobject_get(&data->kobj))
 210                 goto err_out_put_module;
 211
 212         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 213         return data;
 214
 215 err_out_put_module:
 216         module_put(cpufreq_driver->owner);
 217 err_out_unlock:
 218         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 219 err_out:
 220         return NULL;
 221 }
 222
 223 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 224 {
 225         if (cpufreq_disabled())
 226                 return NULL;
 227
 228         return __cpufreq_cpu_get(cpu, false);
 229 }
 230 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 231
 232 static struct cpufreq_policy *cpufreq_cpu_get_sysfs(unsigned int cpu)
 233 {
 234         return __cpufreq_cpu_get(cpu, true);
 235 }
 236
 237 static void __cpufreq_cpu_put(struct cpufreq_policy *data, bool sysfs)
 238 {
 239         if (!sysfs)
 240                 kobject_put(&data->kobj);
 241         module_put(cpufreq_driver->owner);
 242 }
 243
 244 void cpufreq_cpu_put(struct cpufreq_policy *data)
 245 {
 246         if (cpufreq_disabled())
 247                 return;
 248
 249         __cpufreq_cpu_put(data, false);
 250 }
 251 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 252
 253 static void cpufreq_cpu_put_sysfs(struct cpufreq_policy *data)
 254 {
 255         __cpufreq_cpu_put(data, true);
 256 }
 257
 258 /*********************************************************************
 259  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 260  *********************************************************************/
 261
 262 /**
 263  * adjust_jiffies - adjust the system "loops_per_jiffy"
 264  *
 265  * This function alters the system "loops_per_jiffy" for the clock
 266  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 267  * systems as each CPU might be scaled differently. So, use the arch
 268  * per-CPU loops_per_jiffy value wherever possible.
 269  */
 270 #ifndef CONFIG_SMP
 271 static unsigned long l_p_j_ref;
 272 static unsigned int l_p_j_ref_freq;
 273
 274 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 275 {
 276         if (ci->flags & CPUFREQ_CONST_LOOPS)
 277                 return;
 278
 279         if (!l_p_j_ref_freq) {
 280                 l_p_j_ref = loops_per_jiffy;
 281                 l_p_j_ref_freq = ci->old;
 282                 pr_debug("saving %lu as reference value for loops_per_jiffy; "
 283                         "freq is %u kHz\n", l_p_j_ref, l_p_j_ref_freq);
 284         }
 285         if ((val == CPUFREQ_POSTCHANGE && ci->old != ci->new) ||
 286             (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) {
 287                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 288                                                                 ci->new);
 289                 pr_debug("scaling loops_per_jiffy to %lu "
 290                         "for frequency %u kHz\n", loops_per_jiffy, ci->new);
 291         }
 292 }
 293 #else
 294 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 295 {
 296         return;
 297 }
 298 #endif
 299
 300 static void __cpufreq_notify_transition(struct cpufreq_policy *policy,
 301                 struct cpufreq_freqs *freqs, unsigned int state)
 302 {
 303         BUG_ON(irqs_disabled());
 304
 305         if (cpufreq_disabled())
 306                 return;
 307
 308         freqs->flags = cpufreq_driver->flags;
 309         pr_debug("notification %u of frequency transition to %u kHz\n",
 310                 state, freqs->new);
 311
 312         switch (state) {
 313
 314         case CPUFREQ_PRECHANGE:
 315                 if (WARN(policy->transition_ongoing ==
 316                                         cpumask_weight(policy->cpus),
 317                                 "In middle of another frequency transition\n"))
 318                         return;
 319
 320                 policy->transition_ongoing++;
 321
 322                 /* detect if the driver reported a value as "old frequency"
 323                  * which is not equal to what the cpufreq core thinks is
 324                  * "old frequency".
 325                  */
 326                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 327                         if ((policy) && (policy->cpu == freqs->cpu) &&
 328                             (policy->cur) && (policy->cur != freqs->old)) {
 329                                 pr_debug("Warning: CPU frequency is"
 330                                         " %u, cpufreq assumed %u kHz.\n",
 331                                         freqs->old, policy->cur);
 332                                 freqs->old = policy->cur;
 333                         }
 334                 }
 335                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 336                                 CPUFREQ_PRECHANGE, freqs);
 337                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 338                 break;
 339
 340         case CPUFREQ_POSTCHANGE:
 341                 if (WARN(!policy->transition_ongoing,
 342                                 "No frequency transition in progress\n"))
 343                         return;
 344
 345                 policy->transition_ongoing--;
 346
 347                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 348                 pr_debug("FREQ: %lu - CPU: %lu", (unsigned long)freqs->new,
 349                         (unsigned long)freqs->cpu);
 350                 trace_cpu_frequency(freqs->new, freqs->cpu);
 351                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 352                                 CPUFREQ_POSTCHANGE, freqs);
 353                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 354                         policy->cur = freqs->new;
 355                 break;
 356         }
 357 }
 358
 359 /**
 360  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 361  * on frequency transition.
 362  *
 363  * This function calls the transition notifiers and the "adjust_jiffies"
 364  * function. It is called twice on all CPU frequency changes that have
 365  * external effects.
 366  */
 367 void cpufreq_notify_transition(struct cpufreq_policy *policy,
 368                 struct cpufreq_freqs *freqs, unsigned int state)
 369 {
 370         for_each_cpu(freqs->cpu, policy->cpus)
 371                 __cpufreq_notify_transition(policy, freqs, state);
 372 }
 373 EXPORT_SYMBOL_GPL(cpufreq_notify_transition);
 374
 375
 376 /*********************************************************************
 377  *                          SYSFS INTERFACE                          *
 378  *********************************************************************/
 379
 380 static struct cpufreq_governor *__find_governor(const char *str_governor)
 381 {
 382         struct cpufreq_governor *t;
 383
 384         list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 385                 if (!strnicmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 386                         return t;
 387
 388         return NULL;
 389 }
 390
 391 /**
 392  * cpufreq_parse_governor - parse a governor string
 393  */
 394 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 395                                 struct cpufreq_governor **governor)
 396 {
 397         int err = -EINVAL;
 398
 399         if (!cpufreq_driver)
 400                 goto out;
 401
 402         if (cpufreq_driver->setpolicy) {
 403                 if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 404                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 405                         err = 0;
 406                 } else if (!strnicmp(str_governor, "powersave",
 407                                                 CPUFREQ_NAME_LEN)) {
 408                         *policy = CPUFREQ_POLICY_POWERSAVE;
 409                         err = 0;
 410                 }
 411         } else if (cpufreq_driver->target) {
 412                 struct cpufreq_governor *t;
 413
 414                 mutex_lock(&cpufreq_governor_mutex);
 415
 416                 t = __find_governor(str_governor);
 417
 418                 if (t == NULL) {
 419                         int ret;
 420
 421                         mutex_unlock(&cpufreq_governor_mutex);
 422                         ret = request_module("cpufreq_%s", str_governor);
 423                         mutex_lock(&cpufreq_governor_mutex);
 424
 425                         if (ret == 0)
 426                                 t = __find_governor(str_governor);
 427                 }
 428
 429                 if (t != NULL) {
 430                         *governor = t;
 431                         err = 0;
 432                 }
 433
 434                 mutex_unlock(&cpufreq_governor_mutex);
 435         }
 436 out:
 437         return err;
 438 }
 439
 440 /**
 441  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 442  * print out cpufreq information
 443  *
 444  * Write out information from cpufreq_driver->policy[cpu]; object must be
 445  * "unsigned int".
 446  */
 447
 448 #define show_one(file_name, object)                     \
 449 static ssize_t show_##file_name                         \
 450 (struct cpufreq_policy *policy, char *buf)              \
 451 {                                                       \
 452         return sprintf(buf, "%u\n", policy->object);    \
 453 }
 454
 455 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 456 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 457 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 458 show_one(scaling_min_freq, min);
 459 show_one(scaling_max_freq, max);
 460 show_one(scaling_cur_freq, cur);
 461
 462 static int __cpufreq_set_policy(struct cpufreq_policy *data,
 463                                 struct cpufreq_policy *policy);
 464
 465 /**
 466  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 467  */
 468 #define store_one(file_name, object)                    \
 469 static ssize_t store_##file_name                                        \
 470 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 471 {                                                                       \
 472         unsigned int ret;                                               \
 473         struct cpufreq_policy new_policy;                               \
 474                                                                         \
 475         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 476         if (ret)                                                        \
 477                 return -EINVAL;                                         \
 478                                                                         \
 479         ret = sscanf(buf, "%u", &new_policy.object);                    \
 480         if (ret != 1)                                                   \
 481                 return -EINVAL;                                         \
 482                                                                         \
 483         ret = __cpufreq_set_policy(policy, &new_policy);                \
 484         policy->user_policy.object = policy->object;                    \
 485                                                                         \
 486         return ret ? ret : count;                                       \
 487 }
 488
 489 store_one(scaling_min_freq, min);
 490 store_one(scaling_max_freq, max);
 491
 492 /**
 493  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 494  */
 495 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 496                                         char *buf)
 497 {
 498         unsigned int cur_freq = __cpufreq_get(policy->cpu);
 499         if (!cur_freq)
 500                 return sprintf(buf, "<unknown>");
 501         return sprintf(buf, "%u\n", cur_freq);
 502 }
 503
 504 /**
 505  * show_scaling_governor - show the current policy for the specified CPU
 506  */
 507 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 508 {
 509         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 510                 return sprintf(buf, "powersave\n");
 511         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 512                 return sprintf(buf, "performance\n");
 513         else if (policy->governor)
 514                 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
 515                                 policy->governor->name);
 516         return -EINVAL;
 517 }
 518
 519 /**
 520  * store_scaling_governor - store policy for the specified CPU
 521  */
 522 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 523                                         const char *buf, size_t count)
 524 {
 525         unsigned int ret;
 526         char    str_governor[16];
 527         struct cpufreq_policy new_policy;
 528
 529         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 530         if (ret)
 531                 return ret;
 532
 533         ret = sscanf(buf, "%15s", str_governor);
 534         if (ret != 1)
 535                 return -EINVAL;
 536
 537         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 538                                                 &new_policy.governor))
 539                 return -EINVAL;
 540
 541         /*
 542          * Do not use cpufreq_set_policy here or the user_policy.max
 543          * will be wrongly overridden
 544          */
 545         ret = __cpufreq_set_policy(policy, &new_policy);
 546
 547         policy->user_policy.policy = policy->policy;
 548         policy->user_policy.governor = policy->governor;
 549
 550         if (ret)
 551                 return ret;
 552         else
 553                 return count;
 554 }
 555
 556 /**
 557  * show_scaling_driver - show the cpufreq driver currently loaded
 558  */
 559 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 560 {
 561         return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
 562 }
 563
 564 /**
 565  * show_scaling_available_governors - show the available CPUfreq governors
 566  */
 567 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 568                                                 char *buf)
 569 {
 570         ssize_t i = 0;
 571         struct cpufreq_governor *t;
 572
 573         if (!cpufreq_driver->target) {
 574                 i += sprintf(buf, "performance powersave");
 575                 goto out;
 576         }
 577
 578         list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 579                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 580                     - (CPUFREQ_NAME_LEN + 2)))
 581                         goto out;
 582                 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
 583         }
 584 out:
 585         i += sprintf(&buf[i], "\n");
 586         return i;
 587 }
 588
 589 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
 590 {
 591         ssize_t i = 0;
 592         unsigned int cpu;
 593
 594         for_each_cpu(cpu, mask) {
 595                 if (i)
 596                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 597                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 598                 if (i >= (PAGE_SIZE - 5))
 599                         break;
 600         }
 601         i += sprintf(&buf[i], "\n");
 602         return i;
 603 }
 604 EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
 605
 606 /**
 607  * show_related_cpus - show the CPUs affected by each transition even if
 608  * hw coordination is in use
 609  */
 610 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 611 {
 612         return cpufreq_show_cpus(policy->related_cpus, buf);
 613 }
 614
 615 /**
 616  * show_affected_cpus - show the CPUs affected by each transition
 617  */
 618 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 619 {
 620         return cpufreq_show_cpus(policy->cpus, buf);
 621 }
 622
 623 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 624                                         const char *buf, size_t count)
 625 {
 626         unsigned int freq = 0;
 627         unsigned int ret;
 628
 629         if (!policy->governor || !policy->governor->store_setspeed)
 630                 return -EINVAL;
 631
 632         ret = sscanf(buf, "%u", &freq);
 633         if (ret != 1)
 634                 return -EINVAL;
 635
 636         policy->governor->store_setspeed(policy, freq);
 637
 638         return count;
 639 }
 640
 641 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 642 {
 643         if (!policy->governor || !policy->governor->show_setspeed)
 644                 return sprintf(buf, "<unsupported>\n");
 645
 646         return policy->governor->show_setspeed(policy, buf);
 647 }
 648
 649 /**
 650  * show_bios_limit - show the current cpufreq HW/BIOS limitation
 651  */
 652 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
 653 {
 654         unsigned int limit;
 655         int ret;
 656         if (cpufreq_driver->bios_limit) {
 657                 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
 658                 if (!ret)
 659                         return sprintf(buf, "%u\n", limit);
 660         }
 661         return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
 662 }
 663
 664 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
 665 cpufreq_freq_attr_ro(cpuinfo_min_freq);
 666 cpufreq_freq_attr_ro(cpuinfo_max_freq);
 667 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
 668 cpufreq_freq_attr_ro(scaling_available_governors);
 669 cpufreq_freq_attr_ro(scaling_driver);
 670 cpufreq_freq_attr_ro(scaling_cur_freq);
 671 cpufreq_freq_attr_ro(bios_limit);
 672 cpufreq_freq_attr_ro(related_cpus);
 673 cpufreq_freq_attr_ro(affected_cpus);
 674 cpufreq_freq_attr_rw(scaling_min_freq);
 675 cpufreq_freq_attr_rw(scaling_max_freq);
 676 cpufreq_freq_attr_rw(scaling_governor);
 677 cpufreq_freq_attr_rw(scaling_setspeed);
 678
 679 static struct attribute *default_attrs[] = {
 680         &cpuinfo_min_freq.attr,
 681         &cpuinfo_max_freq.attr,
 682         &cpuinfo_transition_latency.attr,
 683         &scaling_min_freq.attr,
 684         &scaling_max_freq.attr,
 685         &affected_cpus.attr,
 686         &related_cpus.attr,
 687         &scaling_governor.attr,
 688         &scaling_driver.attr,
 689         &scaling_available_governors.attr,
 690         &scaling_setspeed.attr,
 691         NULL
 692 };
 693
 694 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 695 #define to_attr(a) container_of(a, struct freq_attr, attr)
 696
 697 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 698 {
 699         struct cpufreq_policy *policy = to_policy(kobj);
 700         struct freq_attr *fattr = to_attr(attr);
 701         ssize_t ret = -EINVAL;
 702         policy = cpufreq_cpu_get_sysfs(policy->cpu);
 703         if (!policy)
 704                 goto no_policy;
 705
 706         if (lock_policy_rwsem_read(policy->cpu) < 0)
 707                 goto fail;
 708
 709         if (fattr->show)
 710                 ret = fattr->show(policy, buf);
 711         else
 712                 ret = -EIO;
 713
 714         unlock_policy_rwsem_read(policy->cpu);
 715 fail:
 716         cpufreq_cpu_put_sysfs(policy);
 717 no_policy:
 718         return ret;
 719 }
 720
 721 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 722                      const char *buf, size_t count)
 723 {
 724         struct cpufreq_policy *policy = to_policy(kobj);
 725         struct freq_attr *fattr = to_attr(attr);
 726         ssize_t ret = -EINVAL;
 727         policy = cpufreq_cpu_get_sysfs(policy->cpu);
 728         if (!policy)
 729                 goto no_policy;
 730
 731         if (lock_policy_rwsem_write(policy->cpu) < 0)
 732                 goto fail;
 733
 734         if (fattr->store)
 735                 ret = fattr->store(policy, buf, count);
 736         else
 737                 ret = -EIO;
 738
 739         unlock_policy_rwsem_write(policy->cpu);
 740 fail:
 741         cpufreq_cpu_put_sysfs(policy);
 742 no_policy:
 743         return ret;
 744 }
 745
 746 static void cpufreq_sysfs_release(struct kobject *kobj)
 747 {
 748         struct cpufreq_policy *policy = to_policy(kobj);
 749         pr_debug("last reference is dropped\n");
 750         complete(&policy->kobj_unregister);
 751 }
 752
 753 static const struct sysfs_ops sysfs_ops = {
 754         .show   = show,
 755         .store  = store,
 756 };
 757
 758 static struct kobj_type ktype_cpufreq = {
 759         .sysfs_ops      = &sysfs_ops,
 760         .default_attrs  = default_attrs,
 761         .release        = cpufreq_sysfs_release,
 762 };
 763
 764 struct kobject *cpufreq_global_kobject;
 765 EXPORT_SYMBOL(cpufreq_global_kobject);
 766
 767 static int cpufreq_global_kobject_usage;
 768
 769 int cpufreq_get_global_kobject(void)
 770 {
 771         if (!cpufreq_global_kobject_usage++)
 772                 return kobject_add(cpufreq_global_kobject,
 773                                 &cpu_subsys.dev_root->kobj, "%s", "cpufreq");
 774
 775         return 0;
 776 }
 777 EXPORT_SYMBOL(cpufreq_get_global_kobject);
 778
 779 void cpufreq_put_global_kobject(void)
 780 {
 781         if (!--cpufreq_global_kobject_usage)
 782                 kobject_del(cpufreq_global_kobject);
 783 }
 784 EXPORT_SYMBOL(cpufreq_put_global_kobject);
 785
 786 int cpufreq_sysfs_create_file(const struct attribute *attr)
 787 {
 788         int ret = cpufreq_get_global_kobject();
 789
 790         if (!ret) {
 791                 ret = sysfs_create_file(cpufreq_global_kobject, attr);
 792                 if (ret)
 793                         cpufreq_put_global_kobject();
 794         }
 795
 796         return ret;
 797 }
 798 EXPORT_SYMBOL(cpufreq_sysfs_create_file);
 799
 800 void cpufreq_sysfs_remove_file(const struct attribute *attr)
 801 {
 802         sysfs_remove_file(cpufreq_global_kobject, attr);
 803         cpufreq_put_global_kobject();
 804 }
 805 EXPORT_SYMBOL(cpufreq_sysfs_remove_file);
 806
 807 /* symlink affected CPUs */
 808 static int cpufreq_add_dev_symlink(unsigned int cpu,
 809                                    struct cpufreq_policy *policy)
 810 {
 811         unsigned int j;
 812         int ret = 0;
 813
 814         for_each_cpu(j, policy->cpus) {
 815                 struct cpufreq_policy *managed_policy;
 816                 struct device *cpu_dev;
 817
 818                 if (j == cpu)
 819                         continue;
 820
 821                 pr_debug("CPU %u already managed, adding link\n", j);
 822                 managed_policy = cpufreq_cpu_get(cpu);
 823                 cpu_dev = get_cpu_device(j);
 824                 ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
 825                                         "cpufreq");
 826                 if (ret) {
 827                         cpufreq_cpu_put(managed_policy);
 828                         return ret;
 829                 }
 830         }
 831         return ret;
 832 }
 833
 834 static int cpufreq_add_dev_interface(unsigned int cpu,
 835                                      struct cpufreq_policy *policy,
 836                                      struct device *dev)
 837 {
 838         struct cpufreq_policy new_policy;
 839         struct freq_attr **drv_attr;
 840         unsigned long flags;
 841         int ret = 0;
 842         unsigned int j;
 843
 844         /* prepare interface data */
 845         ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
 846                                    &dev->kobj, "cpufreq");
 847         if (ret)
 848                 return ret;
 849
 850         /* set up files for this cpu device */
 851         drv_attr = cpufreq_driver->attr;
 852         while ((drv_attr) && (*drv_attr)) {
 853                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 854                 if (ret)
 855                         goto err_out_kobj_put;
 856                 drv_attr++;
 857         }
 858         if (cpufreq_driver->get) {
 859                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 860                 if (ret)
 861                         goto err_out_kobj_put;
 862         }
 863         if (cpufreq_driver->target) {
 864                 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 865                 if (ret)
 866                         goto err_out_kobj_put;
 867         }
 868         if (cpufreq_driver->bios_limit) {
 869                 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
 870                 if (ret)
 871                         goto err_out_kobj_put;
 872         }
 873
 874         write_lock_irqsave(&cpufreq_driver_lock, flags);
 875         for_each_cpu(j, policy->cpus) {
 876                 per_cpu(cpufreq_cpu_data, j) = policy;
 877                 per_cpu(cpufreq_policy_cpu, j) = policy->cpu;
 878         }
 879         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
 880
 881         ret = cpufreq_add_dev_symlink(cpu, policy);
 882         if (ret)
 883                 goto err_out_kobj_put;
 884
 885         memcpy(&new_policy, policy, sizeof(struct cpufreq_policy));
 886         /* assure that the starting sequence is run in __cpufreq_set_policy */
 887         policy->governor = NULL;
 888
 889         /* set default policy */
 890         ret = __cpufreq_set_policy(policy, &new_policy);
 891         policy->user_policy.policy = policy->policy;
 892         policy->user_policy.governor = policy->governor;
 893
 894         if (ret) {
 895                 pr_debug("setting policy failed\n");
 896                 if (cpufreq_driver->exit)
 897                         cpufreq_driver->exit(policy);
 898         }
 899         return ret;
 900
 901 err_out_kobj_put:
 902         kobject_put(&policy->kobj);
 903         wait_for_completion(&policy->kobj_unregister);
 904         return ret;
 905 }
 906
 907 #ifdef CONFIG_HOTPLUG_CPU
 908 static int cpufreq_add_policy_cpu(unsigned int cpu, unsigned int sibling,
 909                                   struct device *dev)
 910 {
 911         struct cpufreq_policy *policy;
 912         int ret = 0, has_target = !!cpufreq_driver->target;
 913         unsigned long flags;
 914
 915         policy = cpufreq_cpu_get(sibling);
 916         WARN_ON(!policy);
 917
 918         if (has_target)
 919                 __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
 920
 921         lock_policy_rwsem_write(sibling);
 922
 923         write_lock_irqsave(&cpufreq_driver_lock, flags);
 924
 925         cpumask_set_cpu(cpu, policy->cpus);
 926         per_cpu(cpufreq_policy_cpu, cpu) = policy->cpu;
 927         per_cpu(cpufreq_cpu_data, cpu) = policy;
 928         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
 929
 930         unlock_policy_rwsem_write(sibling);
 931
 932         if (has_target) {
 933                 __cpufreq_governor(policy, CPUFREQ_GOV_START);
 934                 __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
 935         }
 936
 937         ret = sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq");
 938         if (ret) {
 939                 cpufreq_cpu_put(policy);
 940                 return ret;
 941         }
 942
 943         return 0;
 944 }
 945 #endif
 946
 947 /**
 948  * cpufreq_add_dev - add a CPU device
 949  *
 950  * Adds the cpufreq interface for a CPU device.
 951  *
 952  * The Oracle says: try running cpufreq registration/unregistration concurrently
 953  * with with cpu hotplugging and all hell will break loose. Tried to clean this
 954  * mess up, but more thorough testing is needed. - Mathieu
 955  */
 956 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
 957 {
 958         unsigned int j, cpu = dev->id;
 959         int ret = -ENOMEM;
 960         struct cpufreq_policy *policy;
 961         unsigned long flags;
 962 #ifdef CONFIG_HOTPLUG_CPU
 963         struct cpufreq_governor *gov;
 964         int sibling;
 965 #endif
 966
 967         if (cpu_is_offline(cpu))
 968                 return 0;
 969
 970         pr_debug("adding CPU %u\n", cpu);
 971
 972 #ifdef CONFIG_SMP
 973         /* check whether a different CPU already registered this
 974          * CPU because it is in the same boat. */
 975         policy = cpufreq_cpu_get(cpu);
 976         if (unlikely(policy)) {
 977                 cpufreq_cpu_put(policy);
 978                 return 0;
 979         }
 980
 981 #ifdef CONFIG_HOTPLUG_CPU
 982         /* Check if this cpu was hot-unplugged earlier and has siblings */
 983         read_lock_irqsave(&cpufreq_driver_lock, flags);
 984         for_each_online_cpu(sibling) {
 985                 struct cpufreq_policy *cp = per_cpu(cpufreq_cpu_data, sibling);
 986                 if (cp && cpumask_test_cpu(cpu, cp->related_cpus)) {
 987                         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 988                         return cpufreq_add_policy_cpu(cpu, sibling, dev);
 989                 }
 990         }
 991         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 992 #endif
 993 #endif
 994
 995         if (!try_module_get(cpufreq_driver->owner)) {
 996                 ret = -EINVAL;
 997                 goto module_out;
 998         }
 999
1000         policy = kzalloc(sizeof(struct cpufreq_policy), GFP_KERNEL);
1001         if (!policy)
1002                 goto nomem_out;
1003
1004         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
1005                 goto err_free_policy;
1006
1007         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
1008                 goto err_free_cpumask;
1009
1010         policy->cpu = cpu;
1011         policy->governor = CPUFREQ_DEFAULT_GOVERNOR;
1012         cpumask_copy(policy->cpus, cpumask_of(cpu));
1013
1014         /* Initially set CPU itself as the policy_cpu */
1015         per_cpu(cpufreq_policy_cpu, cpu) = cpu;
1016
1017         init_completion(&policy->kobj_unregister);
1018         INIT_WORK(&policy->update, handle_update);
1019
1020         /* call driver. From then on the cpufreq must be able
1021          * to accept all calls to ->verify and ->setpolicy for this CPU
1022          */
1023         ret = cpufreq_driver->init(policy);
1024         if (ret) {
1025                 pr_debug("initialization failed\n");
1026                 goto err_set_policy_cpu;
1027         }
1028
1029         /* related cpus should atleast have policy->cpus */
1030         cpumask_or(policy->related_cpus, policy->related_cpus, policy->cpus);
1031
1032         /*
1033          * affected cpus must always be the one, which are online. We aren't
1034          * managing offline cpus here.
1035          */
1036         cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1037
1038         policy->user_policy.min = policy->min;
1039         policy->user_policy.max = policy->max;
1040
1041         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1042                                      CPUFREQ_START, policy);
1043
1044 #ifdef CONFIG_HOTPLUG_CPU
1045         gov = __find_governor(per_cpu(cpufreq_cpu_governor, cpu));
1046         if (gov) {
1047                 policy->governor = gov;
1048                 pr_debug("Restoring governor %s for cpu %d\n",
1049                        policy->governor->name, cpu);
1050         }
1051 #endif
1052
1053         ret = cpufreq_add_dev_interface(cpu, policy, dev);
1054         if (ret)
1055                 goto err_out_unregister;
1056
1057         kobject_uevent(&policy->kobj, KOBJ_ADD);
1058         module_put(cpufreq_driver->owner);
1059         pr_debug("initialization complete\n");
1060
1061         return 0;
1062
1063 err_out_unregister:
1064         write_lock_irqsave(&cpufreq_driver_lock, flags);
1065         for_each_cpu(j, policy->cpus)
1066                 per_cpu(cpufreq_cpu_data, j) = NULL;
1067         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1068
1069         kobject_put(&policy->kobj);
1070         wait_for_completion(&policy->kobj_unregister);
1071
1072 err_set_policy_cpu:
1073         per_cpu(cpufreq_policy_cpu, cpu) = -1;
1074         free_cpumask_var(policy->related_cpus);
1075 err_free_cpumask:
1076         free_cpumask_var(policy->cpus);
1077 err_free_policy:
1078         kfree(policy);
1079 nomem_out:
1080         module_put(cpufreq_driver->owner);
1081 module_out:
1082         return ret;
1083 }
1084
1085 static void update_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
1086 {
1087         int j;
1088
1089         policy->last_cpu = policy->cpu;
1090         policy->cpu = cpu;
1091
1092         for_each_cpu(j, policy->cpus)
1093                 per_cpu(cpufreq_policy_cpu, j) = cpu;
1094
1095 #ifdef CONFIG_CPU_FREQ_TABLE
1096         cpufreq_frequency_table_update_policy_cpu(policy);
1097 #endif
1098         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1099                         CPUFREQ_UPDATE_POLICY_CPU, policy);
1100 }
1101
1102 /**
1103  * __cpufreq_remove_dev - remove a CPU device
1104  *
1105  * Removes the cpufreq interface for a CPU device.
1106  * Caller should already have policy_rwsem in write mode for this CPU.
1107  * This routine frees the rwsem before returning.
1108  */
1109 static int __cpufreq_remove_dev(struct device *dev,
1110                 struct subsys_interface *sif)
1111 {
1112         unsigned int cpu = dev->id, ret, cpus;
1113         unsigned long flags;
1114         struct cpufreq_policy *data;
1115         struct kobject *kobj;
1116         struct completion *cmp;
1117         struct device *cpu_dev;
1118
1119         pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1120
1121         write_lock_irqsave(&cpufreq_driver_lock, flags);
1122
1123         data = per_cpu(cpufreq_cpu_data, cpu);
1124         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1125
1126         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1127
1128         if (!data) {
1129                 pr_debug("%s: No cpu_data found\n", __func__);
1130                 return -EINVAL;
1131         }
1132
1133         if (cpufreq_driver->target)
1134                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1135
1136 #ifdef CONFIG_HOTPLUG_CPU
1137         if (!cpufreq_driver->setpolicy)
1138                 strncpy(per_cpu(cpufreq_cpu_governor, cpu),
1139                         data->governor->name, CPUFREQ_NAME_LEN);
1140 #endif
1141
1142         WARN_ON(lock_policy_rwsem_write(cpu));
1143         cpus = cpumask_weight(data->cpus);
1144
1145         if (cpus > 1)
1146                 cpumask_clear_cpu(cpu, data->cpus);
1147         unlock_policy_rwsem_write(cpu);
1148
1149         if (cpu != data->cpu) {
1150                 sysfs_remove_link(&dev->kobj, "cpufreq");
1151         } else if (cpus > 1) {
1152                 /* first sibling now owns the new sysfs dir */
1153                 cpu_dev = get_cpu_device(cpumask_first(data->cpus));
1154                 sysfs_remove_link(&cpu_dev->kobj, "cpufreq");
1155                 ret = kobject_move(&data->kobj, &cpu_dev->kobj);
1156                 if (ret) {
1157                         pr_err("%s: Failed to move kobj: %d", __func__, ret);
1158
1159                         WARN_ON(lock_policy_rwsem_write(cpu));
1160                         cpumask_set_cpu(cpu, data->cpus);
1161
1162                         write_lock_irqsave(&cpufreq_driver_lock, flags);
1163                         per_cpu(cpufreq_cpu_data, cpu) = data;
1164                         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1165
1166                         unlock_policy_rwsem_write(cpu);
1167
1168                         ret = sysfs_create_link(&cpu_dev->kobj, &data->kobj,
1169                                         "cpufreq");
1170                         return -EINVAL;
1171                 }
1172
1173                 WARN_ON(lock_policy_rwsem_write(cpu));
1174                 update_policy_cpu(data, cpu_dev->id);
1175                 unlock_policy_rwsem_write(cpu);
1176                 pr_debug("%s: policy Kobject moved to cpu: %d from: %d\n",
1177                                 __func__, cpu_dev->id, cpu);
1178         }
1179
1180         /* If cpu is last user of policy, free policy */
1181         if (cpus == 1) {
1182                 if (cpufreq_driver->target)
1183                         __cpufreq_governor(data, CPUFREQ_GOV_POLICY_EXIT);
1184
1185                 lock_policy_rwsem_read(cpu);
1186                 kobj = &data->kobj;
1187                 cmp = &data->kobj_unregister;
1188                 unlock_policy_rwsem_read(cpu);
1189                 kobject_put(kobj);
1190
1191                 /* we need to make sure that the underlying kobj is actually
1192                  * not referenced anymore by anybody before we proceed with
1193                  * unloading.
1194                  */
1195                 pr_debug("waiting for dropping of refcount\n");
1196                 wait_for_completion(cmp);
1197                 pr_debug("wait complete\n");
1198
1199                 if (cpufreq_driver->exit)
1200                         cpufreq_driver->exit(data);
1201
1202                 free_cpumask_var(data->related_cpus);
1203                 free_cpumask_var(data->cpus);
1204                 kfree(data);
1205         } else {
1206                 pr_debug("%s: removing link, cpu: %d\n", __func__, cpu);
1207                 cpufreq_cpu_put(data);
1208                 if (cpufreq_driver->target) {
1209                         __cpufreq_governor(data, CPUFREQ_GOV_START);
1210                         __cpufreq_governor(data, CPUFREQ_GOV_LIMITS);
1211                 }
1212         }
1213
1214         per_cpu(cpufreq_policy_cpu, cpu) = -1;
1215         return 0;
1216 }
1217
1218 static int cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1219 {
1220         unsigned int cpu = dev->id;
1221         int retval;
1222
1223         if (cpu_is_offline(cpu))
1224                 return 0;
1225
1226         retval = __cpufreq_remove_dev(dev, sif);
1227         return retval;
1228 }
1229
1230 static void handle_update(struct work_struct *work)
1231 {
1232         struct cpufreq_policy *policy =
1233                 container_of(work, struct cpufreq_policy, update);
1234         unsigned int cpu = policy->cpu;
1235         pr_debug("handle_update for cpu %u called\n", cpu);
1236         cpufreq_update_policy(cpu);
1237 }
1238
1239 /**
1240  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're
1241  *      in deep trouble.
1242  *      @cpu: cpu number
1243  *      @old_freq: CPU frequency the kernel thinks the CPU runs at
1244  *      @new_freq: CPU frequency the CPU actually runs at
1245  *
1246  *      We adjust to current frequency first, and need to clean up later.
1247  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1248  */
1249 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1250                                 unsigned int new_freq)
1251 {
1252         struct cpufreq_policy *policy;
1253         struct cpufreq_freqs freqs;
1254         unsigned long flags;
1255
1256         pr_debug("Warning: CPU frequency out of sync: cpufreq and timing "
1257                "core thinks of %u, is %u kHz.\n", old_freq, new_freq);
1258
1259         freqs.old = old_freq;
1260         freqs.new = new_freq;
1261
1262         read_lock_irqsave(&cpufreq_driver_lock, flags);
1263         policy = per_cpu(cpufreq_cpu_data, cpu);
1264         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1265
1266         cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE);
1267         cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE);
1268 }
1269
1270 /**
1271  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1272  * @cpu: CPU number
1273  *
1274  * This is the last known freq, without actually getting it from the driver.
1275  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1276  */
1277 unsigned int cpufreq_quick_get(unsigned int cpu)
1278 {
1279         struct cpufreq_policy *policy;
1280         unsigned int ret_freq = 0;
1281
1282         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
1283                 return cpufreq_driver->get(cpu);
1284
1285         policy = cpufreq_cpu_get(cpu);
1286         if (policy) {
1287                 ret_freq = policy->cur;
1288                 cpufreq_cpu_put(policy);
1289         }
1290
1291         return ret_freq;
1292 }
1293 EXPORT_SYMBOL(cpufreq_quick_get);
1294
1295 /**
1296  * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1297  * @cpu: CPU number
1298  *
1299  * Just return the max possible frequency for a given CPU.
1300  */
1301 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1302 {
1303         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1304         unsigned int ret_freq = 0;
1305
1306         if (policy) {
1307                 ret_freq = policy->max;
1308                 cpufreq_cpu_put(policy);
1309         }
1310
1311         return ret_freq;
1312 }
1313 EXPORT_SYMBOL(cpufreq_quick_get_max);
1314
1315 static unsigned int __cpufreq_get(unsigned int cpu)
1316 {
1317         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1318         unsigned int ret_freq = 0;
1319
1320         if (!cpufreq_driver->get)
1321                 return ret_freq;
1322
1323         ret_freq = cpufreq_driver->get(cpu);
1324
1325         if (ret_freq && policy->cur &&
1326                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1327                 /* verify no discrepancy between actual and
1328                                         saved value exists */
1329                 if (unlikely(ret_freq != policy->cur)) {
1330                         cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1331                         schedule_work(&policy->update);
1332                 }
1333         }
1334
1335         return ret_freq;
1336 }
1337
1338 /**
1339  * cpufreq_get - get the current CPU frequency (in kHz)
1340  * @cpu: CPU number
1341  *
1342  * Get the CPU current (static) CPU frequency
1343  */
1344 unsigned int cpufreq_get(unsigned int cpu)
1345 {
1346         unsigned int ret_freq = 0;
1347         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1348
1349         if (!policy)
1350                 goto out;
1351
1352         if (unlikely(lock_policy_rwsem_read(cpu)))
1353                 goto out_policy;
1354
1355         ret_freq = __cpufreq_get(cpu);
1356
1357         unlock_policy_rwsem_read(cpu);
1358
1359 out_policy:
1360         cpufreq_cpu_put(policy);
1361 out:
1362         return ret_freq;
1363 }
1364 EXPORT_SYMBOL(cpufreq_get);
1365
1366 static struct subsys_interface cpufreq_interface = {
1367         .name           = "cpufreq",
1368         .subsys         = &cpu_subsys,
1369         .add_dev        = cpufreq_add_dev,
1370         .remove_dev     = cpufreq_remove_dev,
1371 };
1372
1373 /**
1374  * cpufreq_bp_suspend - Prepare the boot CPU for system suspend.
1375  *
1376  * This function is only executed for the boot processor.  The other CPUs
1377  * have been put offline by means of CPU hotplug.
1378  */
1379 static int cpufreq_bp_suspend(void)
1380 {
1381         int ret = 0;
1382
1383         int cpu = smp_processor_id();
1384         struct cpufreq_policy *cpu_policy;
1385
1386         pr_debug("suspending cpu %u\n", cpu);
1387
1388         /* If there's no policy for the boot CPU, we have nothing to do. */
1389         cpu_policy = cpufreq_cpu_get(cpu);
1390         if (!cpu_policy)
1391                 return 0;
1392
1393         if (cpufreq_driver->suspend) {
1394                 ret = cpufreq_driver->suspend(cpu_policy);
1395                 if (ret)
1396                         printk(KERN_ERR "cpufreq: suspend failed in ->suspend "
1397                                         "step on CPU %u\n", cpu_policy->cpu);
1398         }
1399
1400         cpufreq_cpu_put(cpu_policy);
1401         return ret;
1402 }
1403
1404 /**
1405  * cpufreq_bp_resume - Restore proper frequency handling of the boot CPU.
1406  *
1407  *      1.) resume CPUfreq hardware support (cpufreq_driver->resume())
1408  *      2.) schedule call cpufreq_update_policy() ASAP as interrupts are
1409  *          restored. It will verify that the current freq is in sync with
1410  *          what we believe it to be. This is a bit later than when it
1411  *          should be, but nonethteless it's better than calling
1412  *          cpufreq_driver->get() here which might re-enable interrupts...
1413  *
1414  * This function is only executed for the boot CPU.  The other CPUs have not
1415  * been turned on yet.
1416  */
1417 static void cpufreq_bp_resume(void)
1418 {
1419         int ret = 0;
1420
1421         int cpu = smp_processor_id();
1422         struct cpufreq_policy *cpu_policy;
1423
1424         pr_debug("resuming cpu %u\n", cpu);
1425
1426         /* If there's no policy for the boot CPU, we have nothing to do. */
1427         cpu_policy = cpufreq_cpu_get(cpu);
1428         if (!cpu_policy)
1429                 return;
1430
1431         if (cpufreq_driver->resume) {
1432                 ret = cpufreq_driver->resume(cpu_policy);
1433                 if (ret) {
1434                         printk(KERN_ERR "cpufreq: resume failed in ->resume "
1435                                         "step on CPU %u\n", cpu_policy->cpu);
1436                         goto fail;
1437                 }
1438         }
1439
1440         schedule_work(&cpu_policy->update);
1441
1442 fail:
1443         cpufreq_cpu_put(cpu_policy);
1444 }
1445
1446 static struct syscore_ops cpufreq_syscore_ops = {
1447         .suspend        = cpufreq_bp_suspend,
1448         .resume         = cpufreq_bp_resume,
1449 };
1450
1451 /**
1452  *      cpufreq_get_current_driver - return current driver's name
1453  *
1454  *      Return the name string of the currently loaded cpufreq driver
1455  *      or NULL, if none.
1456  */
1457 const char *cpufreq_get_current_driver(void)
1458 {
1459         if (cpufreq_driver)
1460                 return cpufreq_driver->name;
1461
1462         return NULL;
1463 }
1464 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
1465
1466 /*********************************************************************
1467  *                     NOTIFIER LISTS INTERFACE                      *
1468  *********************************************************************/
1469
1470 /**
1471  *      cpufreq_register_notifier - register a driver with cpufreq
1472  *      @nb: notifier function to register
1473  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1474  *
1475  *      Add a driver to one of two lists: either a list of drivers that
1476  *      are notified about clock rate changes (once before and once after
1477  *      the transition), or a list of drivers that are notified about
1478  *      changes in cpufreq policy.
1479  *
1480  *      This function may sleep, and has the same return conditions as
1481  *      blocking_notifier_chain_register.
1482  */
1483 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1484 {
1485         int ret;
1486
1487         if (cpufreq_disabled())
1488                 return -EINVAL;
1489
1490         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1491
1492         switch (list) {
1493         case CPUFREQ_TRANSITION_NOTIFIER:
1494                 ret = srcu_notifier_chain_register(
1495                                 &cpufreq_transition_notifier_list, nb);
1496                 break;
1497         case CPUFREQ_POLICY_NOTIFIER:
1498                 ret = blocking_notifier_chain_register(
1499                                 &cpufreq_policy_notifier_list, nb);
1500                 break;
1501         default:
1502                 ret = -EINVAL;
1503         }
1504
1505         return ret;
1506 }
1507 EXPORT_SYMBOL(cpufreq_register_notifier);
1508
1509 /**
1510  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1511  *      @nb: notifier block to be unregistered
1512  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1513  *
1514  *      Remove a driver from the CPU frequency notifier list.
1515  *
1516  *      This function may sleep, and has the same return conditions as
1517  *      blocking_notifier_chain_unregister.
1518  */
1519 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1520 {
1521         int ret;
1522
1523         if (cpufreq_disabled())
1524                 return -EINVAL;
1525
1526         switch (list) {
1527         case CPUFREQ_TRANSITION_NOTIFIER:
1528                 ret = srcu_notifier_chain_unregister(
1529                                 &cpufreq_transition_notifier_list, nb);
1530                 break;
1531         case CPUFREQ_POLICY_NOTIFIER:
1532                 ret = blocking_notifier_chain_unregister(
1533                                 &cpufreq_policy_notifier_list, nb);
1534                 break;
1535         default:
1536                 ret = -EINVAL;
1537         }
1538
1539         return ret;
1540 }
1541 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1542
1543
1544 /*********************************************************************
1545  *                              GOVERNORS                            *
1546  *********************************************************************/
1547
1548 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1549                             unsigned int target_freq,
1550                             unsigned int relation)
1551 {
1552         int retval = -EINVAL;
1553         unsigned int old_target_freq = target_freq;
1554
1555         if (cpufreq_disabled())
1556                 return -ENODEV;
1557         if (policy->transition_ongoing)
1558                 return -EBUSY;
1559
1560         /* Make sure that target_freq is within supported range */
1561         if (target_freq > policy->max)
1562                 target_freq = policy->max;
1563         if (target_freq < policy->min)
1564                 target_freq = policy->min;
1565
1566         pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
1567                         policy->cpu, target_freq, relation, old_target_freq);
1568
1569         if (target_freq == policy->cur)
1570                 return 0;
1571
1572         if (cpufreq_driver->target)
1573                 retval = cpufreq_driver->target(policy, target_freq, relation);
1574
1575         return retval;
1576 }
1577 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1578
1579 int cpufreq_driver_target(struct cpufreq_policy *policy,
1580                           unsigned int target_freq,
1581                           unsigned int relation)
1582 {
1583         int ret = -EINVAL;
1584
1585         if (unlikely(lock_policy_rwsem_write(policy->cpu)))
1586                 goto fail;
1587
1588         ret = __cpufreq_driver_target(policy, target_freq, relation);
1589
1590         unlock_policy_rwsem_write(policy->cpu);
1591
1592 fail:
1593         return ret;
1594 }
1595 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1596
1597 int __cpufreq_driver_getavg(struct cpufreq_policy *policy, unsigned int cpu)
1598 {
1599         if (cpufreq_disabled())
1600                 return 0;
1601
1602         if (!cpufreq_driver->getavg)
1603                 return 0;
1604
1605         return cpufreq_driver->getavg(policy, cpu);
1606 }
1607 EXPORT_SYMBOL_GPL(__cpufreq_driver_getavg);
1608
1609 /*
1610  * when "event" is CPUFREQ_GOV_LIMITS
1611  */
1612
1613 static int __cpufreq_governor(struct cpufreq_policy *policy,
1614                                         unsigned int event)
1615 {
1616         int ret;
1617
1618         /* Only must be defined when default governor is known to have latency
1619            restrictions, like e.g. conservative or ondemand.
1620            That this is the case is already ensured in Kconfig
1621         */
1622 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1623         struct cpufreq_governor *gov = &cpufreq_gov_performance;
1624 #else
1625         struct cpufreq_governor *gov = NULL;
1626 #endif
1627
1628         if (policy->governor->max_transition_latency &&
1629             policy->cpuinfo.transition_latency >
1630             policy->governor->max_transition_latency) {
1631                 if (!gov)
1632                         return -EINVAL;
1633                 else {
1634                         printk(KERN_WARNING "%s governor failed, too long"
1635                                " transition latency of HW, fallback"
1636                                " to %s governor\n",
1637                                policy->governor->name,
1638                                gov->name);
1639                         policy->governor = gov;
1640                 }
1641         }
1642
1643         if (!try_module_get(policy->governor->owner))
1644                 return -EINVAL;
1645
1646         pr_debug("__cpufreq_governor for CPU %u, event %u\n",
1647                                                 policy->cpu, event);
1648
1649         mutex_lock(&cpufreq_governor_lock);
1650         if ((!policy->governor_enabled && (event == CPUFREQ_GOV_STOP)) ||
1651             (policy->governor_enabled && (event == CPUFREQ_GOV_START))) {
1652                 mutex_unlock(&cpufreq_governor_lock);
1653                 return -EBUSY;
1654         }
1655
1656         if (event == CPUFREQ_GOV_STOP)
1657                 policy->governor_enabled = false;
1658         else if (event == CPUFREQ_GOV_START)
1659                 policy->governor_enabled = true;
1660
1661         mutex_unlock(&cpufreq_governor_lock);
1662
1663         ret = policy->governor->governor(policy, event);
1664
1665         if (!ret) {
1666                 if (event == CPUFREQ_GOV_POLICY_INIT)
1667                         policy->governor->initialized++;
1668                 else if (event == CPUFREQ_GOV_POLICY_EXIT)
1669                         policy->governor->initialized--;
1670         } else {
1671                 /* Restore original values */
1672                 mutex_lock(&cpufreq_governor_lock);
1673                 if (event == CPUFREQ_GOV_STOP)
1674                         policy->governor_enabled = true;
1675                 else if (event == CPUFREQ_GOV_START)
1676                         policy->governor_enabled = false;
1677                 mutex_unlock(&cpufreq_governor_lock);
1678         }
1679
1680         /* we keep one module reference alive for
1681                         each CPU governed by this CPU */
1682         if ((event != CPUFREQ_GOV_START) || ret)
1683                 module_put(policy->governor->owner);
1684         if ((event == CPUFREQ_GOV_STOP) && !ret)
1685                 module_put(policy->governor->owner);
1686
1687         return ret;
1688 }
1689
1690 int cpufreq_register_governor(struct cpufreq_governor *governor)
1691 {
1692         int err;
1693
1694         if (!governor)
1695                 return -EINVAL;
1696
1697         if (cpufreq_disabled())
1698                 return -ENODEV;
1699
1700         mutex_lock(&cpufreq_governor_mutex);
1701
1702         governor->initialized = 0;
1703         err = -EBUSY;
1704         if (__find_governor(governor->name) == NULL) {
1705                 err = 0;
1706                 list_add(&governor->governor_list, &cpufreq_governor_list);
1707         }
1708
1709         mutex_unlock(&cpufreq_governor_mutex);
1710         return err;
1711 }
1712 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
1713
1714 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
1715 {
1716 #ifdef CONFIG_HOTPLUG_CPU
1717         int cpu;
1718 #endif
1719
1720         if (!governor)
1721                 return;
1722
1723         if (cpufreq_disabled())
1724                 return;
1725
1726 #ifdef CONFIG_HOTPLUG_CPU
1727         for_each_present_cpu(cpu) {
1728                 if (cpu_online(cpu))
1729                         continue;
1730                 if (!strcmp(per_cpu(cpufreq_cpu_governor, cpu), governor->name))
1731                         strcpy(per_cpu(cpufreq_cpu_governor, cpu), "\0");
1732         }
1733 #endif
1734
1735         mutex_lock(&cpufreq_governor_mutex);
1736         list_del(&governor->governor_list);
1737         mutex_unlock(&cpufreq_governor_mutex);
1738         return;
1739 }
1740 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
1741
1742
1743 /*********************************************************************
1744  *                          POLICY INTERFACE                         *
1745  *********************************************************************/
1746
1747 /**
1748  * cpufreq_get_policy - get the current cpufreq_policy
1749  * @policy: struct cpufreq_policy into which the current cpufreq_policy
1750  *      is written
1751  *
1752  * Reads the current cpufreq policy.
1753  */
1754 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
1755 {
1756         struct cpufreq_policy *cpu_policy;
1757         if (!policy)
1758                 return -EINVAL;
1759
1760         cpu_policy = cpufreq_cpu_get(cpu);
1761         if (!cpu_policy)
1762                 return -EINVAL;
1763
1764         memcpy(policy, cpu_policy, sizeof(struct cpufreq_policy));
1765
1766         cpufreq_cpu_put(cpu_policy);
1767         return 0;
1768 }
1769 EXPORT_SYMBOL(cpufreq_get_policy);
1770
1771 /*
1772  * data   : current policy.
1773  * policy : policy to be set.
1774  */
1775 static int __cpufreq_set_policy(struct cpufreq_policy *data,
1776                                 struct cpufreq_policy *policy)
1777 {
1778         int ret = 0, failed = 1;
1779
1780         pr_debug("setting new policy for CPU %u: %u - %u kHz\n", policy->cpu,
1781                 policy->min, policy->max);
1782
1783         memcpy(&policy->cpuinfo, &data->cpuinfo,
1784                                 sizeof(struct cpufreq_cpuinfo));
1785
1786         if (policy->min > data->max || policy->max < data->min) {
1787                 ret = -EINVAL;
1788                 goto error_out;
1789         }
1790
1791         /* verify the cpu speed can be set within this limit */
1792         ret = cpufreq_driver->verify(policy);
1793         if (ret)
1794                 goto error_out;
1795
1796         /* adjust if necessary - all reasons */
1797         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1798                         CPUFREQ_ADJUST, policy);
1799
1800         /* adjust if necessary - hardware incompatibility*/
1801         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1802                         CPUFREQ_INCOMPATIBLE, policy);
1803
1804         /*
1805          * verify the cpu speed can be set within this limit, which might be
1806          * different to the first one
1807          */
1808         ret = cpufreq_driver->verify(policy);
1809         if (ret)
1810                 goto error_out;
1811
1812         /* notification of the new policy */
1813         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1814                         CPUFREQ_NOTIFY, policy);
1815
1816         data->min = policy->min;
1817         data->max = policy->max;
1818
1819         pr_debug("new min and max freqs are %u - %u kHz\n",
1820                                         data->min, data->max);
1821
1822         if (cpufreq_driver->setpolicy) {
1823                 data->policy = policy->policy;
1824                 pr_debug("setting range\n");
1825                 ret = cpufreq_driver->setpolicy(policy);
1826         } else {
1827                 if (policy->governor != data->governor) {
1828                         /* save old, working values */
1829                         struct cpufreq_governor *old_gov = data->governor;
1830
1831                         pr_debug("governor switch\n");
1832
1833                         /* end old governor */
1834                         if (data->governor) {
1835                                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1836                                 unlock_policy_rwsem_write(policy->cpu);
1837                                 __cpufreq_governor(data,
1838                                                 CPUFREQ_GOV_POLICY_EXIT);
1839                                 lock_policy_rwsem_write(policy->cpu);
1840                         }
1841
1842                         /* start new governor */
1843                         data->governor = policy->governor;
1844                         if (!__cpufreq_governor(data, CPUFREQ_GOV_POLICY_INIT)) {
1845                                 if (!__cpufreq_governor(data, CPUFREQ_GOV_START)) {
1846                                         failed = 0;
1847                                 } else {
1848                                         unlock_policy_rwsem_write(policy->cpu);
1849                                         __cpufreq_governor(data,
1850                                                         CPUFREQ_GOV_POLICY_EXIT);
1851                                         lock_policy_rwsem_write(policy->cpu);
1852                                 }
1853                         }
1854
1855                         if (failed) {
1856                                 /* new governor failed, so re-start old one */
1857                                 pr_debug("starting governor %s failed\n",
1858                                                         data->governor->name);
1859                                 if (old_gov) {
1860                                         data->governor = old_gov;
1861                                         __cpufreq_governor(data,
1862                                                         CPUFREQ_GOV_POLICY_INIT);
1863                                         __cpufreq_governor(data,
1864                                                            CPUFREQ_GOV_START);
1865                                 }
1866                                 ret = -EINVAL;
1867                                 goto error_out;
1868                         }
1869                         /* might be a policy change, too, so fall through */
1870                 }
1871                 pr_debug("governor: change or update limits\n");
1872                 __cpufreq_governor(data, CPUFREQ_GOV_LIMITS);
1873         }
1874
1875 error_out:
1876         return ret;
1877 }
1878
1879 /**
1880  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
1881  *      @cpu: CPU which shall be re-evaluated
1882  *
1883  *      Useful for policy notifiers which have different necessities
1884  *      at different times.
1885  */
1886 int cpufreq_update_policy(unsigned int cpu)
1887 {
1888         struct cpufreq_policy *data = cpufreq_cpu_get(cpu);
1889         struct cpufreq_policy policy;
1890         int ret;
1891
1892         if (!data) {
1893                 ret = -ENODEV;
1894                 goto no_policy;
1895         }
1896
1897         if (unlikely(lock_policy_rwsem_write(cpu))) {
1898                 ret = -EINVAL;
1899                 goto fail;
1900         }
1901
1902         pr_debug("updating policy for CPU %u\n", cpu);
1903         memcpy(&policy, data, sizeof(struct cpufreq_policy));
1904         policy.min = data->user_policy.min;
1905         policy.max = data->user_policy.max;
1906         policy.policy = data->user_policy.policy;
1907         policy.governor = data->user_policy.governor;
1908
1909         /*
1910          * BIOS might change freq behind our back
1911          * -> ask driver for current freq and notify governors about a change
1912          */
1913         if (cpufreq_driver->get) {
1914                 policy.cur = cpufreq_driver->get(cpu);
1915                 if (!data->cur) {
1916                         pr_debug("Driver did not initialize current freq");
1917                         data->cur = policy.cur;
1918                 } else {
1919                         if (data->cur != policy.cur && cpufreq_driver->target)
1920                                 cpufreq_out_of_sync(cpu, data->cur,
1921                                                                 policy.cur);
1922                 }
1923         }
1924
1925         ret = __cpufreq_set_policy(data, &policy);
1926
1927         unlock_policy_rwsem_write(cpu);
1928
1929 fail:
1930         cpufreq_cpu_put(data);
1931 no_policy:
1932         return ret;
1933 }
1934 EXPORT_SYMBOL(cpufreq_update_policy);
1935
1936 static int cpufreq_cpu_callback(struct notifier_block *nfb,
1937                                         unsigned long action, void *hcpu)
1938 {
1939         unsigned int cpu = (unsigned long)hcpu;
1940         struct device *dev;
1941
1942         dev = get_cpu_device(cpu);
1943         if (dev) {
1944                 switch (action) {
1945                 case CPU_ONLINE:
1946                 case CPU_ONLINE_FROZEN:
1947                         cpufreq_add_dev(dev, NULL);
1948                         break;
1949                 case CPU_DOWN_PREPARE:
1950                 case CPU_DOWN_PREPARE_FROZEN:
1951                         __cpufreq_remove_dev(dev, NULL);
1952                         break;
1953                 case CPU_DOWN_FAILED:
1954                 case CPU_DOWN_FAILED_FROZEN:
1955                         cpufreq_add_dev(dev, NULL);
1956                         break;
1957                 }
1958         }
1959         return NOTIFY_OK;
1960 }
1961
1962 static struct notifier_block __refdata cpufreq_cpu_notifier = {
1963         .notifier_call = cpufreq_cpu_callback,
1964 };
1965
1966 /*********************************************************************
1967  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
1968  *********************************************************************/
1969
1970 /**
1971  * cpufreq_register_driver - register a CPU Frequency driver
1972  * @driver_data: A struct cpufreq_driver containing the values#
1973  * submitted by the CPU Frequency driver.
1974  *
1975  * Registers a CPU Frequency driver to this core code. This code
1976  * returns zero on success, -EBUSY when another driver got here first
1977  * (and isn't unregistered in the meantime).
1978  *
1979  */
1980 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
1981 {
1982         unsigned long flags;
1983         int ret;
1984
1985         if (cpufreq_disabled())
1986                 return -ENODEV;
1987
1988         if (!driver_data || !driver_data->verify || !driver_data->init ||
1989             ((!driver_data->setpolicy) && (!driver_data->target)))
1990                 return -EINVAL;
1991
1992         pr_debug("trying to register driver %s\n", driver_data->name);
1993
1994         if (driver_data->setpolicy)
1995                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
1996
1997         write_lock_irqsave(&cpufreq_driver_lock, flags);
1998         if (cpufreq_driver) {
1999                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2000                 return -EBUSY;
2001         }
2002         cpufreq_driver = driver_data;
2003         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2004
2005         ret = subsys_interface_register(&cpufreq_interface);
2006         if (ret)
2007                 goto err_null_driver;
2008
2009         if (!(cpufreq_driver->flags & CPUFREQ_STICKY)) {
2010                 int i;
2011                 ret = -ENODEV;
2012
2013                 /* check for at least one working CPU */
2014                 for (i = 0; i < nr_cpu_ids; i++)
2015                         if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
2016                                 ret = 0;
2017                                 break;
2018                         }
2019
2020                 /* if all ->init() calls failed, unregister */
2021                 if (ret) {
2022                         pr_debug("no CPU initialized for driver %s\n",
2023                                                         driver_data->name);
2024                         goto err_if_unreg;
2025                 }
2026         }
2027
2028         register_hotcpu_notifier(&cpufreq_cpu_notifier);
2029         pr_debug("driver %s up and running\n", driver_data->name);
2030
2031         return 0;
2032 err_if_unreg:
2033         subsys_interface_unregister(&cpufreq_interface);
2034 err_null_driver:
2035         write_lock_irqsave(&cpufreq_driver_lock, flags);
2036         cpufreq_driver = NULL;
2037         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2038         return ret;
2039 }
2040 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
2041
2042 /**
2043  * cpufreq_unregister_driver - unregister the current CPUFreq driver
2044  *
2045  * Unregister the current CPUFreq driver. Only call this if you have
2046  * the right to do so, i.e. if you have succeeded in initialising before!
2047  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
2048  * currently not initialised.
2049  */
2050 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
2051 {
2052         unsigned long flags;
2053
2054         if (!cpufreq_driver || (driver != cpufreq_driver))
2055                 return -EINVAL;
2056
2057         pr_debug("unregistering driver %s\n", driver->name);
2058
2059         subsys_interface_unregister(&cpufreq_interface);
2060         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
2061
2062         write_lock_irqsave(&cpufreq_driver_lock, flags);
2063         cpufreq_driver = NULL;
2064         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2065
2066         return 0;
2067 }
2068 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2069
2070 static int __init cpufreq_core_init(void)
2071 {
2072         int cpu;
2073
2074         if (cpufreq_disabled())
2075                 return -ENODEV;
2076
2077         for_each_possible_cpu(cpu) {
2078                 per_cpu(cpufreq_policy_cpu, cpu) = -1;
2079                 init_rwsem(&per_cpu(cpu_policy_rwsem, cpu));
2080         }
2081
2082         cpufreq_global_kobject = kobject_create();
2083         BUG_ON(!cpufreq_global_kobject);
2084         register_syscore_ops(&cpufreq_syscore_ops);
2085
2086         return 0;
2087 }
2088 core_initcall(cpufreq_core_init);