drivers/soc/tegra/tegra-dvfs.c

   1 /*
   2  * Copyright (C) 2010 Google, Inc.
   3  *
   4  * Author:
   5  *      Colin Cross <ccross@google.com>
   6  *
   7  * Copyright (c) 2014, NVIDIA CORPORATION.  All rights reserved.
   8  *
   9  * This program is free software; you can redistribute it and/or modify it
  10  * under the terms and conditions of the GNU General Public License,
  11  * version 2, as published by the Free Software Foundation.
  12  *
  13  * This program is distributed in the hope it will be useful, but WITHOUT
  14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  16  * more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20  */
  21
  22 #include <linux/kernel.h>
  23 #include <linux/clk.h>
  24 #include <linux/clkdev.h>
  25 #include <linux/clk-provider.h>
  26 #include <linux/debugfs.h>
  27 #include <linux/init.h>
  28 #include <linux/list.h>
  29 #include <linux/list_sort.h>
  30 #include <linux/module.h>
  31 #include <linux/regulator/consumer.h>
  32 #include <linux/suspend.h>
  33 #include <linux/reboot.h>
  34 #include <linux/platform_device.h>
  35 #include <linux/of.h>
  36 #include <linux/pm_opp.h>
  37 #include <linux/cpu.h>
  38
  39 #include <soc/tegra/tegra-dvfs.h>
  40
  41 struct dvfs_rail *tegra_cpu_rail;
  42 struct dvfs_rail *tegra_core_rail;
  43 static struct dvfs_rail *tegra_gpu_rail;
  44
  45 bool core_dvfs_started;
  46
  47 static LIST_HEAD(dvfs_rail_list);
  48 static DEFINE_MUTEX(dvfs_lock);
  49
  50 static int dvfs_rail_update(struct dvfs_rail *rail);
  51
  52 static inline int tegra_dvfs_rail_get_disable_level(struct dvfs_rail *rail)
  53 {
  54         return rail->disable_millivolts ? : rail->nominal_millivolts;
  55 }
  56
  57 static inline int tegra_dvfs_rail_get_suspend_level(struct dvfs_rail *rail)
  58 {
  59         return rail->suspend_millivolts ? : rail->nominal_millivolts;
  60 }
  61
  62 void tegra_dvfs_add_relationships(struct dvfs_relationship *rels, int n)
  63 {
  64         int i;
  65         struct dvfs_relationship *rel;
  66
  67         mutex_lock(&dvfs_lock);
  68
  69         for (i = 0; i < n; i++) {
  70                 rel = &rels[i];
  71                 list_add_tail(&rel->from_node, &rel->to->relationships_from);
  72                 list_add_tail(&rel->to_node, &rel->from->relationships_to);
  73         }
  74
  75         mutex_unlock(&dvfs_lock);
  76 }
  77
  78 int tegra_dvfs_init_rails(struct dvfs_rail *rails[], int n)
  79 {
  80         int i, mv;
  81
  82         mutex_lock(&dvfs_lock);
  83
  84         for (i = 0; i < n; i++) {
  85                 INIT_LIST_HEAD(&rails[i]->dvfs);
  86                 INIT_LIST_HEAD(&rails[i]->relationships_from);
  87                 INIT_LIST_HEAD(&rails[i]->relationships_to);
  88
  89                 mv = rails[i]->nominal_millivolts;
  90                 if (rails[i]->disable_millivolts > mv)
  91                         rails[i]->disable_millivolts = mv;
  92                 if (rails[i]->suspend_millivolts > mv)
  93                         rails[i]->suspend_millivolts = mv;
  94
  95                 rails[i]->millivolts = mv;
  96                 rails[i]->new_millivolts = mv;
  97                 if (!rails[i]->step)
  98                         rails[i]->step = rails[i]->max_millivolts;
  99                 if (!rails[i]->step_up)
 100                         rails[i]->step_up = rails[i]->step;
 101
 102                 list_add_tail(&rails[i]->node, &dvfs_rail_list);
 103
 104                 if (!strcmp("vdd-cpu", rails[i]->reg_id))
 105                         tegra_cpu_rail = rails[i];
 106                 else if (!strcmp("vdd-core", rails[i]->reg_id))
 107                         tegra_core_rail = rails[i];
 108                 else if (!strcmp("vdd-gpu", rails[i]->reg_id))
 109                         tegra_gpu_rail = rails[i];
 110         }
 111
 112         mutex_unlock(&dvfs_lock);
 113
 114         return 0;
 115 }
 116
 117 static int dvfs_solve_relationship(struct dvfs_relationship *rel)
 118 {
 119         return rel->solve(rel->from, rel->to);
 120 }
 121
 122 static void dvfs_rail_stats_init(struct dvfs_rail *rail, int millivolts)
 123 {
 124         int dvfs_rail_stats_range;
 125
 126         if (!rail->stats.bin_uv)
 127                 rail->stats.bin_uv = DVFS_RAIL_STATS_BIN;
 128
 129         dvfs_rail_stats_range =
 130                 (DVFS_RAIL_STATS_TOP_BIN - 1) * rail->stats.bin_uv / 1000;
 131
 132         rail->stats.last_update = ktime_get();
 133         if (millivolts >= rail->min_millivolts) {
 134                 int i = 1 + (2 * (millivolts - rail->min_millivolts) * 1000 +
 135                              rail->stats.bin_uv) / (2 * rail->stats.bin_uv);
 136                 rail->stats.last_index = min(i, DVFS_RAIL_STATS_TOP_BIN);
 137         }
 138
 139         if (rail->max_millivolts >
 140             rail->min_millivolts + dvfs_rail_stats_range)
 141                 pr_warn("tegra_dvfs: %s: stats above %d mV will be squashed\n",
 142                         rail->reg_id,
 143                         rail->min_millivolts + dvfs_rail_stats_range);
 144 }
 145
 146 static void dvfs_rail_stats_update(
 147         struct dvfs_rail *rail, int millivolts, ktime_t now)
 148 {
 149         rail->stats.time_at_mv[rail->stats.last_index] = ktime_add(
 150                 rail->stats.time_at_mv[rail->stats.last_index], ktime_sub(
 151                         now, rail->stats.last_update));
 152         rail->stats.last_update = now;
 153
 154         if (rail->stats.off)
 155                 return;
 156
 157         if (millivolts >= rail->min_millivolts) {
 158                 int i = 1 + (2 * (millivolts - rail->min_millivolts) * 1000 +
 159                              rail->stats.bin_uv) / (2 * rail->stats.bin_uv);
 160                 rail->stats.last_index = min(i, DVFS_RAIL_STATS_TOP_BIN);
 161         } else if (millivolts == 0)
 162                         rail->stats.last_index = 0;
 163 }
 164
 165 static int dvfs_rail_set_voltage_reg(struct dvfs_rail *rail, int millivolts)
 166 {
 167         int ret;
 168
 169         ret = regulator_set_voltage(rail->reg,
 170                 millivolts * 1000,
 171                 rail->max_millivolts * 1000);
 172
 173         return ret;
 174 }
 175
 176 /**
 177  * dvfs_rail_set_voltage - set voltage in millivolts to specific rail
 178  *
 179  * @rail: struct dvfs_rail * power rail context
 180  * @millivolts: voltage in millivolts to be set to regulator
 181  *
 182  * Sets the voltage on a dvfs rail to a specific value, and updates any
 183  * rails that depend on this rail.
 184  */
 185 static int dvfs_rail_set_voltage(struct dvfs_rail *rail, int millivolts)
 186 {
 187         int ret = 0;
 188         struct dvfs_relationship *rel;
 189         int step, offset;
 190         int i;
 191         int steps;
 192         bool jmp_to_zero;
 193
 194         if (!rail->reg) {
 195                 if (millivolts == rail->millivolts)
 196                         return 0;
 197                 else
 198                         return -EINVAL;
 199         }
 200
 201         if (millivolts > rail->millivolts) {
 202                 step = rail->step_up;
 203                 offset = step;
 204         } else {
 205                 step = rail->step;
 206                 offset = -step;
 207         }
 208
 209         if (rail->dfll_mode) {
 210                 rail->millivolts = millivolts;
 211                 rail->new_millivolts = millivolts;
 212                 dvfs_rail_stats_update(rail, millivolts, ktime_get());
 213                 return 0;
 214         }
 215
 216         if (rail->disabled)
 217                 return 0;
 218
 219         rail->resolving_to = true;
 220         jmp_to_zero = rail->jmp_to_zero &&
 221                         ((millivolts == 0) || (rail->millivolts == 0));
 222         if (jmp_to_zero || (rail->in_band_pm && rail->stats.off))
 223                 steps = 1;
 224         else
 225                 steps = DIV_ROUND_UP(abs(millivolts - rail->millivolts), step);
 226
 227         for (i = 0; i < steps; i++) {
 228                 if (!jmp_to_zero &&
 229                     (abs(millivolts - rail->millivolts) > step))
 230                         rail->new_millivolts = rail->millivolts + offset;
 231                 else
 232                         rail->new_millivolts = millivolts;
 233
 234                 /*
 235                  * Before changing the voltage, tell each rail that depends
 236                  * on this rail that the voltage will change.
 237                  * This rail will be the "from" rail in the relationship,
 238                  * the rail that depends on this rail will be the "to" rail.
 239                  * from->millivolts will be the old voltage
 240                  * from->new_millivolts will be the new voltage
 241                  */
 242                 list_for_each_entry(rel, &rail->relationships_to, to_node) {
 243                         ret = dvfs_rail_update(rel->to);
 244                         if (ret)
 245                                 goto out;
 246                 }
 247
 248                 ret = dvfs_rail_set_voltage_reg(rail, rail->new_millivolts);
 249                 if (ret) {
 250                         pr_err("Failed to set dvfs regulator %s\n",
 251                                         rail->reg_id);
 252                         goto out;
 253                 }
 254
 255                 rail->millivolts = rail->new_millivolts;
 256                 dvfs_rail_stats_update(rail, rail->millivolts, ktime_get());
 257
 258                 /*
 259                  * After changing the voltage, tell each rail that depends
 260                  * on this rail that the voltage has changed.
 261                  * from->millivolts and from->new_millivolts will be the
 262                  * new voltage
 263                  */
 264                 list_for_each_entry(rel, &rail->relationships_to, to_node) {
 265                         ret = dvfs_rail_update(rel->to);
 266                         if (ret)
 267                                 goto out;
 268                 }
 269         }
 270
 271         if (unlikely(rail->millivolts != millivolts)) {
 272                 pr_err("%s: rail didn't reach target %d in %d steps (%d)\n",
 273                         __func__, millivolts, steps, rail->millivolts);
 274                 ret = -EINVAL;
 275         }
 276
 277 out:
 278         rail->resolving_to = false;
 279         return ret;
 280 }
 281
 282 static inline int dvfs_rail_apply_limits(struct dvfs_rail *rail, int millivolts)
 283 {
 284         int min_mv = rail->min_millivolts;
 285         int max_mv = rail->max_millivolts;
 286
 287         if (rail->therm_floors) {
 288                 int i = rail->therm_floor_idx;
 289
 290                 if (i < rail->therm_floors_size)
 291                         min_mv = rail->therm_floors[i].mv;
 292         }
 293
 294         if (rail->therm_caps) {
 295                 int i = rail->therm_cap_idx;
 296
 297                 if (i > 0)
 298                         max_mv = rail->therm_caps[i - 1].mv;
 299         }
 300
 301         if (rail->override_millivolts)
 302                 millivolts = rail->override_millivolts;
 303
 304         clamp_val(millivolts, min_mv, max_mv);
 305
 306         return millivolts;
 307 }
 308
 309 /**
 310  * dvfs_rail_update - update rail voltage
 311  *
 312  * @rail: struct dvfs_rail * power rail context
 313  *
 314  * Determine the minimum valid voltage for a rail, taking into account
 315  * the dvfs clocks and any rails that this rail depends on.  Calls
 316  * dvfs_rail_set_voltage with the new voltage, which will call
 317  * dvfs_rail_update on any rails that depend on this rail.
 318  */
 319 static int dvfs_rail_update(struct dvfs_rail *rail)
 320 {
 321         int millivolts = 0;
 322         struct dvfs *d;
 323         struct dvfs_relationship *rel;
 324         int ret = 0;
 325         int steps;
 326
 327         if (rail->disabled)
 328                 return 0;
 329
 330         /* if dvfs is suspended, return and handle it during resume */
 331         if (rail->suspended)
 332                 return 0;
 333
 334         /* if regulators are not connected yet, return and handle it later */
 335         if (!rail->reg)
 336                 return 0;
 337
 338         /* if rail update is entered while resolving circular dependencies,
 339            abort recursion */
 340         if (rail->resolving_to)
 341                 return 0;
 342
 343         /* Find the maximum voltage requested by any clock */
 344         list_for_each_entry(d, &rail->dvfs, reg_node)
 345                 millivolts = max(d->cur_millivolts, millivolts);
 346
 347         /* Apply offset and min/max limits if any clock is requesting voltage */
 348         if (millivolts)
 349                 millivolts = dvfs_rail_apply_limits(rail, millivolts);
 350         /* Keep current voltage if regulator is to be disabled via explicitly */
 351         else if (rail->in_band_pm)
 352                 return 0;
 353         /* Keep current voltage if regulator must not be disabled at run time */
 354         else if (!rail->jmp_to_zero) {
 355                 WARN(1, "%s cannot be turned off by dvfs\n", rail->reg_id);
 356                 return 0;
 357         }
 358
 359         /*
 360          * retry update if limited by from-relationship to account for
 361          * circular dependencies
 362          */
 363         steps = DIV_ROUND_UP(abs(millivolts - rail->millivolts), rail->step);
 364         for (; steps >= 0; steps--) {
 365                 rail->new_millivolts = millivolts;
 366
 367                 /* Check any rails that this rail depends on */
 368                 list_for_each_entry(rel, &rail->relationships_from, from_node)
 369                         rail->new_millivolts = dvfs_solve_relationship(rel);
 370
 371                 if (rail->new_millivolts == rail->millivolts)
 372                         break;
 373
 374                 ret = dvfs_rail_set_voltage(rail, rail->new_millivolts);
 375         }
 376
 377         return ret;
 378 }
 379
 380 static int dvfs_rail_connect_to_regulator(struct device *dev,
 381                                           struct dvfs_rail *rail)
 382 {
 383         struct regulator *reg;
 384         int v;
 385
 386         if (!rail->reg) {
 387                 mutex_unlock(&dvfs_lock);
 388                 reg = regulator_get(dev, rail->reg_id);
 389                 mutex_lock(&dvfs_lock);
 390                 if (IS_ERR(reg)) {
 391                         pr_err("tegra_dvfs: failed to connect %s rail\n",
 392                                rail->reg_id);
 393                         return PTR_ERR(reg);
 394                 }
 395                 rail->reg = reg;
 396         }
 397
 398         if (!rail->in_band_pm) {
 399                 v = regulator_enable(rail->reg);
 400                 if (v < 0) {
 401                         pr_err("tegra_dvfs: failed on enabling regulator %s\n, err %d",
 402                                 rail->reg_id, v);
 403                         return v;
 404                 }
 405         }
 406
 407         v = regulator_get_voltage(rail->reg);
 408         if (v < 0) {
 409                 pr_err("tegra_dvfs: failed initial get %s voltage\n",
 410                        rail->reg_id);
 411                 return v;
 412         }
 413
 414         if (!rail->min_millivolts) {
 415                 int min_uv, max_uv;
 416
 417                 if (!regulator_get_constraint_voltages(rail->reg, &min_uv,
 418                                                        &max_uv))
 419                         rail->min_millivolts = min_uv / 1000;
 420         }
 421
 422         rail->millivolts = v / 1000;
 423         rail->new_millivolts = rail->millivolts;
 424         dvfs_rail_stats_init(rail, rail->millivolts);
 425
 426         return 0;
 427 }
 428
 429 static inline const int *dvfs_get_millivolts(struct dvfs *d, unsigned long rate)
 430 {
 431         if (tegra_dvfs_is_dfll_scale(d, rate))
 432                 return d->dfll_millivolts;
 433
 434         return d->millivolts;
 435 }
 436
 437 static unsigned long *dvfs_get_freqs(struct dvfs *d)
 438 {
 439         if (d->use_alt_freqs)
 440                 return &d->alt_freqs[0];
 441         else
 442                 return &d->freqs[0];
 443 }
 444
 445 static int __tegra_dvfs_set_rate(struct dvfs *d, unsigned long rate)
 446 {
 447         int i = 0;
 448         int ret, mv;
 449         unsigned long *freqs = dvfs_get_freqs(d);
 450         const int *millivolts = dvfs_get_millivolts(d, rate);
 451
 452         if (freqs == NULL || millivolts == NULL)
 453                 return -ENODEV;
 454
 455         /*
 456          * On entry to dfll range limit 1st step to range bottom (full ramp of
 457          * voltage/rate is completed automatically in dfll mode)
 458          */
 459         if (tegra_dvfs_is_dfll_range_entry(d, rate))
 460                 rate = d->use_dfll_rate_min;
 461
 462         if (rate > freqs[d->num_freqs - 1]) {
 463                 pr_warn("tegra-dvfs: rate %lu too high for dvfs on %s\n", rate,
 464                         d->clk_name);
 465                 return -EINVAL;
 466         }
 467
 468         if (rate == 0) {
 469                 d->cur_millivolts = 0;
 470         } else {
 471                 while (i < d->num_freqs && rate > freqs[i])
 472                         i++;
 473
 474                 if ((d->max_millivolts) &&
 475                     (millivolts[i] > d->max_millivolts)) {
 476                         pr_warn("tegra-dvfs: voltage %d too high for dvfs on %s\n",
 477                                         millivolts[i], d->clk_name);
 478                         return -EINVAL;
 479                 }
 480
 481                 mv = millivolts[i];
 482                 d->cur_millivolts = millivolts[i];
 483         }
 484
 485         d->cur_rate = rate;
 486
 487         ret = dvfs_rail_update(d->dvfs_rail);
 488         if (ret)
 489                 pr_err("Failed to set regulator %s for clock %s to %d mV\n",
 490                         d->dvfs_rail->reg_id, d->clk_name, d->cur_millivolts);
 491
 492         return ret;
 493 }
 494
 495 static struct dvfs *tegra_clk_to_dvfs(struct clk *c)
 496 {
 497         struct dvfs *d;
 498         struct dvfs_rail *rail;
 499
 500         list_for_each_entry(rail, &dvfs_rail_list, node) {
 501                 list_for_each_entry(d, &rail->dvfs, reg_node) {
 502                         if (clk_is_match(c, d->clk))
 503                                 return d;
 504                 }
 505         }
 506         return NULL;
 507 }
 508
 509 static int predict_millivolts(struct dvfs *d, const int *millivolts,
 510                               unsigned long rate)
 511 {
 512         int i;
 513         unsigned long *freqs = dvfs_get_freqs(d);
 514
 515         if (!millivolts)
 516                 return -ENODEV;
 517
 518         for (i = 0; i < d->num_freqs; i++) {
 519                 if (rate <= freqs[i])
 520                         break;
 521         }
 522
 523         if (i == d->num_freqs)
 524                 return -EINVAL;
 525
 526         return millivolts[i];
 527 }
 528
 529 int opp_millivolts[MAX_DVFS_FREQS];
 530 unsigned long opp_frequencies[MAX_DVFS_FREQS];
 531
 532 /**
 533  * tegra_get_cpu_fv_table - get CPU frequencies/voltages table
 534  *
 535  * @num_freqs: number of frequencies
 536  * @freqs: the array of frequencies
 537  * @mvs: the array of voltages
 538  *
 539  * Get the frequency and voltage table using CPU OPP which were built by the
 540  * DFLL driver.
 541  */
 542 int tegra_get_cpu_fv_table(int *num_freqs, unsigned long **freqs, int **mvs)
 543 {
 544         struct device *cpu_dev;
 545         struct dev_pm_opp *opp;
 546         unsigned long rate;
 547         int i, ret = 0;
 548
 549         cpu_dev = get_cpu_device(0);
 550         if (!cpu_dev)
 551                 return -EINVAL;
 552
 553         mutex_lock(&dvfs_lock);
 554         for (i = 0, rate = 0;; rate++) {
 555                 rcu_read_lock();
 556                 opp = dev_pm_opp_find_freq_ceil(cpu_dev, &rate);
 557                 if (IS_ERR(opp)) {
 558                         rcu_read_unlock();
 559                         break;
 560                 }
 561                 opp_frequencies[i] = rate;
 562                 opp_millivolts[i++] = dev_pm_opp_get_voltage(opp);
 563                 rcu_read_unlock();
 564         }
 565         if (!i) {
 566                 ret = -EINVAL;
 567                 goto out;
 568         }
 569
 570         *num_freqs = i;
 571         *freqs = opp_frequencies;
 572         *mvs = opp_millivolts;
 573 out:
 574         mutex_unlock(&dvfs_lock);
 575         return ret;
 576 }
 577 EXPORT_SYMBOL(tegra_get_cpu_fv_table);
 578
 579 /**
 580  * tegra_dvfs_predict_millivolts - return the safe voltage for running
 581  *                                      the clock at one sepcific rate
 582  *
 583  * @c: struct clk * the clock which needs the voltage info
 584  * @rate: the rate being predicted
 585  *
 586  * Extract the voltage table associated with the clock and return the safe
 587  * voltage for ticking the clock at the specified rate
 588  */
 589 int tegra_dvfs_predict_millivolts(struct clk *c, unsigned long rate)
 590 {
 591         int ret;
 592         const int *millivolts;
 593         struct dvfs *d;
 594
 595         mutex_lock(&dvfs_lock);
 596
 597         d = tegra_clk_to_dvfs(c);
 598         if (d == NULL) {
 599                 mutex_unlock(&dvfs_lock);
 600                 return -EINVAL;
 601         }
 602
 603         if (!rate) {
 604                 mutex_unlock(&dvfs_lock);
 605                 return 0;
 606         }
 607
 608         millivolts = dvfs_is_dfll_range(d, rate) ?
 609                         d->dfll_millivolts :
 610                         d->millivolts;
 611
 612         ret = predict_millivolts(d, millivolts, rate);
 613
 614         mutex_unlock(&dvfs_lock);
 615
 616         return ret;
 617 }
 618 EXPORT_SYMBOL(tegra_dvfs_predict_millivolts);
 619
 620 int tegra_dvfs_predict_mv_at_hz_cur_tfloor(struct clk *c, unsigned long rate)
 621 {
 622         return tegra_dvfs_predict_millivolts(c, rate);
 623 }
 624
 625 /**
 626  * tegra_dvfs_set_rate - update rail voltage due to the clock rate change
 627  *
 628  * @c: struct clk * the clock which has changed rate
 629  * @rate: the changed rate
 630  *
 631  * Check if the voltage of the power rail need to be updated due to the clock
 632  * rate change.
 633  */
 634 int tegra_dvfs_set_rate(struct clk *c, unsigned long rate)
 635 {
 636         int ret = 0;
 637         struct dvfs *d;
 638
 639         if (!core_dvfs_started)
 640                 return ret;
 641
 642         mutex_lock(&dvfs_lock);
 643
 644         d = tegra_clk_to_dvfs(c);
 645         if (d)
 646                 ret = __tegra_dvfs_set_rate(d, rate);
 647
 648         mutex_unlock(&dvfs_lock);
 649
 650         return ret;
 651 }
 652 EXPORT_SYMBOL(tegra_dvfs_set_rate);
 653
 654 /**
 655  * tegra_dvfs_get_rate - get current rate used for determining rail voltage
 656  *
 657  * @c: struct clk * clock we want to know the rate of used for determining
 658  *                  rail voltage
 659  *
 660  * Returns 0 if there is no dvfs for the clock.
 661  */
 662 unsigned long tegra_dvfs_get_rate(struct clk *c)
 663 {
 664         unsigned long rate = 0;
 665         struct dvfs *d;
 666
 667         if (!core_dvfs_started)
 668                 return rate;
 669
 670         mutex_lock(&dvfs_lock);
 671
 672         d = tegra_clk_to_dvfs(c);
 673         if (d)
 674                 rate = d->cur_rate;
 675
 676         mutex_unlock(&dvfs_lock);
 677
 678         return rate;
 679 }
 680 EXPORT_SYMBOL(tegra_dvfs_get_rate);
 681
 682 /**
 683  * tegra_dvfs_get_freqs - export dvfs frequency array associated with the clock
 684  *
 685  * @c: struct clk * the clock which needs the frequency table
 686  * @freqs: the array of the frequencies
 687  * @num_freqs: number of the frequencies
 688  *
 689  * Check if the voltage of the power rail need to be updated due to the clock
 690  * rate change.
 691  */
 692 int tegra_dvfs_get_freqs(struct clk *c, unsigned long **freqs, int *num_freqs)
 693 {
 694         struct dvfs *d;
 695
 696         if (!core_dvfs_started)
 697                 return -EINVAL;
 698
 699         d = tegra_clk_to_dvfs(c);
 700         if (d == NULL) {
 701                 pr_err("Failed to get dvfs structure\n");
 702                 return -ENOSYS;
 703         }
 704
 705         *num_freqs = d->num_freqs;
 706         *freqs = dvfs_get_freqs(d);
 707
 708         return 0;
 709 }
 710 EXPORT_SYMBOL(tegra_dvfs_get_freqs);
 711
 712 unsigned long tegra_dvfs_get_maxrate(struct clk *c)
 713 {
 714         unsigned long rate = 0;
 715         int err, num_freqs;
 716         unsigned long *freqs;
 717
 718         if (!core_dvfs_started)
 719                 return rate;
 720
 721         err = tegra_dvfs_get_freqs(c, &freqs, &num_freqs);
 722         if (err < 0)
 723                 return rate;
 724
 725         return freqs[num_freqs - 1];
 726 }
 727
 728 unsigned long tegra_dvfs_round_rate(struct clk *c, unsigned long rate)
 729 {
 730         int i, err, num_freqs;
 731         unsigned long *freqs;
 732
 733         if (!core_dvfs_started)
 734                 return rate;
 735
 736         err = tegra_dvfs_get_freqs(c, &freqs, &num_freqs);
 737         if (err < 0)
 738                 return rate;
 739
 740         for (i = 0; i < num_freqs; i++)
 741                 if (freqs[i] >= rate)
 742                         return freqs[i];
 743
 744         return freqs[i - 1];
 745 }
 746
 747 int tegra_dvfs_use_alt_freqs_on_clk(struct clk *c, bool use_alt_freq)
 748 {
 749         struct dvfs *d;
 750         int err = -ENOENT;
 751
 752         mutex_lock(&dvfs_lock);
 753
 754         d = tegra_clk_to_dvfs(c);
 755         if (!d && d->alt_freqs) {
 756                 err = 0;
 757                 if (d->use_alt_freqs != use_alt_freq) {
 758                         d->use_alt_freqs = use_alt_freq;
 759                         if (__tegra_dvfs_set_rate(d, d->cur_rate) < 0) {
 760                                 d->use_alt_freqs = !use_alt_freq;
 761                                 pr_err("%s: %s: %s alt dvfs failed\n", __func__,
 762                                         d->clk_name,
 763                                         use_alt_freq ? "set" : "clear");
 764                                 __tegra_dvfs_set_rate(d, d->cur_rate);
 765                                 err = -EINVAL;
 766                         }
 767                 }
 768         }
 769
 770         mutex_unlock(&dvfs_lock);
 771
 772         return err;
 773 }
 774 EXPORT_SYMBOL(tegra_dvfs_use_alt_freqs_on_clk);
 775
 776 static int tegra_dvfs_clk_event(struct notifier_block *this,
 777                 unsigned long event, void *ptr)
 778 {
 779         struct clk_notifier_data *cnd = ptr;
 780         struct dvfs *d;
 781
 782         d = tegra_clk_to_dvfs(cnd->clk);
 783         if (d == NULL)
 784                 return NOTIFY_DONE;
 785
 786         if (d->dvfs_rail == tegra_core_rail && !core_dvfs_started)
 787                 return NOTIFY_DONE;
 788
 789         if (!__clk_is_enabled(cnd->clk) && !__clk_is_prepared(cnd->clk))
 790                 return NOTIFY_DONE;
 791
 792         switch (event) {
 793         case PRE_RATE_CHANGE:
 794                 if (cnd->old_rate < cnd->new_rate)
 795                         tegra_dvfs_set_rate(cnd->clk, cnd->new_rate);
 796                 break;
 797         case POST_RATE_CHANGE:
 798                 if (cnd->old_rate > cnd->new_rate)
 799                         tegra_dvfs_set_rate(cnd->clk, cnd->new_rate);
 800                 break;
 801         case ABORT_RATE_CHANGE:
 802                 break;
 803         }
 804
 805         return NOTIFY_DONE;
 806 }
 807
 808 static struct notifier_block tegra_dvfs_nb = {
 809         .notifier_call = tegra_dvfs_clk_event,
 810         .priority = 1,
 811 };
 812
 813 static void cleanup_dvfs_table(struct dvfs *d)
 814 {
 815         int i;
 816
 817         for (i = 0; i < MAX_DVFS_FREQS; i++) {
 818                 if (d->millivolts[i] == 0)
 819                         break;
 820
 821                 if (d->freqs_mult)
 822                         d->freqs[i] *= d->freqs_mult;
 823
 824                 /* If final frequencies are 0, pad with previous frequency */
 825                 if (d->freqs[i] == 0 && i > 1)
 826                         d->freqs[i] = d->freqs[i - 1];
 827         }
 828
 829         d->num_freqs = i;
 830 }
 831
 832 int tegra_setup_dvfs(struct clk *c, struct dvfs *d)
 833 {
 834         cleanup_dvfs_table(d);
 835
 836         d->clk = c;
 837
 838         mutex_lock(&dvfs_lock);
 839         list_add_tail(&d->reg_node, &d->dvfs_rail->dvfs);
 840         mutex_unlock(&dvfs_lock);
 841
 842         return 0;
 843 }
 844
 845 int tegra_dvfs_add_alt_freqs(struct clk *c, struct dvfs *alt_d)
 846 {
 847         struct dvfs *d;
 848         int err = 0;
 849
 850         mutex_lock(&dvfs_lock);
 851
 852         d = tegra_clk_to_dvfs(c);
 853         if (!d) {
 854                 err = -EINVAL;
 855                 goto out;
 856         }
 857
 858         cleanup_dvfs_table(alt_d);
 859
 860         d->alt_freqs = alt_d->freqs;
 861
 862 out:
 863         mutex_unlock(&dvfs_lock);
 864
 865         return 0;
 866 }
 867
 868 static bool tegra_dvfs_all_rails_suspended(void)
 869 {
 870         struct dvfs_rail *rail;
 871
 872         list_for_each_entry(rail, &dvfs_rail_list, node)
 873                 if (!rail->suspended && !rail->disabled)
 874                         return false;
 875
 876         return true;
 877 }
 878
 879 static bool tegra_dvfs_from_rails_suspended_or_solved(struct dvfs_rail *to)
 880 {
 881         struct dvfs_relationship *rel;
 882
 883         list_for_each_entry(rel, &to->relationships_from, from_node)
 884                 if ((!rel->from->suspended) &&
 885                     (!rel->from->disabled) &&
 886                     (!rel->solved_at_nominal))
 887                         return false;
 888
 889         return true;
 890 }
 891
 892 static int tegra_dvfs_suspend_one(void)
 893 {
 894         struct dvfs_rail *rail;
 895         int mv;
 896         int ret = 0;
 897
 898         list_for_each_entry(rail, &dvfs_rail_list, node) {
 899                 if ((rail->suspended) ||
 900                     (rail->disabled) ||
 901                     (!tegra_dvfs_from_rails_suspended_or_solved(rail)))
 902                         continue;
 903
 904                 mv = tegra_dvfs_rail_get_suspend_level(rail);
 905                 mv = dvfs_rail_apply_limits(rail, mv);
 906                 /* apply suspend limit only if it is above current mv */
 907                 if (mv >= rail->millivolts)
 908                         ret = dvfs_rail_set_voltage(rail, mv);
 909                 if (ret) {
 910                         pr_err("tegra_dvfs: failed %s suspend at %d\n",
 911                                rail->reg_id, rail->millivolts);
 912                         return ret;
 913                 }
 914
 915                 rail->suspended = true;
 916                 return 0;
 917         }
 918         return -EINVAL;
 919 }
 920
 921 static void tegra_dvfs_resume(void)
 922 {
 923         struct dvfs_rail *rail;
 924
 925         mutex_lock(&dvfs_lock);
 926
 927         list_for_each_entry(rail, &dvfs_rail_list, node)
 928                 rail->suspended = false;
 929
 930         list_for_each_entry(rail, &dvfs_rail_list, node)
 931                 dvfs_rail_update(rail);
 932
 933         mutex_unlock(&dvfs_lock);
 934 }
 935
 936 static int tegra_dvfs_suspend(void)
 937 {
 938         int ret = 0;
 939
 940         mutex_lock(&dvfs_lock);
 941
 942         while (!tegra_dvfs_all_rails_suspended()) {
 943                 ret = tegra_dvfs_suspend_one();
 944                 if (ret)
 945                         break;
 946         }
 947
 948         mutex_unlock(&dvfs_lock);
 949
 950         if (ret)
 951                 tegra_dvfs_resume();
 952
 953         return ret;
 954 }
 955
 956 int tegra_dvfs_init_thermal_dvfs_voltages(int *therm_voltages,
 957         int *peak_voltages, int freqs_num, int ranges_num, struct dvfs *d)
 958 {
 959         int *millivolts;
 960         int freq_idx, therm_idx;
 961
 962         for (therm_idx = 0; therm_idx < ranges_num; therm_idx++) {
 963                 millivolts = therm_voltages + therm_idx * MAX_DVFS_FREQS;
 964                 for (freq_idx = 0; freq_idx < freqs_num; freq_idx++) {
 965                         int mv = millivolts[freq_idx];
 966                         if ((mv > d->dvfs_rail->max_millivolts) ||
 967                             (mv < d->dvfs_rail->min_millivolts) ||
 968                             (freq_idx && (mv < millivolts[freq_idx - 1]))) {
 969                                 WARN(1, "%s: invalid thermal dvfs entry %d(%d, %d)\n",
 970                                      d->clk_name, mv, freq_idx, therm_idx);
 971                                 return -EINVAL;
 972                         }
 973                         if (mv > peak_voltages[freq_idx])
 974                                 peak_voltages[freq_idx] = mv;
 975                 }
 976         }
 977
 978         d->millivolts = therm_voltages;
 979         d->peak_millivolts = peak_voltages;
 980         d->therm_dvfs = ranges_num > 1;
 981
 982         return 0;
 983 }
 984
 985 static int tegra_dvfs_pm_notifier_event(struct notifier_block *nb,
 986                                 unsigned long event, void *data)
 987 {
 988         if (event == PM_SUSPEND_PREPARE) {
 989                 if (tegra_dvfs_suspend())
 990                         return NOTIFY_STOP;
 991                 pr_info("tegra_dvfs: suspended\n");
 992         } else if (event == PM_POST_SUSPEND) {
 993                 tegra_dvfs_resume();
 994                 pr_info("tegra_dvfs: resumed\n");
 995         }
 996         return NOTIFY_OK;
 997 };
 998
 999 static struct notifier_block tegra_dvfs_pm_nb = {
1000         .notifier_call = tegra_dvfs_pm_notifier_event,
1001         .priority = -1,
1002 };
1003
1004 static int tegra_dvfs_reboot_notify(struct notifier_block *nb,
1005                                 unsigned long event, void *data)
1006 {
1007         switch (event) {
1008         case SYS_RESTART:
1009         case SYS_HALT:
1010         case SYS_POWER_OFF:
1011                 tegra_dvfs_suspend();
1012                 return NOTIFY_OK;
1013         }
1014         return NOTIFY_DONE;
1015 }
1016
1017 static struct notifier_block tegra_dvfs_reboot_nb = {
1018         .notifier_call = tegra_dvfs_reboot_notify,
1019 };
1020
1021 static void __tegra_dvfs_rail_disable(struct dvfs_rail *rail)
1022 {
1023         int ret = -EPERM;
1024         int mv;
1025
1026         if (rail->dfll_mode) {
1027                 rail->disabled = true;
1028                 return;
1029         }
1030
1031         mv = tegra_dvfs_rail_get_disable_level(rail);
1032         mv = dvfs_rail_apply_limits(rail, mv);
1033
1034         if (mv >= rail->millivolts)
1035                 ret = dvfs_rail_set_voltage(rail, mv);
1036         if (ret) {
1037                 pr_err("tegra_dvfs: failed to disable %s at %d\n",
1038                        rail->reg_id, rail->millivolts);
1039                 return;
1040         }
1041         rail->disabled = true;
1042 }
1043
1044 static void __tegra_dvfs_rail_enable(struct dvfs_rail *rail)
1045 {
1046         rail->disabled = false;
1047         dvfs_rail_update(rail);
1048 }
1049
1050 void tegra_dvfs_rail_enable(struct dvfs_rail *rail)
1051 {
1052         if (!rail)
1053                 return;
1054
1055         mutex_lock(&dvfs_lock);
1056
1057         if (rail->disabled)
1058                 __tegra_dvfs_rail_enable(rail);
1059
1060         mutex_unlock(&dvfs_lock);
1061 }
1062
1063 void tegra_dvfs_rail_disable(struct dvfs_rail *rail)
1064 {
1065         if (!rail)
1066                 return;
1067
1068         mutex_lock(&dvfs_lock);
1069         if (rail->disabled)
1070                 goto out;
1071
1072         __tegra_dvfs_rail_disable(rail);
1073 out:
1074         mutex_unlock(&dvfs_lock);
1075 }
1076
1077 bool tegra_dvfs_is_dfll_range(struct clk *c, unsigned long rate)
1078 {
1079         struct dvfs *d;
1080
1081         d = tegra_clk_to_dvfs(c);
1082         if (d == NULL) {
1083                 pr_err("Failed to get dvfs structure\n");
1084                 return false;
1085         }
1086
1087         return dvfs_is_dfll_range(d, rate);
1088 }
1089 EXPORT_SYMBOL(tegra_dvfs_is_dfll_range);
1090
1091 int tegra_dvfs_set_dfll_range(struct clk *c, int range)
1092 {
1093         struct dvfs *d;
1094         int ret = -EINVAL;
1095
1096         mutex_lock(&dvfs_lock);
1097         d = tegra_clk_to_dvfs(c);
1098         if (d == NULL) {
1099                 pr_err("Failed to get dvfs structure\n");
1100                 goto out;
1101         }
1102
1103         if (!d->dfll_millivolts)
1104                 goto out;
1105
1106         if ((range < DFLL_RANGE_NONE) || (range > DFLL_RANGE_HIGH_RATES))
1107                 goto out;
1108
1109         d->range = range;
1110         ret = 0;
1111 out:
1112         mutex_unlock(&dvfs_lock);
1113         return ret;
1114 }
1115 EXPORT_SYMBOL(tegra_dvfs_set_dfll_range);
1116
1117 int tegra_dvfs_dfll_mode_set(struct clk *c, unsigned long rate)
1118 {
1119         struct dvfs *d;
1120
1121         mutex_lock(&dvfs_lock);
1122
1123         d = tegra_clk_to_dvfs(c);
1124         if (d == NULL) {
1125                 pr_err("Failed to get dvfs structure\n");
1126                 mutex_unlock(&dvfs_lock);
1127                 return -EINVAL;
1128         }
1129
1130         if (!d->dvfs_rail->dfll_mode) {
1131                 d->dvfs_rail->dfll_mode = true;
1132                 __tegra_dvfs_set_rate(d, rate);
1133         }
1134
1135         mutex_unlock(&dvfs_lock);
1136
1137         return 0;
1138 }
1139 EXPORT_SYMBOL(tegra_dvfs_dfll_mode_set);
1140
1141 int tegra_dvfs_dfll_mode_clear(struct clk *c, unsigned long rate)
1142 {
1143         int ret = 0;
1144         struct dvfs *d;
1145
1146         mutex_lock(&dvfs_lock);
1147
1148         d = tegra_clk_to_dvfs(c);
1149         if (d == NULL) {
1150                 pr_err("Failed to get dvfs structure\n");
1151                 mutex_unlock(&dvfs_lock);
1152                 return -EINVAL;
1153         }
1154
1155         if (d->dvfs_rail->dfll_mode) {
1156                 d->dvfs_rail->dfll_mode = false;
1157                 d->dvfs_rail->millivolts = regulator_get_voltage(
1158                                 d->dvfs_rail->reg) / 1000;
1159                 if (d->dvfs_rail->disabled) {
1160                         d->dvfs_rail->disabled = false;
1161                         __tegra_dvfs_rail_disable(d->dvfs_rail);
1162                 }
1163                 ret = __tegra_dvfs_set_rate(d, rate);
1164         }
1165
1166         mutex_unlock(&dvfs_lock);
1167
1168         return ret;
1169 }
1170 EXPORT_SYMBOL(tegra_dvfs_dfll_mode_clear);
1171
1172 int tegra_dvfs_get_dfll_threshold(struct clk *c, unsigned long *rate)
1173 {
1174         struct dvfs *d;
1175
1176         d = tegra_clk_to_dvfs(c);
1177         if (d == NULL) {
1178                 pr_err("Failed to get dvfs structure\n");
1179                 return -EINVAL;
1180         }
1181
1182         if (d->dvfs_rail && d->use_dfll_rate_min)
1183                 *rate = d->use_dfll_rate_min;
1184
1185         return 0;
1186 }
1187 EXPORT_SYMBOL(tegra_dvfs_get_dfll_threshold);
1188
1189 int tegra_dvfs_core_count_thermal_states(enum tegra_dvfs_core_thermal_type type)
1190 {
1191         if (IS_ERR_OR_NULL(tegra_core_rail) || !tegra_core_rail->is_ready)
1192                 return -EINVAL;
1193
1194         if (type == TEGRA_DVFS_CORE_THERMAL_FLOOR)
1195                 return tegra_core_rail->therm_floors_size;
1196         else if (type == TEGRA_DVFS_CORE_THERMAL_CAP)
1197                 return tegra_core_rail->therm_caps_size;
1198         else
1199                 return -EINVAL;
1200 }
1201 EXPORT_SYMBOL(tegra_dvfs_core_count_thermal_states);
1202
1203 int tegra_dvfs_core_get_thermal_index(enum tegra_dvfs_core_thermal_type type)
1204 {
1205         if (IS_ERR_OR_NULL(tegra_core_rail) || !tegra_core_rail->is_ready)
1206                 return -EINVAL;
1207
1208         if (type == TEGRA_DVFS_CORE_THERMAL_FLOOR)
1209                 return tegra_core_rail->therm_floor_idx;
1210         else if (type == TEGRA_DVFS_CORE_THERMAL_CAP)
1211                 return tegra_core_rail->therm_cap_idx;
1212         else
1213                 return -EINVAL;
1214 }
1215 EXPORT_SYMBOL(tegra_dvfs_core_get_thermal_index);
1216
1217 int tegra_dvfs_core_update_thermal_index(enum tegra_dvfs_core_thermal_type type,
1218                                          unsigned long new_idx)
1219 {
1220         struct dvfs_rail *rail = tegra_core_rail;
1221         int ret = 0;
1222
1223         if (IS_ERR_OR_NULL(tegra_core_rail) || !tegra_core_rail->is_ready)
1224                 return -EINVAL;
1225
1226         mutex_lock(&dvfs_lock);
1227         if (type == TEGRA_DVFS_CORE_THERMAL_FLOOR) {
1228                 if (rail->therm_floor_idx != new_idx) {
1229                         rail->therm_floor_idx = new_idx;
1230                         dvfs_rail_update(rail);
1231                 }
1232         } else if (type == TEGRA_DVFS_CORE_THERMAL_CAP) {
1233                 if (rail->therm_cap_idx != new_idx) {
1234                         rail->therm_cap_idx = new_idx;
1235                         dvfs_rail_update(rail);
1236                 }
1237         } else {
1238                 ret = -EINVAL;
1239         }
1240         mutex_unlock(&dvfs_lock);
1241
1242         return ret;
1243 }
1244 EXPORT_SYMBOL(tegra_dvfs_core_update_thermal_index);
1245
1246 struct dvfs_rail *tegra_dvfs_get_rail_by_name(char *name)
1247 {
1248         struct dvfs_rail *rail;
1249
1250         list_for_each_entry(rail, &dvfs_rail_list, node) {
1251                 if (!strcmp(rail->reg_id, name))
1252                         return rail;
1253         }
1254
1255         return NULL;
1256 }
1257
1258 bool tegra_dvfs_is_rail_up(struct dvfs_rail *rail)
1259 {
1260         if (!rail)
1261                 return false;
1262
1263         if (!rail->in_band_pm)
1264                 return true;
1265
1266         return regulator_is_enabled(rail->reg);
1267 }
1268
1269 int tegra_dvfs_rail_power_up(struct dvfs_rail *rail)
1270 {
1271         if (!rail || !rail->in_band_pm)
1272                 return -EINVAL;
1273
1274         return regulator_enable(rail->reg);
1275 }
1276
1277 int tegra_dvfs_rail_power_down(struct dvfs_rail *rail)
1278 {
1279         if (!rail || !rail->in_band_pm)
1280                 return -EINVAL;
1281
1282         return regulator_disable(rail->reg);
1283 }
1284
1285 /*
1286  * Validate rail thermal floors/caps, and get its size.
1287  * Valid floors/caps:
1288  * - voltage limits are descending with temperature increasing.
1289  * - the lowest limit is above rail minimum voltage in pll and
1290  *   in dfll mode (if applicable).
1291  * - the highest limit is below rail nominal voltage.
1292  */
1293 static int get_thermal_limits_size(struct dvfs_rail *rail,
1294                                    enum tegra_dvfs_core_thermal_type type)
1295 {
1296         const struct dvfs_therm_limits *limits;
1297         int i;
1298
1299         if (type == TEGRA_DVFS_CORE_THERMAL_FLOOR)
1300                 limits = rail->therm_floors;
1301         else if (type == TEGRA_DVFS_CORE_THERMAL_CAP)
1302                 limits = rail->therm_caps;
1303         else
1304                 return -EINVAL;
1305
1306         if (!limits[0].mv) {
1307                 pr_warn("%s: Missing thermal limits\n", rail->reg_id);
1308                 return -EINVAL;
1309         }
1310
1311         for (i = 0; i < MAX_THERMAL_LIMITS - 1; i++) {
1312                 if (!limits[i + 1].mv)
1313                         break;
1314
1315                 if ((limits[i].temperature >= limits[i + 1].temperature) ||
1316                     (limits[i].mv < limits[i + 1].mv)) {
1317                         pr_warn("%s: Unordered thermal limits\n",
1318                                 rail->reg_id);
1319                         return -EINVAL;
1320                 }
1321         }
1322
1323         if (limits[i].mv < rail->min_millivolts) {
1324                 pr_warn("%s: Thermal floors below minimum voltage\n",
1325                         rail->reg_id);
1326                 return -EINVAL;
1327         }
1328
1329         return i + 1;
1330 }
1331
1332 void tegra_dvfs_init_therm_limits(struct dvfs_rail *rail)
1333 {
1334         int size;
1335
1336         size = get_thermal_limits_size(rail, TEGRA_DVFS_CORE_THERMAL_FLOOR);
1337         if (size <= 0 || rail->therm_floors[0].mv > rail->nominal_millivolts) {
1338                 rail->therm_floors = NULL;
1339                 rail->therm_floors_size = 0;
1340                 pr_warn("%s: invalid Vmin thermal floors\n", rail->reg_id);
1341         } else {
1342                 rail->therm_floors_size = size;
1343                 rail->therm_floor_idx = 0;
1344         }
1345
1346         size = get_thermal_limits_size(rail, TEGRA_DVFS_CORE_THERMAL_CAP);
1347         if (size <= 0) {
1348                 rail->therm_caps = NULL;
1349                 rail->therm_caps_size = 0;
1350                 pr_warn("%s: invalid Vmax thermal caps\n", rail->reg_id);
1351         } else {
1352                 rail->therm_caps_size = size;
1353                 rail->therm_cap_idx = size;
1354         }
1355 }
1356
1357 static int tegra_config_dvfs(struct dvfs_rail *rail)
1358 {
1359         int i;
1360         struct dvfs *d;
1361
1362         list_for_each_entry(d, &rail->dvfs, reg_node) {
1363                 if (__clk_is_enabled(d->clk) || __clk_is_prepared(d->clk)) {
1364                         d->cur_rate = clk_get_rate(d->clk);
1365                         d->cur_millivolts = d->max_millivolts;
1366
1367                         for (i = 0; i < d->num_freqs; i++)
1368                                 if (d->cur_rate <= d->freqs[i])
1369                                         break;
1370
1371                         if (i != d->num_freqs)
1372                                 d->cur_millivolts = d->millivolts[i];
1373                 }
1374
1375                 mutex_unlock(&dvfs_lock);
1376                 clk_notifier_register(d->clk, &tegra_dvfs_nb);
1377                 mutex_lock(&dvfs_lock);
1378         }
1379
1380         return 0;
1381 }
1382
1383 static int tegra_dvfs_regulator_init(struct device *dev)
1384 {
1385         struct dvfs_rail *rail;
1386         int err;
1387
1388         mutex_lock(&dvfs_lock);
1389
1390         list_for_each_entry(rail, &dvfs_rail_list, node) {
1391                 err = dvfs_rail_connect_to_regulator(dev, rail);
1392                 if (err) {
1393                         if (!rail->disabled)
1394                                 __tegra_dvfs_rail_disable(rail);
1395
1396                         mutex_unlock(&dvfs_lock);
1397                         return err;
1398                 }
1399         }
1400
1401         list_for_each_entry(rail, &dvfs_rail_list, node) {
1402                 tegra_config_dvfs(rail);
1403                 __tegra_dvfs_rail_enable(rail);
1404         }
1405
1406         core_dvfs_started = true;
1407
1408         mutex_unlock(&dvfs_lock);
1409
1410         register_pm_notifier(&tegra_dvfs_pm_nb);
1411         register_reboot_notifier(&tegra_dvfs_reboot_nb);
1412
1413         return 0;
1414 }
1415
1416 #ifdef CONFIG_DEBUG_FS
1417 static int dvfs_tree_sort_cmp(void *p, struct list_head *a, struct list_head *b)
1418 {
1419         struct dvfs *da = list_entry(a, struct dvfs, reg_node);
1420         struct dvfs *db = list_entry(b, struct dvfs, reg_node);
1421         int ret;
1422
1423         ret = strcmp(da->dvfs_rail->reg_id, db->dvfs_rail->reg_id);
1424         if (ret != 0)
1425                 return ret;
1426
1427         if (da->cur_millivolts < db->cur_millivolts)
1428                 return 1;
1429         if (da->cur_millivolts > db->cur_millivolts)
1430                 return -1;
1431
1432         return strcmp(da->clk_name, db->clk_name);
1433 }
1434
1435 /* To emulate and show rail relations with 0 mV on dependent rail-to */
1436 static struct dvfs_rail show_to;
1437 static struct dvfs_relationship show_rel;
1438
1439 static int dvfs_tree_show(struct seq_file *s, void *data)
1440 {
1441         struct dvfs *d;
1442         struct dvfs_rail *rail;
1443         struct dvfs_relationship *rel;
1444
1445         seq_puts(s, "   clock           rate       mV\n");
1446         seq_puts(s, "-------------------------------------\n");
1447
1448         mutex_lock(&dvfs_lock);
1449
1450         list_for_each_entry(rail, &dvfs_rail_list, node) {
1451                 int therm_mv = 0;
1452
1453                 seq_printf(s, "%s %d mV%s:\n", rail->reg_id,
1454                            rail->stats.off ? 0 : rail->millivolts,
1455                            rail->dfll_mode ? " dfll mode" :
1456                                 rail->disabled ? " disabled" : "");
1457                 list_for_each_entry(rel, &rail->relationships_from, from_node) {
1458                         show_rel = *rel;
1459                         show_rel.to = &show_to;
1460                         show_to = *rel->to;
1461                         show_to.millivolts = show_to.new_millivolts = 0;
1462                         seq_printf(s, "   %-10s %-7d mV %-4d mV .. %-4d mV\n",
1463                                 rel->from->reg_id, rel->from->millivolts,
1464                                 dvfs_solve_relationship(&show_rel),
1465                                 dvfs_solve_relationship(rel));
1466                 }
1467                 seq_printf(s, "   nominal    %-7d mV\n",
1468                            rail->nominal_millivolts);
1469
1470                 if ((rail->therm_floors) &&
1471                     (rail->therm_floor_idx < rail->therm_floors_size)) {
1472                         therm_mv = rail->therm_floors[rail->therm_floor_idx].mv;
1473                 }
1474                 seq_printf(s, "   therm_floor    %-7d mV\n", therm_mv);
1475
1476                 if ((rail->therm_caps) &&
1477                     (rail->therm_cap_idx > 0)) {
1478                         therm_mv = rail->therm_caps[rail->therm_cap_idx - 1].mv;
1479                 }
1480                 seq_printf(s, "   therm_cap    %-7d mV\n", therm_mv);
1481
1482                 list_sort(NULL, &rail->dvfs, dvfs_tree_sort_cmp);
1483
1484                 list_for_each_entry(d, &rail->dvfs, reg_node) {
1485                         seq_printf(s, "   %-15s %-10lu %-4d mV\n", d->clk_name,
1486                                 d->cur_rate, d->cur_millivolts);
1487                 }
1488         }
1489
1490         mutex_unlock(&dvfs_lock);
1491
1492         return 0;
1493 }
1494
1495 static int dvfs_tree_open(struct inode *inode, struct file *file)
1496 {
1497         return single_open(file, dvfs_tree_show, inode->i_private);
1498 }
1499
1500 static const struct file_operations dvfs_tree_fops = {
1501         .open           = dvfs_tree_open,
1502         .read           = seq_read,
1503         .llseek         = seq_lseek,
1504         .release        = single_release,
1505 };
1506
1507 static int dvfs_table_show(struct seq_file *s, void *data)
1508 {
1509         int i;
1510         struct dvfs *d;
1511         struct dvfs_rail *rail;
1512         const int *v_pll, *last_v_pll = NULL;
1513         const int *v_dfll, *last_v_dfll = NULL;
1514
1515         seq_puts(s, "DVFS tables: units mV/MHz\n");
1516
1517         mutex_lock(&dvfs_lock);
1518
1519         list_for_each_entry(rail, &dvfs_rail_list, node) {
1520                 list_for_each_entry(d, &rail->dvfs, reg_node) {
1521                         bool mv_done = false;
1522                         v_pll = d->millivolts;
1523                         v_dfll = d->dfll_millivolts;
1524
1525                         if (v_pll && (last_v_pll != v_pll)) {
1526                                 if (!mv_done) {
1527                                         seq_puts(s, "\n");
1528                                         mv_done = true;
1529                                 }
1530                                 last_v_pll = v_pll;
1531                                 seq_printf(s, "%-16s", rail->reg_id);
1532                                 for (i = 0; i < d->num_freqs; i++)
1533                                         seq_printf(s, "%7d", v_pll[i]);
1534                                 seq_puts(s, "\n");
1535                         }
1536
1537                         if (v_dfll && (last_v_dfll != v_dfll)) {
1538                                 if (!mv_done) {
1539                                         seq_puts(s, "\n");
1540                                         mv_done = true;
1541                                 }
1542                                 last_v_dfll = v_dfll;
1543                                 seq_printf(s, "%-8s (dfll) ", rail->reg_id);
1544                                 for (i = 0; i < d->num_freqs; i++)
1545                                         seq_printf(s, "%7d", v_dfll[i]);
1546                                 seq_puts(s, "\n");
1547                         }
1548
1549                         seq_printf(s, "%-16s", d->clk_name);
1550                         for (i = 0; i < d->num_freqs; i++) {
1551                                 unsigned int f = d->freqs[i]/100000;
1552                                 seq_printf(s, " %4u.%u", f/10, f%10);
1553                         }
1554                         if (d->alt_freqs) {
1555                                 seq_puts(s, "\n");
1556                                 seq_printf(s, "%-10s (alt)", d->clk_name);
1557                                 for (i = 0; i < d->num_freqs; i++) {
1558                                         unsigned int f = d->alt_freqs[i]/100000;
1559                                         seq_printf(s, " %4u.%u", f/10, f%10);
1560                                 }
1561                         }
1562
1563                         seq_puts(s, "\n");
1564                 }
1565         }
1566
1567         mutex_unlock(&dvfs_lock);
1568
1569         return 0;
1570 }
1571
1572 static int dvfs_table_open(struct inode *inode, struct file *file)
1573 {
1574         return single_open(file, dvfs_table_show, inode->i_private);
1575 }
1576
1577 static const struct file_operations dvfs_table_fops = {
1578         .open           = dvfs_table_open,
1579         .read           = seq_read,
1580         .llseek         = seq_lseek,
1581         .release        = single_release,
1582 };
1583
1584 static int rail_stats_save_to_buf(char *buf, int len)
1585 {
1586         int i;
1587         struct dvfs_rail *rail;
1588         char *str = buf;
1589         char *end = buf + len;
1590
1591         str += scnprintf(str, end - str, "%-12s %-10s\n", "millivolts", "time");
1592
1593         mutex_lock(&dvfs_lock);
1594
1595         list_for_each_entry(rail, &dvfs_rail_list, node) {
1596                 str += scnprintf(str, end - str, "%s (bin: %d.%dmV)\n",
1597                            rail->reg_id,
1598                            rail->stats.bin_uv / 1000,
1599                            (rail->stats.bin_uv / 10) % 100);
1600
1601                 dvfs_rail_stats_update(rail, -1, ktime_get());
1602
1603                 str += scnprintf(str, end - str, "%-12d %-10llu\n", 0,
1604                         cputime64_to_clock_t(msecs_to_jiffies(
1605                                 ktime_to_ms(rail->stats.time_at_mv[0]))));
1606
1607                 for (i = 1; i <= DVFS_RAIL_STATS_TOP_BIN; i++) {
1608                         ktime_t ktime_zero = ktime_set(0, 0);
1609                         if (ktime_equal(rail->stats.time_at_mv[i], ktime_zero))
1610                                 continue;
1611                         str += scnprintf(str, end - str, "%-12d %-10llu\n",
1612                                 rail->min_millivolts +
1613                                 (i - 1) * rail->stats.bin_uv / 1000,
1614                                 cputime64_to_clock_t(msecs_to_jiffies(
1615                                         ktime_to_ms(rail->stats.time_at_mv[i])))
1616                         );
1617                 }
1618         }
1619         mutex_unlock(&dvfs_lock);
1620         return str - buf;
1621 }
1622
1623 static int rail_stats_show(struct seq_file *s, void *data)
1624 {
1625         char *buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
1626         int size = 0;
1627
1628         if (!buf)
1629                 return -ENOMEM;
1630
1631         size = rail_stats_save_to_buf(buf, PAGE_SIZE);
1632         seq_write(s, buf, size);
1633         kfree(buf);
1634         return 0;
1635 }
1636
1637 static int rail_stats_open(struct inode *inode, struct file *file)
1638 {
1639         return single_open(file, rail_stats_show, inode->i_private);
1640 }
1641
1642 static const struct file_operations rail_stats_fops = {
1643         .open           = rail_stats_open,
1644         .read           = seq_read,
1645         .llseek         = seq_lseek,
1646         .release        = single_release,
1647 };
1648
1649 static int gpu_dvfs_t_show(struct seq_file *s, void *data)
1650 {
1651         int i, j;
1652         int num_ranges = 1;
1653         int *trips = NULL;
1654         struct dvfs *d;
1655         struct dvfs_rail *rail = tegra_gpu_rail;
1656         int max_mv[MAX_DVFS_FREQS] = {};
1657
1658         if (!tegra_gpu_rail) {
1659                 seq_printf(s, "Only supported for T124 or higher\n");
1660                 return -ENOSYS;
1661         }
1662
1663         mutex_lock(&dvfs_lock);
1664
1665         d = list_first_entry(&rail->dvfs, struct dvfs, reg_node);
1666         if (rail->vts_cdev && d->therm_dvfs) {
1667                 num_ranges = rail->vts_number_of_trips + 1;
1668                 trips = rail->vts_trips_table;
1669         }
1670
1671         seq_printf(s, "%-11s", "T(C)\\F(kHz)");
1672         for (i = 0; i < d->num_freqs; i++) {
1673                 unsigned int f = d->freqs[i]/1000;
1674                 seq_printf(s, " %7u", f);
1675         }
1676         seq_printf(s, "\n");
1677
1678         for (j = 0; j < num_ranges; j++) {
1679                 seq_printf(s, "%s", j == rail->therm_scale_idx ? ">" : " ");
1680
1681                 if (!trips || (num_ranges == 1))
1682                         seq_printf(s, "%4s..%-4s", "", "");
1683                 else if (j == 0)
1684                         seq_printf(s, "%4s..%-4d", "", trips[j]);
1685                 else if (j == num_ranges - 1)
1686                         seq_printf(s, "%4d..%-4s", trips[j], "");
1687                 else
1688                         seq_printf(s, "%4d..%-4d", trips[j-1], trips[j]);
1689
1690                 for (i = 0; i < d->num_freqs; i++) {
1691                         int mv = *(d->millivolts + j * MAX_DVFS_FREQS + i);
1692                         seq_printf(s, " %7d", mv);
1693                         max_mv[i] = max(max_mv[i], mv);
1694                 }
1695                 seq_printf(s, " mV\n");
1696         }
1697
1698         seq_printf(s, "%3s%-8s\n", "", "------");
1699         seq_printf(s, "%3s%-8s", "", "max(T)");
1700         for (i = 0; i < d->num_freqs; i++)
1701                 seq_printf(s, " %7d", max_mv[i]);
1702         seq_printf(s, " mV\n");
1703
1704         mutex_unlock(&dvfs_lock);
1705
1706         return 0;
1707 }
1708
1709 static int gpu_dvfs_t_open(struct inode *inode, struct file *file)
1710 {
1711         return single_open(file, gpu_dvfs_t_show, NULL);
1712 }
1713
1714 static const struct file_operations gpu_dvfs_t_fops = {
1715         .open           = gpu_dvfs_t_open,
1716         .read           = seq_read,
1717         .llseek         = seq_lseek,
1718         .release        = single_release,
1719 };
1720
1721 static int dvfs_debugfs_init(void)
1722 {
1723         struct dentry *d_root, *d;
1724
1725         d_root = debugfs_create_dir("tegra_dvfs", NULL);
1726         if (!d_root)
1727                 return -ENOMEM;
1728
1729         d = debugfs_create_file("dvfs", S_IRUGO, d_root, NULL,
1730                 &dvfs_tree_fops);
1731         if (!d)
1732                 return -ENOMEM;
1733
1734         d = debugfs_create_file("dvfs_table", S_IRUGO, d_root, NULL,
1735                 &dvfs_table_fops);
1736         if (!d)
1737                 return -ENOMEM;
1738
1739         d = debugfs_create_file("rails", S_IRUGO, d_root, NULL,
1740                 &rail_stats_fops);
1741         if (!d)
1742                 return -ENOMEM;
1743
1744         d = debugfs_create_file("gpu_dvfs_t", S_IRUGO | S_IWUSR, d_root, NULL,
1745                                 &gpu_dvfs_t_fops);
1746         if (!d)
1747                 return -ENOMEM;
1748
1749         return 0;
1750 }
1751
1752 #endif
1753
1754 typedef int (*dvfs_init_cb_t)(void);
1755
1756 static const struct of_device_id tegra_dvfs_of_match[] = {
1757         { .compatible = "nvidia,tegra124-dvfs", .data = tegra124_init_dvfs },
1758         { .compatible = "nvidia,tegra210-dvfs", .data = tegra210_init_dvfs },
1759         {},
1760 };
1761
1762 static int tegra_dvfs_probe(struct platform_device *pdev)
1763 {
1764         const struct of_device_id *match;
1765         dvfs_init_cb_t dvfs_init_cb;
1766         struct dvfs_rail *rail;
1767         int ret = -EINVAL;
1768
1769         match = of_match_node(tegra_dvfs_of_match, pdev->dev.of_node);
1770         if (!match)
1771                 goto out;
1772
1773         dvfs_init_cb = (dvfs_init_cb_t)match->data;
1774         ret = dvfs_init_cb();
1775         if (ret)
1776                 goto out;
1777
1778         ret = tegra_dvfs_regulator_init(&pdev->dev);
1779         if (ret)
1780                 goto out;
1781
1782         list_for_each_entry(rail, &dvfs_rail_list, node) {
1783                 rail->is_ready = true;
1784         }
1785
1786 #ifdef CONFIG_DEBUG_FS
1787         dvfs_debugfs_init();
1788 #endif
1789         return 0;
1790 out:
1791         return ret;
1792 }
1793
1794 static int tegra_dvfs_remove(struct platform_device *pdev)
1795 {
1796         struct dvfs *d;
1797
1798         core_dvfs_started = false;
1799
1800         unregister_pm_notifier(&tegra_dvfs_reboot_nb);
1801         unregister_pm_notifier(&tegra_dvfs_pm_nb);
1802
1803         list_for_each_entry(d, &tegra_core_rail->dvfs, reg_node) {
1804                 clk_notifier_unregister(d->clk, &tegra_dvfs_nb);
1805         }
1806
1807         return 0;
1808 }
1809
1810
1811 static struct platform_driver tegra_dvfs_platdrv = {
1812         .driver = {
1813                 .name   = "tegra-dvfs",
1814                 .owner  = THIS_MODULE,
1815                 .of_match_table = tegra_dvfs_of_match,
1816         },
1817         .probe          = tegra_dvfs_probe,
1818         .remove         = tegra_dvfs_remove,
1819 };
1820 module_platform_driver(tegra_dvfs_platdrv);