drivers/gpu/drm/radeon/ni_dpm.c

   1 /*
   2  * Copyright 2012 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include "drmP.h"
  25 #include "radeon.h"
  26 #include "nid.h"
  27 #include "r600_dpm.h"
  28 #include "ni_dpm.h"
  29 #include "atom.h"
  30 #include <linux/math64.h>
  31
  32 #define MC_CG_ARB_FREQ_F0           0x0a
  33 #define MC_CG_ARB_FREQ_F1           0x0b
  34 #define MC_CG_ARB_FREQ_F2           0x0c
  35 #define MC_CG_ARB_FREQ_F3           0x0d
  36
  37 #define SMC_RAM_END 0xC000
  38
  39 static const struct ni_cac_weights cac_weights_cayman_xt =
  40 {
  41         0x15,
  42         0x2,
  43         0x19,
  44         0x2,
  45         0x8,
  46         0x14,
  47         0x2,
  48         0x16,
  49         0xE,
  50         0x17,
  51         0x13,
  52         0x2B,
  53         0x10,
  54         0x7,
  55         0x5,
  56         0x5,
  57         0x5,
  58         0x2,
  59         0x3,
  60         0x9,
  61         0x10,
  62         0x10,
  63         0x2B,
  64         0xA,
  65         0x9,
  66         0x4,
  67         0xD,
  68         0xD,
  69         0x3E,
  70         0x18,
  71         0x14,
  72         0,
  73         0x3,
  74         0x3,
  75         0x5,
  76         0,
  77         0x2,
  78         0,
  79         0,
  80         0,
  81         0,
  82         0,
  83         0,
  84         0,
  85         0,
  86         0,
  87         0x1CC,
  88         0,
  89         0x164,
  90         1,
  91         1,
  92         1,
  93         1,
  94         12,
  95         12,
  96         12,
  97         0x12,
  98         0x1F,
  99         132,
 100         5,
 101         7,
 102         0,
 103         { 0, 0, 0, 0, 0, 0, 0, 0 },
 104         { 0, 0, 0, 0 },
 105         true
 106 };
 107
 108 static const struct ni_cac_weights cac_weights_cayman_pro =
 109 {
 110         0x16,
 111         0x4,
 112         0x10,
 113         0x2,
 114         0xA,
 115         0x16,
 116         0x2,
 117         0x18,
 118         0x10,
 119         0x1A,
 120         0x16,
 121         0x2D,
 122         0x12,
 123         0xA,
 124         0x6,
 125         0x6,
 126         0x6,
 127         0x2,
 128         0x4,
 129         0xB,
 130         0x11,
 131         0x11,
 132         0x2D,
 133         0xC,
 134         0xC,
 135         0x7,
 136         0x10,
 137         0x10,
 138         0x3F,
 139         0x1A,
 140         0x16,
 141         0,
 142         0x7,
 143         0x4,
 144         0x6,
 145         1,
 146         0x2,
 147         0x1,
 148         0,
 149         0,
 150         0,
 151         0,
 152         0,
 153         0,
 154         0x30,
 155         0,
 156         0x1CF,
 157         0,
 158         0x166,
 159         1,
 160         1,
 161         1,
 162         1,
 163         12,
 164         12,
 165         12,
 166         0x15,
 167         0x1F,
 168         132,
 169         6,
 170         6,
 171         0,
 172         { 0, 0, 0, 0, 0, 0, 0, 0 },
 173         { 0, 0, 0, 0 },
 174         true
 175 };
 176
 177 static const struct ni_cac_weights cac_weights_cayman_le =
 178 {
 179         0x7,
 180         0xE,
 181         0x1,
 182         0xA,
 183         0x1,
 184         0x3F,
 185         0x2,
 186         0x18,
 187         0x10,
 188         0x1A,
 189         0x1,
 190         0x3F,
 191         0x1,
 192         0xE,
 193         0x6,
 194         0x6,
 195         0x6,
 196         0x2,
 197         0x4,
 198         0x9,
 199         0x1A,
 200         0x1A,
 201         0x2C,
 202         0xA,
 203         0x11,
 204         0x8,
 205         0x19,
 206         0x19,
 207         0x1,
 208         0x1,
 209         0x1A,
 210         0,
 211         0x8,
 212         0x5,
 213         0x8,
 214         0x1,
 215         0x3,
 216         0x1,
 217         0,
 218         0,
 219         0,
 220         0,
 221         0,
 222         0,
 223         0x38,
 224         0x38,
 225         0x239,
 226         0x3,
 227         0x18A,
 228         1,
 229         1,
 230         1,
 231         1,
 232         12,
 233         12,
 234         12,
 235         0x15,
 236         0x22,
 237         132,
 238         6,
 239         6,
 240         0,
 241         { 0, 0, 0, 0, 0, 0, 0, 0 },
 242         { 0, 0, 0, 0 },
 243         true
 244 };
 245
 246 #define NISLANDS_MGCG_SEQUENCE  300
 247
 248 static const u32 cayman_cgcg_cgls_default[] =
 249 {
 250         0x000008f8, 0x00000010, 0xffffffff,
 251         0x000008fc, 0x00000000, 0xffffffff,
 252         0x000008f8, 0x00000011, 0xffffffff,
 253         0x000008fc, 0x00000000, 0xffffffff,
 254         0x000008f8, 0x00000012, 0xffffffff,
 255         0x000008fc, 0x00000000, 0xffffffff,
 256         0x000008f8, 0x00000013, 0xffffffff,
 257         0x000008fc, 0x00000000, 0xffffffff,
 258         0x000008f8, 0x00000014, 0xffffffff,
 259         0x000008fc, 0x00000000, 0xffffffff,
 260         0x000008f8, 0x00000015, 0xffffffff,
 261         0x000008fc, 0x00000000, 0xffffffff,
 262         0x000008f8, 0x00000016, 0xffffffff,
 263         0x000008fc, 0x00000000, 0xffffffff,
 264         0x000008f8, 0x00000017, 0xffffffff,
 265         0x000008fc, 0x00000000, 0xffffffff,
 266         0x000008f8, 0x00000018, 0xffffffff,
 267         0x000008fc, 0x00000000, 0xffffffff,
 268         0x000008f8, 0x00000019, 0xffffffff,
 269         0x000008fc, 0x00000000, 0xffffffff,
 270         0x000008f8, 0x0000001a, 0xffffffff,
 271         0x000008fc, 0x00000000, 0xffffffff,
 272         0x000008f8, 0x0000001b, 0xffffffff,
 273         0x000008fc, 0x00000000, 0xffffffff,
 274         0x000008f8, 0x00000020, 0xffffffff,
 275         0x000008fc, 0x00000000, 0xffffffff,
 276         0x000008f8, 0x00000021, 0xffffffff,
 277         0x000008fc, 0x00000000, 0xffffffff,
 278         0x000008f8, 0x00000022, 0xffffffff,
 279         0x000008fc, 0x00000000, 0xffffffff,
 280         0x000008f8, 0x00000023, 0xffffffff,
 281         0x000008fc, 0x00000000, 0xffffffff,
 282         0x000008f8, 0x00000024, 0xffffffff,
 283         0x000008fc, 0x00000000, 0xffffffff,
 284         0x000008f8, 0x00000025, 0xffffffff,
 285         0x000008fc, 0x00000000, 0xffffffff,
 286         0x000008f8, 0x00000026, 0xffffffff,
 287         0x000008fc, 0x00000000, 0xffffffff,
 288         0x000008f8, 0x00000027, 0xffffffff,
 289         0x000008fc, 0x00000000, 0xffffffff,
 290         0x000008f8, 0x00000028, 0xffffffff,
 291         0x000008fc, 0x00000000, 0xffffffff,
 292         0x000008f8, 0x00000029, 0xffffffff,
 293         0x000008fc, 0x00000000, 0xffffffff,
 294         0x000008f8, 0x0000002a, 0xffffffff,
 295         0x000008fc, 0x00000000, 0xffffffff,
 296         0x000008f8, 0x0000002b, 0xffffffff,
 297         0x000008fc, 0x00000000, 0xffffffff
 298 };
 299 #define CAYMAN_CGCG_CGLS_DEFAULT_LENGTH sizeof(cayman_cgcg_cgls_default) / (3 * sizeof(u32))
 300
 301 static const u32 cayman_cgcg_cgls_disable[] =
 302 {
 303         0x000008f8, 0x00000010, 0xffffffff,
 304         0x000008fc, 0xffffffff, 0xffffffff,
 305         0x000008f8, 0x00000011, 0xffffffff,
 306         0x000008fc, 0xffffffff, 0xffffffff,
 307         0x000008f8, 0x00000012, 0xffffffff,
 308         0x000008fc, 0xffffffff, 0xffffffff,
 309         0x000008f8, 0x00000013, 0xffffffff,
 310         0x000008fc, 0xffffffff, 0xffffffff,
 311         0x000008f8, 0x00000014, 0xffffffff,
 312         0x000008fc, 0xffffffff, 0xffffffff,
 313         0x000008f8, 0x00000015, 0xffffffff,
 314         0x000008fc, 0xffffffff, 0xffffffff,
 315         0x000008f8, 0x00000016, 0xffffffff,
 316         0x000008fc, 0xffffffff, 0xffffffff,
 317         0x000008f8, 0x00000017, 0xffffffff,
 318         0x000008fc, 0xffffffff, 0xffffffff,
 319         0x000008f8, 0x00000018, 0xffffffff,
 320         0x000008fc, 0xffffffff, 0xffffffff,
 321         0x000008f8, 0x00000019, 0xffffffff,
 322         0x000008fc, 0xffffffff, 0xffffffff,
 323         0x000008f8, 0x0000001a, 0xffffffff,
 324         0x000008fc, 0xffffffff, 0xffffffff,
 325         0x000008f8, 0x0000001b, 0xffffffff,
 326         0x000008fc, 0xffffffff, 0xffffffff,
 327         0x000008f8, 0x00000020, 0xffffffff,
 328         0x000008fc, 0x00000000, 0xffffffff,
 329         0x000008f8, 0x00000021, 0xffffffff,
 330         0x000008fc, 0x00000000, 0xffffffff,
 331         0x000008f8, 0x00000022, 0xffffffff,
 332         0x000008fc, 0x00000000, 0xffffffff,
 333         0x000008f8, 0x00000023, 0xffffffff,
 334         0x000008fc, 0x00000000, 0xffffffff,
 335         0x000008f8, 0x00000024, 0xffffffff,
 336         0x000008fc, 0x00000000, 0xffffffff,
 337         0x000008f8, 0x00000025, 0xffffffff,
 338         0x000008fc, 0x00000000, 0xffffffff,
 339         0x000008f8, 0x00000026, 0xffffffff,
 340         0x000008fc, 0x00000000, 0xffffffff,
 341         0x000008f8, 0x00000027, 0xffffffff,
 342         0x000008fc, 0x00000000, 0xffffffff,
 343         0x000008f8, 0x00000028, 0xffffffff,
 344         0x000008fc, 0x00000000, 0xffffffff,
 345         0x000008f8, 0x00000029, 0xffffffff,
 346         0x000008fc, 0x00000000, 0xffffffff,
 347         0x000008f8, 0x0000002a, 0xffffffff,
 348         0x000008fc, 0x00000000, 0xffffffff,
 349         0x000008f8, 0x0000002b, 0xffffffff,
 350         0x000008fc, 0x00000000, 0xffffffff,
 351         0x00000644, 0x000f7902, 0x001f4180,
 352         0x00000644, 0x000f3802, 0x001f4180
 353 };
 354 #define CAYMAN_CGCG_CGLS_DISABLE_LENGTH sizeof(cayman_cgcg_cgls_disable) / (3 * sizeof(u32))
 355
 356 static const u32 cayman_cgcg_cgls_enable[] =
 357 {
 358         0x00000644, 0x000f7882, 0x001f4080,
 359         0x000008f8, 0x00000010, 0xffffffff,
 360         0x000008fc, 0x00000000, 0xffffffff,
 361         0x000008f8, 0x00000011, 0xffffffff,
 362         0x000008fc, 0x00000000, 0xffffffff,
 363         0x000008f8, 0x00000012, 0xffffffff,
 364         0x000008fc, 0x00000000, 0xffffffff,
 365         0x000008f8, 0x00000013, 0xffffffff,
 366         0x000008fc, 0x00000000, 0xffffffff,
 367         0x000008f8, 0x00000014, 0xffffffff,
 368         0x000008fc, 0x00000000, 0xffffffff,
 369         0x000008f8, 0x00000015, 0xffffffff,
 370         0x000008fc, 0x00000000, 0xffffffff,
 371         0x000008f8, 0x00000016, 0xffffffff,
 372         0x000008fc, 0x00000000, 0xffffffff,
 373         0x000008f8, 0x00000017, 0xffffffff,
 374         0x000008fc, 0x00000000, 0xffffffff,
 375         0x000008f8, 0x00000018, 0xffffffff,
 376         0x000008fc, 0x00000000, 0xffffffff,
 377         0x000008f8, 0x00000019, 0xffffffff,
 378         0x000008fc, 0x00000000, 0xffffffff,
 379         0x000008f8, 0x0000001a, 0xffffffff,
 380         0x000008fc, 0x00000000, 0xffffffff,
 381         0x000008f8, 0x0000001b, 0xffffffff,
 382         0x000008fc, 0x00000000, 0xffffffff,
 383         0x000008f8, 0x00000020, 0xffffffff,
 384         0x000008fc, 0xffffffff, 0xffffffff,
 385         0x000008f8, 0x00000021, 0xffffffff,
 386         0x000008fc, 0xffffffff, 0xffffffff,
 387         0x000008f8, 0x00000022, 0xffffffff,
 388         0x000008fc, 0xffffffff, 0xffffffff,
 389         0x000008f8, 0x00000023, 0xffffffff,
 390         0x000008fc, 0xffffffff, 0xffffffff,
 391         0x000008f8, 0x00000024, 0xffffffff,
 392         0x000008fc, 0xffffffff, 0xffffffff,
 393         0x000008f8, 0x00000025, 0xffffffff,
 394         0x000008fc, 0xffffffff, 0xffffffff,
 395         0x000008f8, 0x00000026, 0xffffffff,
 396         0x000008fc, 0xffffffff, 0xffffffff,
 397         0x000008f8, 0x00000027, 0xffffffff,
 398         0x000008fc, 0xffffffff, 0xffffffff,
 399         0x000008f8, 0x00000028, 0xffffffff,
 400         0x000008fc, 0xffffffff, 0xffffffff,
 401         0x000008f8, 0x00000029, 0xffffffff,
 402         0x000008fc, 0xffffffff, 0xffffffff,
 403         0x000008f8, 0x0000002a, 0xffffffff,
 404         0x000008fc, 0xffffffff, 0xffffffff,
 405         0x000008f8, 0x0000002b, 0xffffffff,
 406         0x000008fc, 0xffffffff, 0xffffffff
 407 };
 408 #define CAYMAN_CGCG_CGLS_ENABLE_LENGTH  sizeof(cayman_cgcg_cgls_enable) / (3 * sizeof(u32))
 409
 410 static const u32 cayman_mgcg_default[] =
 411 {
 412         0x0000802c, 0xc0000000, 0xffffffff,
 413         0x00003fc4, 0xc0000000, 0xffffffff,
 414         0x00005448, 0x00000100, 0xffffffff,
 415         0x000055e4, 0x00000100, 0xffffffff,
 416         0x0000160c, 0x00000100, 0xffffffff,
 417         0x00008984, 0x06000100, 0xffffffff,
 418         0x0000c164, 0x00000100, 0xffffffff,
 419         0x00008a18, 0x00000100, 0xffffffff,
 420         0x0000897c, 0x06000100, 0xffffffff,
 421         0x00008b28, 0x00000100, 0xffffffff,
 422         0x00009144, 0x00800200, 0xffffffff,
 423         0x00009a60, 0x00000100, 0xffffffff,
 424         0x00009868, 0x00000100, 0xffffffff,
 425         0x00008d58, 0x00000100, 0xffffffff,
 426         0x00009510, 0x00000100, 0xffffffff,
 427         0x0000949c, 0x00000100, 0xffffffff,
 428         0x00009654, 0x00000100, 0xffffffff,
 429         0x00009030, 0x00000100, 0xffffffff,
 430         0x00009034, 0x00000100, 0xffffffff,
 431         0x00009038, 0x00000100, 0xffffffff,
 432         0x0000903c, 0x00000100, 0xffffffff,
 433         0x00009040, 0x00000100, 0xffffffff,
 434         0x0000a200, 0x00000100, 0xffffffff,
 435         0x0000a204, 0x00000100, 0xffffffff,
 436         0x0000a208, 0x00000100, 0xffffffff,
 437         0x0000a20c, 0x00000100, 0xffffffff,
 438         0x00009744, 0x00000100, 0xffffffff,
 439         0x00003f80, 0x00000100, 0xffffffff,
 440         0x0000a210, 0x00000100, 0xffffffff,
 441         0x0000a214, 0x00000100, 0xffffffff,
 442         0x000004d8, 0x00000100, 0xffffffff,
 443         0x00009664, 0x00000100, 0xffffffff,
 444         0x00009698, 0x00000100, 0xffffffff,
 445         0x000004d4, 0x00000200, 0xffffffff,
 446         0x000004d0, 0x00000000, 0xffffffff,
 447         0x000030cc, 0x00000104, 0xffffffff,
 448         0x0000d0c0, 0x00000100, 0xffffffff,
 449         0x0000d8c0, 0x00000100, 0xffffffff,
 450         0x0000802c, 0x40000000, 0xffffffff,
 451         0x00003fc4, 0x40000000, 0xffffffff,
 452         0x0000915c, 0x00010000, 0xffffffff,
 453         0x00009160, 0x00030002, 0xffffffff,
 454         0x00009164, 0x00050004, 0xffffffff,
 455         0x00009168, 0x00070006, 0xffffffff,
 456         0x00009178, 0x00070000, 0xffffffff,
 457         0x0000917c, 0x00030002, 0xffffffff,
 458         0x00009180, 0x00050004, 0xffffffff,
 459         0x0000918c, 0x00010006, 0xffffffff,
 460         0x00009190, 0x00090008, 0xffffffff,
 461         0x00009194, 0x00070000, 0xffffffff,
 462         0x00009198, 0x00030002, 0xffffffff,
 463         0x0000919c, 0x00050004, 0xffffffff,
 464         0x000091a8, 0x00010006, 0xffffffff,
 465         0x000091ac, 0x00090008, 0xffffffff,
 466         0x000091b0, 0x00070000, 0xffffffff,
 467         0x000091b4, 0x00030002, 0xffffffff,
 468         0x000091b8, 0x00050004, 0xffffffff,
 469         0x000091c4, 0x00010006, 0xffffffff,
 470         0x000091c8, 0x00090008, 0xffffffff,
 471         0x000091cc, 0x00070000, 0xffffffff,
 472         0x000091d0, 0x00030002, 0xffffffff,
 473         0x000091d4, 0x00050004, 0xffffffff,
 474         0x000091e0, 0x00010006, 0xffffffff,
 475         0x000091e4, 0x00090008, 0xffffffff,
 476         0x000091e8, 0x00000000, 0xffffffff,
 477         0x000091ec, 0x00070000, 0xffffffff,
 478         0x000091f0, 0x00030002, 0xffffffff,
 479         0x000091f4, 0x00050004, 0xffffffff,
 480         0x00009200, 0x00010006, 0xffffffff,
 481         0x00009204, 0x00090008, 0xffffffff,
 482         0x00009208, 0x00070000, 0xffffffff,
 483         0x0000920c, 0x00030002, 0xffffffff,
 484         0x00009210, 0x00050004, 0xffffffff,
 485         0x0000921c, 0x00010006, 0xffffffff,
 486         0x00009220, 0x00090008, 0xffffffff,
 487         0x00009224, 0x00070000, 0xffffffff,
 488         0x00009228, 0x00030002, 0xffffffff,
 489         0x0000922c, 0x00050004, 0xffffffff,
 490         0x00009238, 0x00010006, 0xffffffff,
 491         0x0000923c, 0x00090008, 0xffffffff,
 492         0x00009240, 0x00070000, 0xffffffff,
 493         0x00009244, 0x00030002, 0xffffffff,
 494         0x00009248, 0x00050004, 0xffffffff,
 495         0x00009254, 0x00010006, 0xffffffff,
 496         0x00009258, 0x00090008, 0xffffffff,
 497         0x0000925c, 0x00070000, 0xffffffff,
 498         0x00009260, 0x00030002, 0xffffffff,
 499         0x00009264, 0x00050004, 0xffffffff,
 500         0x00009270, 0x00010006, 0xffffffff,
 501         0x00009274, 0x00090008, 0xffffffff,
 502         0x00009278, 0x00070000, 0xffffffff,
 503         0x0000927c, 0x00030002, 0xffffffff,
 504         0x00009280, 0x00050004, 0xffffffff,
 505         0x0000928c, 0x00010006, 0xffffffff,
 506         0x00009290, 0x00090008, 0xffffffff,
 507         0x000092a8, 0x00070000, 0xffffffff,
 508         0x000092ac, 0x00030002, 0xffffffff,
 509         0x000092b0, 0x00050004, 0xffffffff,
 510         0x000092bc, 0x00010006, 0xffffffff,
 511         0x000092c0, 0x00090008, 0xffffffff,
 512         0x000092c4, 0x00070000, 0xffffffff,
 513         0x000092c8, 0x00030002, 0xffffffff,
 514         0x000092cc, 0x00050004, 0xffffffff,
 515         0x000092d8, 0x00010006, 0xffffffff,
 516         0x000092dc, 0x00090008, 0xffffffff,
 517         0x00009294, 0x00000000, 0xffffffff,
 518         0x0000802c, 0x40010000, 0xffffffff,
 519         0x00003fc4, 0x40010000, 0xffffffff,
 520         0x0000915c, 0x00010000, 0xffffffff,
 521         0x00009160, 0x00030002, 0xffffffff,
 522         0x00009164, 0x00050004, 0xffffffff,
 523         0x00009168, 0x00070006, 0xffffffff,
 524         0x00009178, 0x00070000, 0xffffffff,
 525         0x0000917c, 0x00030002, 0xffffffff,
 526         0x00009180, 0x00050004, 0xffffffff,
 527         0x0000918c, 0x00010006, 0xffffffff,
 528         0x00009190, 0x00090008, 0xffffffff,
 529         0x00009194, 0x00070000, 0xffffffff,
 530         0x00009198, 0x00030002, 0xffffffff,
 531         0x0000919c, 0x00050004, 0xffffffff,
 532         0x000091a8, 0x00010006, 0xffffffff,
 533         0x000091ac, 0x00090008, 0xffffffff,
 534         0x000091b0, 0x00070000, 0xffffffff,
 535         0x000091b4, 0x00030002, 0xffffffff,
 536         0x000091b8, 0x00050004, 0xffffffff,
 537         0x000091c4, 0x00010006, 0xffffffff,
 538         0x000091c8, 0x00090008, 0xffffffff,
 539         0x000091cc, 0x00070000, 0xffffffff,
 540         0x000091d0, 0x00030002, 0xffffffff,
 541         0x000091d4, 0x00050004, 0xffffffff,
 542         0x000091e0, 0x00010006, 0xffffffff,
 543         0x000091e4, 0x00090008, 0xffffffff,
 544         0x000091e8, 0x00000000, 0xffffffff,
 545         0x000091ec, 0x00070000, 0xffffffff,
 546         0x000091f0, 0x00030002, 0xffffffff,
 547         0x000091f4, 0x00050004, 0xffffffff,
 548         0x00009200, 0x00010006, 0xffffffff,
 549         0x00009204, 0x00090008, 0xffffffff,
 550         0x00009208, 0x00070000, 0xffffffff,
 551         0x0000920c, 0x00030002, 0xffffffff,
 552         0x00009210, 0x00050004, 0xffffffff,
 553         0x0000921c, 0x00010006, 0xffffffff,
 554         0x00009220, 0x00090008, 0xffffffff,
 555         0x00009224, 0x00070000, 0xffffffff,
 556         0x00009228, 0x00030002, 0xffffffff,
 557         0x0000922c, 0x00050004, 0xffffffff,
 558         0x00009238, 0x00010006, 0xffffffff,
 559         0x0000923c, 0x00090008, 0xffffffff,
 560         0x00009240, 0x00070000, 0xffffffff,
 561         0x00009244, 0x00030002, 0xffffffff,
 562         0x00009248, 0x00050004, 0xffffffff,
 563         0x00009254, 0x00010006, 0xffffffff,
 564         0x00009258, 0x00090008, 0xffffffff,
 565         0x0000925c, 0x00070000, 0xffffffff,
 566         0x00009260, 0x00030002, 0xffffffff,
 567         0x00009264, 0x00050004, 0xffffffff,
 568         0x00009270, 0x00010006, 0xffffffff,
 569         0x00009274, 0x00090008, 0xffffffff,
 570         0x00009278, 0x00070000, 0xffffffff,
 571         0x0000927c, 0x00030002, 0xffffffff,
 572         0x00009280, 0x00050004, 0xffffffff,
 573         0x0000928c, 0x00010006, 0xffffffff,
 574         0x00009290, 0x00090008, 0xffffffff,
 575         0x000092a8, 0x00070000, 0xffffffff,
 576         0x000092ac, 0x00030002, 0xffffffff,
 577         0x000092b0, 0x00050004, 0xffffffff,
 578         0x000092bc, 0x00010006, 0xffffffff,
 579         0x000092c0, 0x00090008, 0xffffffff,
 580         0x000092c4, 0x00070000, 0xffffffff,
 581         0x000092c8, 0x00030002, 0xffffffff,
 582         0x000092cc, 0x00050004, 0xffffffff,
 583         0x000092d8, 0x00010006, 0xffffffff,
 584         0x000092dc, 0x00090008, 0xffffffff,
 585         0x00009294, 0x00000000, 0xffffffff,
 586         0x0000802c, 0xc0000000, 0xffffffff,
 587         0x00003fc4, 0xc0000000, 0xffffffff,
 588         0x000008f8, 0x00000010, 0xffffffff,
 589         0x000008fc, 0x00000000, 0xffffffff,
 590         0x000008f8, 0x00000011, 0xffffffff,
 591         0x000008fc, 0x00000000, 0xffffffff,
 592         0x000008f8, 0x00000012, 0xffffffff,
 593         0x000008fc, 0x00000000, 0xffffffff,
 594         0x000008f8, 0x00000013, 0xffffffff,
 595         0x000008fc, 0x00000000, 0xffffffff,
 596         0x000008f8, 0x00000014, 0xffffffff,
 597         0x000008fc, 0x00000000, 0xffffffff,
 598         0x000008f8, 0x00000015, 0xffffffff,
 599         0x000008fc, 0x00000000, 0xffffffff,
 600         0x000008f8, 0x00000016, 0xffffffff,
 601         0x000008fc, 0x00000000, 0xffffffff,
 602         0x000008f8, 0x00000017, 0xffffffff,
 603         0x000008fc, 0x00000000, 0xffffffff,
 604         0x000008f8, 0x00000018, 0xffffffff,
 605         0x000008fc, 0x00000000, 0xffffffff,
 606         0x000008f8, 0x00000019, 0xffffffff,
 607         0x000008fc, 0x00000000, 0xffffffff,
 608         0x000008f8, 0x0000001a, 0xffffffff,
 609         0x000008fc, 0x00000000, 0xffffffff,
 610         0x000008f8, 0x0000001b, 0xffffffff,
 611         0x000008fc, 0x00000000, 0xffffffff
 612 };
 613 #define CAYMAN_MGCG_DEFAULT_LENGTH sizeof(cayman_mgcg_default) / (3 * sizeof(u32))
 614
 615 static const u32 cayman_mgcg_disable[] =
 616 {
 617         0x0000802c, 0xc0000000, 0xffffffff,
 618         0x000008f8, 0x00000000, 0xffffffff,
 619         0x000008fc, 0xffffffff, 0xffffffff,
 620         0x000008f8, 0x00000001, 0xffffffff,
 621         0x000008fc, 0xffffffff, 0xffffffff,
 622         0x000008f8, 0x00000002, 0xffffffff,
 623         0x000008fc, 0xffffffff, 0xffffffff,
 624         0x000008f8, 0x00000003, 0xffffffff,
 625         0x000008fc, 0xffffffff, 0xffffffff,
 626         0x00009150, 0x00600000, 0xffffffff
 627 };
 628 #define CAYMAN_MGCG_DISABLE_LENGTH   sizeof(cayman_mgcg_disable) / (3 * sizeof(u32))
 629
 630 static const u32 cayman_mgcg_enable[] =
 631 {
 632         0x0000802c, 0xc0000000, 0xffffffff,
 633         0x000008f8, 0x00000000, 0xffffffff,
 634         0x000008fc, 0x00000000, 0xffffffff,
 635         0x000008f8, 0x00000001, 0xffffffff,
 636         0x000008fc, 0x00000000, 0xffffffff,
 637         0x000008f8, 0x00000002, 0xffffffff,
 638         0x000008fc, 0x00600000, 0xffffffff,
 639         0x000008f8, 0x00000003, 0xffffffff,
 640         0x000008fc, 0x00000000, 0xffffffff,
 641         0x00009150, 0x96944200, 0xffffffff
 642 };
 643
 644 #define CAYMAN_MGCG_ENABLE_LENGTH   sizeof(cayman_mgcg_enable) / (3 * sizeof(u32))
 645
 646 #define NISLANDS_SYSLS_SEQUENCE  100
 647
 648 static const u32 cayman_sysls_default[] =
 649 {
 650         /* Register,   Value,     Mask bits */
 651         0x000055e8, 0x00000000, 0xffffffff,
 652         0x0000d0bc, 0x00000000, 0xffffffff,
 653         0x0000d8bc, 0x00000000, 0xffffffff,
 654         0x000015c0, 0x000c1401, 0xffffffff,
 655         0x0000264c, 0x000c0400, 0xffffffff,
 656         0x00002648, 0x000c0400, 0xffffffff,
 657         0x00002650, 0x000c0400, 0xffffffff,
 658         0x000020b8, 0x000c0400, 0xffffffff,
 659         0x000020bc, 0x000c0400, 0xffffffff,
 660         0x000020c0, 0x000c0c80, 0xffffffff,
 661         0x0000f4a0, 0x000000c0, 0xffffffff,
 662         0x0000f4a4, 0x00680fff, 0xffffffff,
 663         0x00002f50, 0x00000404, 0xffffffff,
 664         0x000004c8, 0x00000001, 0xffffffff,
 665         0x000064ec, 0x00000000, 0xffffffff,
 666         0x00000c7c, 0x00000000, 0xffffffff,
 667         0x00008dfc, 0x00000000, 0xffffffff
 668 };
 669 #define CAYMAN_SYSLS_DEFAULT_LENGTH sizeof(cayman_sysls_default) / (3 * sizeof(u32))
 670
 671 static const u32 cayman_sysls_disable[] =
 672 {
 673         /* Register,   Value,     Mask bits */
 674         0x0000d0c0, 0x00000000, 0xffffffff,
 675         0x0000d8c0, 0x00000000, 0xffffffff,
 676         0x000055e8, 0x00000000, 0xffffffff,
 677         0x0000d0bc, 0x00000000, 0xffffffff,
 678         0x0000d8bc, 0x00000000, 0xffffffff,
 679         0x000015c0, 0x00041401, 0xffffffff,
 680         0x0000264c, 0x00040400, 0xffffffff,
 681         0x00002648, 0x00040400, 0xffffffff,
 682         0x00002650, 0x00040400, 0xffffffff,
 683         0x000020b8, 0x00040400, 0xffffffff,
 684         0x000020bc, 0x00040400, 0xffffffff,
 685         0x000020c0, 0x00040c80, 0xffffffff,
 686         0x0000f4a0, 0x000000c0, 0xffffffff,
 687         0x0000f4a4, 0x00680000, 0xffffffff,
 688         0x00002f50, 0x00000404, 0xffffffff,
 689         0x000004c8, 0x00000001, 0xffffffff,
 690         0x000064ec, 0x00007ffd, 0xffffffff,
 691         0x00000c7c, 0x0000ff00, 0xffffffff,
 692         0x00008dfc, 0x0000007f, 0xffffffff
 693 };
 694 #define CAYMAN_SYSLS_DISABLE_LENGTH sizeof(cayman_sysls_disable) / (3 * sizeof(u32))
 695
 696 static const u32 cayman_sysls_enable[] =
 697 {
 698         /* Register,   Value,     Mask bits */
 699         0x000055e8, 0x00000001, 0xffffffff,
 700         0x0000d0bc, 0x00000100, 0xffffffff,
 701         0x0000d8bc, 0x00000100, 0xffffffff,
 702         0x000015c0, 0x000c1401, 0xffffffff,
 703         0x0000264c, 0x000c0400, 0xffffffff,
 704         0x00002648, 0x000c0400, 0xffffffff,
 705         0x00002650, 0x000c0400, 0xffffffff,
 706         0x000020b8, 0x000c0400, 0xffffffff,
 707         0x000020bc, 0x000c0400, 0xffffffff,
 708         0x000020c0, 0x000c0c80, 0xffffffff,
 709         0x0000f4a0, 0x000000c0, 0xffffffff,
 710         0x0000f4a4, 0x00680fff, 0xffffffff,
 711         0x00002f50, 0x00000903, 0xffffffff,
 712         0x000004c8, 0x00000000, 0xffffffff,
 713         0x000064ec, 0x00000000, 0xffffffff,
 714         0x00000c7c, 0x00000000, 0xffffffff,
 715         0x00008dfc, 0x00000000, 0xffffffff
 716 };
 717 #define CAYMAN_SYSLS_ENABLE_LENGTH sizeof(cayman_sysls_enable) / (3 * sizeof(u32))
 718
 719 struct rv7xx_power_info *rv770_get_pi(struct radeon_device *rdev);
 720 struct evergreen_power_info *evergreen_get_pi(struct radeon_device *rdev);
 721
 722 static struct ni_power_info *ni_get_pi(struct radeon_device *rdev)
 723 {
 724         struct ni_power_info *pi = rdev->pm.dpm.priv;
 725
 726         return pi;
 727 }
 728
 729 struct ni_ps *ni_get_ps(struct radeon_ps *rps)
 730 {
 731         struct ni_ps *ps = rps->ps_priv;
 732
 733         return ps;
 734 }
 735
 736 static void ni_calculate_leakage_for_v_and_t_formula(const struct ni_leakage_coeffients *coeff,
 737                                                      u16 v, s32 t,
 738                                                      u32 ileakage,
 739                                                      u32 *leakage)
 740 {
 741         s64 kt, kv, leakage_w, i_leakage, vddc, temperature;
 742
 743         i_leakage = div64_s64(drm_int2fixp(ileakage), 1000);
 744         vddc = div64_s64(drm_int2fixp(v), 1000);
 745         temperature = div64_s64(drm_int2fixp(t), 1000);
 746
 747         kt = drm_fixp_mul(div64_s64(drm_int2fixp(coeff->at), 1000),
 748                           drm_fixp_exp(drm_fixp_mul(div64_s64(drm_int2fixp(coeff->bt), 1000), temperature)));
 749         kv = drm_fixp_mul(div64_s64(drm_int2fixp(coeff->av), 1000),
 750                           drm_fixp_exp(drm_fixp_mul(div64_s64(drm_int2fixp(coeff->bv), 1000), vddc)));
 751
 752         leakage_w = drm_fixp_mul(drm_fixp_mul(drm_fixp_mul(i_leakage, kt), kv), vddc);
 753
 754         *leakage = drm_fixp2int(leakage_w * 1000);
 755 }
 756
 757 static void ni_calculate_leakage_for_v_and_t(struct radeon_device *rdev,
 758                                              const struct ni_leakage_coeffients *coeff,
 759                                              u16 v,
 760                                              s32 t,
 761                                              u32 i_leakage,
 762                                              u32 *leakage)
 763 {
 764         ni_calculate_leakage_for_v_and_t_formula(coeff, v, t, i_leakage, leakage);
 765 }
 766
 767 static void ni_apply_state_adjust_rules(struct radeon_device *rdev,
 768                                         struct radeon_ps *rps)
 769 {
 770         struct ni_ps *ps = ni_get_ps(rps);
 771         struct radeon_clock_and_voltage_limits *max_limits;
 772         bool disable_mclk_switching;
 773         u32 mclk, sclk;
 774         u16 vddc, vddci;
 775         int i;
 776
 777         if (rdev->pm.dpm.new_active_crtc_count > 1)
 778                 disable_mclk_switching = true;
 779         else
 780                 disable_mclk_switching = false;
 781
 782         if (rdev->pm.dpm.ac_power)
 783                 max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
 784         else
 785                 max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc;
 786
 787         if (rdev->pm.dpm.ac_power == false) {
 788                 for (i = 0; i < ps->performance_level_count; i++) {
 789                         if (ps->performance_levels[i].mclk > max_limits->mclk)
 790                                 ps->performance_levels[i].mclk = max_limits->mclk;
 791                         if (ps->performance_levels[i].sclk > max_limits->sclk)
 792                                 ps->performance_levels[i].sclk = max_limits->sclk;
 793                         if (ps->performance_levels[i].vddc > max_limits->vddc)
 794                                 ps->performance_levels[i].vddc = max_limits->vddc;
 795                         if (ps->performance_levels[i].vddci > max_limits->vddci)
 796                                 ps->performance_levels[i].vddci = max_limits->vddci;
 797                 }
 798         }
 799
 800         /* XXX validate the min clocks required for display */
 801
 802         if (disable_mclk_switching) {
 803                 mclk  = ps->performance_levels[ps->performance_level_count - 1].mclk;
 804                 sclk = ps->performance_levels[0].sclk;
 805                 vddc = ps->performance_levels[0].vddc;
 806                 vddci = ps->performance_levels[ps->performance_level_count - 1].vddci;
 807         } else {
 808                 sclk = ps->performance_levels[0].sclk;
 809                 mclk = ps->performance_levels[0].mclk;
 810                 vddc = ps->performance_levels[0].vddc;
 811                 vddci = ps->performance_levels[0].vddci;
 812         }
 813
 814         /* adjusted low state */
 815         ps->performance_levels[0].sclk = sclk;
 816         ps->performance_levels[0].mclk = mclk;
 817         ps->performance_levels[0].vddc = vddc;
 818         ps->performance_levels[0].vddci = vddci;
 819
 820         btc_skip_blacklist_clocks(rdev, max_limits->sclk, max_limits->mclk,
 821                                   &ps->performance_levels[0].sclk,
 822                                   &ps->performance_levels[0].mclk);
 823
 824         for (i = 1; i < ps->performance_level_count; i++) {
 825                 if (ps->performance_levels[i].sclk < ps->performance_levels[i - 1].sclk)
 826                         ps->performance_levels[i].sclk = ps->performance_levels[i - 1].sclk;
 827                 if (ps->performance_levels[i].vddc < ps->performance_levels[i - 1].vddc)
 828                         ps->performance_levels[i].vddc = ps->performance_levels[i - 1].vddc;
 829         }
 830
 831         if (disable_mclk_switching) {
 832                 mclk = ps->performance_levels[0].mclk;
 833                 for (i = 1; i < ps->performance_level_count; i++) {
 834                         if (mclk < ps->performance_levels[i].mclk)
 835                                 mclk = ps->performance_levels[i].mclk;
 836                 }
 837                 for (i = 0; i < ps->performance_level_count; i++) {
 838                         ps->performance_levels[i].mclk = mclk;
 839                         ps->performance_levels[i].vddci = vddci;
 840                 }
 841         } else {
 842                 for (i = 1; i < ps->performance_level_count; i++) {
 843                         if (ps->performance_levels[i].mclk < ps->performance_levels[i - 1].mclk)
 844                                 ps->performance_levels[i].mclk = ps->performance_levels[i - 1].mclk;
 845                         if (ps->performance_levels[i].vddci < ps->performance_levels[i - 1].vddci)
 846                                 ps->performance_levels[i].vddci = ps->performance_levels[i - 1].vddci;
 847                 }
 848         }
 849
 850         for (i = 1; i < ps->performance_level_count; i++)
 851                 btc_skip_blacklist_clocks(rdev, max_limits->sclk, max_limits->mclk,
 852                                           &ps->performance_levels[i].sclk,
 853                                           &ps->performance_levels[i].mclk);
 854
 855         for (i = 0; i < ps->performance_level_count; i++)
 856                 btc_adjust_clock_combinations(rdev, max_limits,
 857                                               &ps->performance_levels[i]);
 858
 859         for (i = 0; i < ps->performance_level_count; i++) {
 860                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk,
 861                                                    ps->performance_levels[i].sclk,
 862                                                    max_limits->vddc,  &ps->performance_levels[i].vddc);
 863                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk,
 864                                                    ps->performance_levels[i].mclk,
 865                                                    max_limits->vddci, &ps->performance_levels[i].vddci);
 866                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk,
 867                                                    ps->performance_levels[i].mclk,
 868                                                    max_limits->vddc,  &ps->performance_levels[i].vddc);
 869                 /* XXX validate the voltage required for display */
 870         }
 871
 872         for (i = 0; i < ps->performance_level_count; i++) {
 873                 btc_apply_voltage_delta_rules(rdev,
 874                                               max_limits->vddc, max_limits->vddci,
 875                                               &ps->performance_levels[i].vddc,
 876                                               &ps->performance_levels[i].vddci);
 877         }
 878
 879         ps->dc_compatible = true;
 880         for (i = 0; i < ps->performance_level_count; i++) {
 881                 if (ps->performance_levels[i].vddc > rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.vddc)
 882                         ps->dc_compatible = false;
 883
 884                 if (ps->performance_levels[i].vddc < rdev->pm.dpm.dyn_state.min_vddc_for_pcie_gen2)
 885                         ps->performance_levels[i].flags &= ~ATOM_PPLIB_R600_FLAGS_PCIEGEN2;
 886         }
 887 }
 888
 889 static void ni_cg_clockgating_default(struct radeon_device *rdev)
 890 {
 891         u32 count;
 892         const u32 *ps = NULL;
 893
 894         ps = (const u32 *)&cayman_cgcg_cgls_default;
 895         count = CAYMAN_CGCG_CGLS_DEFAULT_LENGTH;
 896
 897         btc_program_mgcg_hw_sequence(rdev, ps, count);
 898 }
 899
 900 static void ni_gfx_clockgating_enable(struct radeon_device *rdev,
 901                                       bool enable)
 902 {
 903         u32 count;
 904         const u32 *ps = NULL;
 905
 906         if (enable) {
 907                 ps = (const u32 *)&cayman_cgcg_cgls_enable;
 908                 count = CAYMAN_CGCG_CGLS_ENABLE_LENGTH;
 909         } else {
 910                 ps = (const u32 *)&cayman_cgcg_cgls_disable;
 911                 count = CAYMAN_CGCG_CGLS_DISABLE_LENGTH;
 912         }
 913
 914         btc_program_mgcg_hw_sequence(rdev, ps, count);
 915 }
 916
 917 static void ni_mg_clockgating_default(struct radeon_device *rdev)
 918 {
 919         u32 count;
 920         const u32 *ps = NULL;
 921
 922         ps = (const u32 *)&cayman_mgcg_default;
 923         count = CAYMAN_MGCG_DEFAULT_LENGTH;
 924
 925         btc_program_mgcg_hw_sequence(rdev, ps, count);
 926 }
 927
 928 static void ni_mg_clockgating_enable(struct radeon_device *rdev,
 929                                      bool enable)
 930 {
 931         u32 count;
 932         const u32 *ps = NULL;
 933
 934         if (enable) {
 935                 ps = (const u32 *)&cayman_mgcg_enable;
 936                 count = CAYMAN_MGCG_ENABLE_LENGTH;
 937         } else {
 938                 ps = (const u32 *)&cayman_mgcg_disable;
 939                 count = CAYMAN_MGCG_DISABLE_LENGTH;
 940         }
 941
 942         btc_program_mgcg_hw_sequence(rdev, ps, count);
 943 }
 944
 945 static void ni_ls_clockgating_default(struct radeon_device *rdev)
 946 {
 947         u32 count;
 948         const u32 *ps = NULL;
 949
 950         ps = (const u32 *)&cayman_sysls_default;
 951         count = CAYMAN_SYSLS_DEFAULT_LENGTH;
 952
 953         btc_program_mgcg_hw_sequence(rdev, ps, count);
 954 }
 955
 956 static void ni_ls_clockgating_enable(struct radeon_device *rdev,
 957                                      bool enable)
 958 {
 959         u32 count;
 960         const u32 *ps = NULL;
 961
 962         if (enable) {
 963                 ps = (const u32 *)&cayman_sysls_enable;
 964                 count = CAYMAN_SYSLS_ENABLE_LENGTH;
 965         } else {
 966                 ps = (const u32 *)&cayman_sysls_disable;
 967                 count = CAYMAN_SYSLS_DISABLE_LENGTH;
 968         }
 969
 970         btc_program_mgcg_hw_sequence(rdev, ps, count);
 971
 972 }
 973
 974 static int ni_patch_single_dependency_table_based_on_leakage(struct radeon_device *rdev,
 975                                                              struct radeon_clock_voltage_dependency_table *table)
 976 {
 977         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
 978         u32 i;
 979
 980         if (table) {
 981                 for (i = 0; i < table->count; i++) {
 982                         if (0xff01 == table->entries[i].v) {
 983                                 if (pi->max_vddc == 0)
 984                                         return -EINVAL;
 985                                 table->entries[i].v = pi->max_vddc;
 986                         }
 987                 }
 988         }
 989         return 0;
 990 }
 991
 992 static int ni_patch_dependency_tables_based_on_leakage(struct radeon_device *rdev)
 993 {
 994         int ret = 0;
 995
 996         ret = ni_patch_single_dependency_table_based_on_leakage(rdev,
 997                                                                 &rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk);
 998
 999         ret = ni_patch_single_dependency_table_based_on_leakage(rdev,
1000                                                                 &rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk);
1001         return ret;
1002 }
1003
1004 static void ni_stop_dpm(struct radeon_device *rdev)
1005 {
1006         WREG32_P(GENERAL_PWRMGT, 0, ~GLOBAL_PWRMGT_EN);
1007 }
1008
1009 #if 0
1010 static int ni_notify_hw_of_power_source(struct radeon_device *rdev,
1011                                         bool ac_power)
1012 {
1013         if (ac_power)
1014                 return (rv770_send_msg_to_smc(rdev, PPSMC_MSG_RunningOnAC) == PPSMC_Result_OK) ?
1015                         0 : -EINVAL;
1016
1017         return 0;
1018 }
1019 #endif
1020
1021 static PPSMC_Result ni_send_msg_to_smc_with_parameter(struct radeon_device *rdev,
1022                                                       PPSMC_Msg msg, u32 parameter)
1023 {
1024         WREG32(SMC_SCRATCH0, parameter);
1025         return rv770_send_msg_to_smc(rdev, msg);
1026 }
1027
1028 static int ni_restrict_performance_levels_before_switch(struct radeon_device *rdev)
1029 {
1030         if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_NoForcedLevel) != PPSMC_Result_OK)
1031                 return -EINVAL;
1032
1033         return (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 1) == PPSMC_Result_OK) ?
1034                 0 : -EINVAL;
1035 }
1036
1037 #if 0
1038 static int ni_unrestrict_performance_levels_after_switch(struct radeon_device *rdev)
1039 {
1040         if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 0) != PPSMC_Result_OK)
1041                 return -EINVAL;
1042
1043         return (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 0) == PPSMC_Result_OK) ?
1044                 0 : -EINVAL;
1045 }
1046 #endif
1047
1048 static void ni_stop_smc(struct radeon_device *rdev)
1049 {
1050         u32 tmp;
1051         int i;
1052
1053         for (i = 0; i < rdev->usec_timeout; i++) {
1054                 tmp = RREG32(LB_SYNC_RESET_SEL) & LB_SYNC_RESET_SEL_MASK;
1055                 if (tmp != 1)
1056                         break;
1057                 udelay(1);
1058         }
1059
1060         udelay(100);
1061
1062         r7xx_stop_smc(rdev);
1063 }
1064
1065 static int ni_process_firmware_header(struct radeon_device *rdev)
1066 {
1067         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1068         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1069         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1070         u32 tmp;
1071         int ret;
1072
1073         ret = rv770_read_smc_sram_dword(rdev,
1074                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1075                                         NISLANDS_SMC_FIRMWARE_HEADER_stateTable,
1076                                         &tmp, pi->sram_end);
1077
1078         if (ret)
1079                 return ret;
1080
1081         pi->state_table_start = (u16)tmp;
1082
1083         ret = rv770_read_smc_sram_dword(rdev,
1084                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1085                                         NISLANDS_SMC_FIRMWARE_HEADER_softRegisters,
1086                                         &tmp, pi->sram_end);
1087
1088         if (ret)
1089                 return ret;
1090
1091         pi->soft_regs_start = (u16)tmp;
1092
1093         ret = rv770_read_smc_sram_dword(rdev,
1094                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1095                                         NISLANDS_SMC_FIRMWARE_HEADER_mcRegisterTable,
1096                                         &tmp, pi->sram_end);
1097
1098         if (ret)
1099                 return ret;
1100
1101         eg_pi->mc_reg_table_start = (u16)tmp;
1102
1103         ret = rv770_read_smc_sram_dword(rdev,
1104                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1105                                         NISLANDS_SMC_FIRMWARE_HEADER_fanTable,
1106                                         &tmp, pi->sram_end);
1107
1108         if (ret)
1109                 return ret;
1110
1111         ni_pi->fan_table_start = (u16)tmp;
1112
1113         ret = rv770_read_smc_sram_dword(rdev,
1114                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1115                                         NISLANDS_SMC_FIRMWARE_HEADER_mcArbDramAutoRefreshTable,
1116                                         &tmp, pi->sram_end);
1117
1118         if (ret)
1119                 return ret;
1120
1121         ni_pi->arb_table_start = (u16)tmp;
1122
1123         ret = rv770_read_smc_sram_dword(rdev,
1124                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1125                                         NISLANDS_SMC_FIRMWARE_HEADER_cacTable,
1126                                         &tmp, pi->sram_end);
1127
1128         if (ret)
1129                 return ret;
1130
1131         ni_pi->cac_table_start = (u16)tmp;
1132
1133         ret = rv770_read_smc_sram_dword(rdev,
1134                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1135                                         NISLANDS_SMC_FIRMWARE_HEADER_spllTable,
1136                                         &tmp, pi->sram_end);
1137
1138         if (ret)
1139                 return ret;
1140
1141         ni_pi->spll_table_start = (u16)tmp;
1142
1143
1144         return ret;
1145 }
1146
1147 static void ni_read_clock_registers(struct radeon_device *rdev)
1148 {
1149         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1150
1151         ni_pi->clock_registers.cg_spll_func_cntl = RREG32(CG_SPLL_FUNC_CNTL);
1152         ni_pi->clock_registers.cg_spll_func_cntl_2 = RREG32(CG_SPLL_FUNC_CNTL_2);
1153         ni_pi->clock_registers.cg_spll_func_cntl_3 = RREG32(CG_SPLL_FUNC_CNTL_3);
1154         ni_pi->clock_registers.cg_spll_func_cntl_4 = RREG32(CG_SPLL_FUNC_CNTL_4);
1155         ni_pi->clock_registers.cg_spll_spread_spectrum = RREG32(CG_SPLL_SPREAD_SPECTRUM);
1156         ni_pi->clock_registers.cg_spll_spread_spectrum_2 = RREG32(CG_SPLL_SPREAD_SPECTRUM_2);
1157         ni_pi->clock_registers.mpll_ad_func_cntl = RREG32(MPLL_AD_FUNC_CNTL);
1158         ni_pi->clock_registers.mpll_ad_func_cntl_2 = RREG32(MPLL_AD_FUNC_CNTL_2);
1159         ni_pi->clock_registers.mpll_dq_func_cntl = RREG32(MPLL_DQ_FUNC_CNTL);
1160         ni_pi->clock_registers.mpll_dq_func_cntl_2 = RREG32(MPLL_DQ_FUNC_CNTL_2);
1161         ni_pi->clock_registers.mclk_pwrmgt_cntl = RREG32(MCLK_PWRMGT_CNTL);
1162         ni_pi->clock_registers.dll_cntl = RREG32(DLL_CNTL);
1163         ni_pi->clock_registers.mpll_ss1 = RREG32(MPLL_SS1);
1164         ni_pi->clock_registers.mpll_ss2 = RREG32(MPLL_SS2);
1165 }
1166
1167 #if 0
1168 static int ni_enter_ulp_state(struct radeon_device *rdev)
1169 {
1170         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1171
1172         if (pi->gfx_clock_gating) {
1173                 WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN);
1174                 WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON);
1175                 WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON);
1176                 RREG32(GB_ADDR_CONFIG);
1177         }
1178
1179         WREG32_P(SMC_MSG, HOST_SMC_MSG(PPSMC_MSG_SwitchToMinimumPower),
1180                  ~HOST_SMC_MSG_MASK);
1181
1182         udelay(25000);
1183
1184         return 0;
1185 }
1186 #endif
1187
1188 static void ni_program_response_times(struct radeon_device *rdev)
1189 {
1190         u32 voltage_response_time, backbias_response_time, acpi_delay_time, vbi_time_out;
1191         u32 vddc_dly, bb_dly, acpi_dly, vbi_dly, mclk_switch_limit;
1192         u32 reference_clock;
1193
1194         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mvdd_chg_time, 1);
1195
1196         voltage_response_time = (u32)rdev->pm.dpm.voltage_response_time;
1197         backbias_response_time = (u32)rdev->pm.dpm.backbias_response_time;
1198
1199         if (voltage_response_time == 0)
1200                 voltage_response_time = 1000;
1201
1202         if (backbias_response_time == 0)
1203                 backbias_response_time = 1000;
1204
1205         acpi_delay_time = 15000;
1206         vbi_time_out = 100000;
1207
1208         reference_clock = radeon_get_xclk(rdev);
1209
1210         vddc_dly = (voltage_response_time  * reference_clock) / 1600;
1211         bb_dly   = (backbias_response_time * reference_clock) / 1600;
1212         acpi_dly = (acpi_delay_time * reference_clock) / 1600;
1213         vbi_dly  = (vbi_time_out * reference_clock) / 1600;
1214
1215         mclk_switch_limit = (460 * reference_clock) / 100;
1216
1217         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_vreg,  vddc_dly);
1218         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_bbias, bb_dly);
1219         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_acpi,  acpi_dly);
1220         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mclk_chg_timeout, vbi_dly);
1221         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mc_block_delay, 0xAA);
1222         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mclk_switch_lim, mclk_switch_limit);
1223 }
1224
1225 static void ni_populate_smc_voltage_table(struct radeon_device *rdev,
1226                                           struct atom_voltage_table *voltage_table,
1227                                           NISLANDS_SMC_STATETABLE *table)
1228 {
1229         unsigned int i;
1230
1231         for (i = 0; i < voltage_table->count; i++) {
1232                 table->highSMIO[i] = 0;
1233                 table->lowSMIO[i] |= cpu_to_be32(voltage_table->entries[i].smio_low);
1234         }
1235 }
1236
1237 static void ni_populate_smc_voltage_tables(struct radeon_device *rdev,
1238                                            NISLANDS_SMC_STATETABLE *table)
1239 {
1240         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1241         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1242         unsigned char i;
1243
1244         if (eg_pi->vddc_voltage_table.count) {
1245                 ni_populate_smc_voltage_table(rdev, &eg_pi->vddc_voltage_table, table);
1246                 table->voltageMaskTable.highMask[NISLANDS_SMC_VOLTAGEMASK_VDDC] = 0;
1247                 table->voltageMaskTable.lowMask[NISLANDS_SMC_VOLTAGEMASK_VDDC] =
1248                         cpu_to_be32(eg_pi->vddc_voltage_table.mask_low);
1249
1250                 for (i = 0; i < eg_pi->vddc_voltage_table.count; i++) {
1251                         if (pi->max_vddc_in_table <= eg_pi->vddc_voltage_table.entries[i].value) {
1252                                 table->maxVDDCIndexInPPTable = i;
1253                                 break;
1254                         }
1255                 }
1256         }
1257
1258         if (eg_pi->vddci_voltage_table.count) {
1259                 ni_populate_smc_voltage_table(rdev, &eg_pi->vddci_voltage_table, table);
1260
1261                 table->voltageMaskTable.highMask[NISLANDS_SMC_VOLTAGEMASK_VDDCI] = 0;
1262                 table->voltageMaskTable.lowMask[NISLANDS_SMC_VOLTAGEMASK_VDDCI] =
1263                         cpu_to_be32(eg_pi->vddc_voltage_table.mask_low);
1264         }
1265 }
1266
1267 static int ni_populate_voltage_value(struct radeon_device *rdev,
1268                                      struct atom_voltage_table *table,
1269                                      u16 value,
1270                                      NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1271 {
1272         unsigned int i;
1273
1274         for (i = 0; i < table->count; i++) {
1275                 if (value <= table->entries[i].value) {
1276                         voltage->index = (u8)i;
1277                         voltage->value = cpu_to_be16(table->entries[i].value);
1278                         break;
1279                 }
1280         }
1281
1282         if (i >= table->count)
1283                 return -EINVAL;
1284
1285         return 0;
1286 }
1287
1288 static void ni_populate_mvdd_value(struct radeon_device *rdev,
1289                                    u32 mclk,
1290                                    NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1291 {
1292         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1293         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1294
1295         if (!pi->mvdd_control) {
1296                 voltage->index = eg_pi->mvdd_high_index;
1297                 voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1298                 return;
1299         }
1300
1301         if (mclk <= pi->mvdd_split_frequency) {
1302                 voltage->index = eg_pi->mvdd_low_index;
1303                 voltage->value = cpu_to_be16(MVDD_LOW_VALUE);
1304         } else {
1305                 voltage->index = eg_pi->mvdd_high_index;
1306                 voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1307         }
1308 }
1309
1310 static int ni_get_std_voltage_value(struct radeon_device *rdev,
1311                                     NISLANDS_SMC_VOLTAGE_VALUE *voltage,
1312                                     u16 *std_voltage)
1313 {
1314         if (rdev->pm.dpm.dyn_state.cac_leakage_table.entries &&
1315             ((u32)voltage->index < rdev->pm.dpm.dyn_state.cac_leakage_table.count))
1316                 *std_voltage = rdev->pm.dpm.dyn_state.cac_leakage_table.entries[voltage->index].vddc;
1317         else
1318                 *std_voltage = be16_to_cpu(voltage->value);
1319
1320         return 0;
1321 }
1322
1323 static void ni_populate_std_voltage_value(struct radeon_device *rdev,
1324                                           u16 value, u8 index,
1325                                           NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1326 {
1327         voltage->index = index;
1328         voltage->value = cpu_to_be16(value);
1329 }
1330
1331 static u32 ni_get_smc_power_scaling_factor(struct radeon_device *rdev)
1332 {
1333         u32 xclk_period;
1334         u32 xclk = radeon_get_xclk(rdev);
1335         u32 tmp = RREG32(CG_CAC_CTRL) & TID_CNT_MASK;
1336
1337         xclk_period = (1000000000UL / xclk);
1338         xclk_period /= 10000UL;
1339
1340         return tmp * xclk_period;
1341 }
1342
1343 static u32 ni_scale_power_for_smc(u32 power_in_watts, u32 scaling_factor)
1344 {
1345         return (power_in_watts * scaling_factor) << 2;
1346 }
1347
1348 static u32 ni_calculate_power_boost_limit(struct radeon_device *rdev,
1349                                           struct radeon_ps *radeon_state,
1350                                           u32 near_tdp_limit)
1351 {
1352         struct ni_ps *state = ni_get_ps(radeon_state);
1353         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1354         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1355         u32 power_boost_limit = 0;
1356         int ret;
1357
1358         if (ni_pi->enable_power_containment &&
1359             ni_pi->use_power_boost_limit) {
1360                 NISLANDS_SMC_VOLTAGE_VALUE vddc;
1361                 u16 std_vddc_med;
1362                 u16 std_vddc_high;
1363                 u64 tmp, n, d;
1364
1365                 if (state->performance_level_count < 3)
1366                         return 0;
1367
1368                 ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
1369                                                 state->performance_levels[state->performance_level_count - 2].vddc,
1370                                                 &vddc);
1371                 if (ret)
1372                         return 0;
1373
1374                 ret = ni_get_std_voltage_value(rdev, &vddc, &std_vddc_med);
1375                 if (ret)
1376                         return 0;
1377
1378                 ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
1379                                                 state->performance_levels[state->performance_level_count - 1].vddc,
1380                                                 &vddc);
1381                 if (ret)
1382                         return 0;
1383
1384                 ret = ni_get_std_voltage_value(rdev, &vddc, &std_vddc_high);
1385                 if (ret)
1386                         return 0;
1387
1388                 n = ((u64)near_tdp_limit * ((u64)std_vddc_med * (u64)std_vddc_med) * 90);
1389                 d = ((u64)std_vddc_high * (u64)std_vddc_high * 100);
1390                 tmp = div64_u64(n, d);
1391
1392                 if (tmp >> 32)
1393                         return 0;
1394                 power_boost_limit = (u32)tmp;
1395         }
1396
1397         return power_boost_limit;
1398 }
1399
1400 static int ni_calculate_adjusted_tdp_limits(struct radeon_device *rdev,
1401                                             bool adjust_polarity,
1402                                             u32 tdp_adjustment,
1403                                             u32 *tdp_limit,
1404                                             u32 *near_tdp_limit)
1405 {
1406         if (tdp_adjustment > (u32)rdev->pm.dpm.tdp_od_limit)
1407                 return -EINVAL;
1408
1409         if (adjust_polarity) {
1410                 *tdp_limit = ((100 + tdp_adjustment) * rdev->pm.dpm.tdp_limit) / 100;
1411                 *near_tdp_limit = rdev->pm.dpm.near_tdp_limit + (*tdp_limit - rdev->pm.dpm.tdp_limit);
1412         } else {
1413                 *tdp_limit = ((100 - tdp_adjustment) * rdev->pm.dpm.tdp_limit) / 100;
1414                 *near_tdp_limit = rdev->pm.dpm.near_tdp_limit - (rdev->pm.dpm.tdp_limit - *tdp_limit);
1415         }
1416
1417         return 0;
1418 }
1419
1420 static int ni_populate_smc_tdp_limits(struct radeon_device *rdev,
1421                                       struct radeon_ps *radeon_state)
1422 {
1423         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1424         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1425
1426         if (ni_pi->enable_power_containment) {
1427                 NISLANDS_SMC_STATETABLE *smc_table = &ni_pi->smc_statetable;
1428                 u32 scaling_factor = ni_get_smc_power_scaling_factor(rdev);
1429                 u32 tdp_limit;
1430                 u32 near_tdp_limit;
1431                 u32 power_boost_limit;
1432                 int ret;
1433
1434                 if (scaling_factor == 0)
1435                         return -EINVAL;
1436
1437                 memset(smc_table, 0, sizeof(NISLANDS_SMC_STATETABLE));
1438
1439                 ret = ni_calculate_adjusted_tdp_limits(rdev,
1440                                                        false, /* ??? */
1441                                                        rdev->pm.dpm.tdp_adjustment,
1442                                                        &tdp_limit,
1443                                                        &near_tdp_limit);
1444                 if (ret)
1445                         return ret;
1446
1447                 power_boost_limit = ni_calculate_power_boost_limit(rdev, radeon_state,
1448                                                                    near_tdp_limit);
1449
1450                 smc_table->dpm2Params.TDPLimit =
1451                         cpu_to_be32(ni_scale_power_for_smc(tdp_limit, scaling_factor));
1452                 smc_table->dpm2Params.NearTDPLimit =
1453                         cpu_to_be32(ni_scale_power_for_smc(near_tdp_limit, scaling_factor));
1454                 smc_table->dpm2Params.SafePowerLimit =
1455                         cpu_to_be32(ni_scale_power_for_smc((near_tdp_limit * NISLANDS_DPM2_TDP_SAFE_LIMIT_PERCENT) / 100,
1456                                                            scaling_factor));
1457                 smc_table->dpm2Params.PowerBoostLimit =
1458                         cpu_to_be32(ni_scale_power_for_smc(power_boost_limit, scaling_factor));
1459
1460                 ret = rv770_copy_bytes_to_smc(rdev,
1461                                               (u16)(pi->state_table_start + offsetof(NISLANDS_SMC_STATETABLE, dpm2Params) +
1462                                                     offsetof(PP_NIslands_DPM2Parameters, TDPLimit)),
1463                                               (u8 *)(&smc_table->dpm2Params.TDPLimit),
1464                                               sizeof(u32) * 4, pi->sram_end);
1465                 if (ret)
1466                         return ret;
1467         }
1468
1469         return 0;
1470 }
1471
1472 static int ni_copy_and_switch_arb_sets(struct radeon_device *rdev,
1473                                        u32 arb_freq_src, u32 arb_freq_dest)
1474 {
1475         u32 mc_arb_dram_timing;
1476         u32 mc_arb_dram_timing2;
1477         u32 burst_time;
1478         u32 mc_cg_config;
1479
1480         switch (arb_freq_src) {
1481         case MC_CG_ARB_FREQ_F0:
1482                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING);
1483                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2);
1484                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE0_MASK) >> STATE0_SHIFT;
1485                 break;
1486         case MC_CG_ARB_FREQ_F1:
1487                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_1);
1488                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_1);
1489                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE1_MASK) >> STATE1_SHIFT;
1490                 break;
1491         case MC_CG_ARB_FREQ_F2:
1492                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_2);
1493                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_2);
1494                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE2_MASK) >> STATE2_SHIFT;
1495                 break;
1496         case MC_CG_ARB_FREQ_F3:
1497                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_3);
1498                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_3);
1499                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE3_MASK) >> STATE3_SHIFT;
1500                 break;
1501         default:
1502                 return -EINVAL;
1503         }
1504
1505         switch (arb_freq_dest) {
1506         case MC_CG_ARB_FREQ_F0:
1507                 WREG32(MC_ARB_DRAM_TIMING, mc_arb_dram_timing);
1508                 WREG32(MC_ARB_DRAM_TIMING2, mc_arb_dram_timing2);
1509                 WREG32_P(MC_ARB_BURST_TIME, STATE0(burst_time), ~STATE0_MASK);
1510                 break;
1511         case MC_CG_ARB_FREQ_F1:
1512                 WREG32(MC_ARB_DRAM_TIMING_1, mc_arb_dram_timing);
1513                 WREG32(MC_ARB_DRAM_TIMING2_1, mc_arb_dram_timing2);
1514                 WREG32_P(MC_ARB_BURST_TIME, STATE1(burst_time), ~STATE1_MASK);
1515                 break;
1516         case MC_CG_ARB_FREQ_F2:
1517                 WREG32(MC_ARB_DRAM_TIMING_2, mc_arb_dram_timing);
1518                 WREG32(MC_ARB_DRAM_TIMING2_2, mc_arb_dram_timing2);
1519                 WREG32_P(MC_ARB_BURST_TIME, STATE2(burst_time), ~STATE2_MASK);
1520                 break;
1521         case MC_CG_ARB_FREQ_F3:
1522                 WREG32(MC_ARB_DRAM_TIMING_3, mc_arb_dram_timing);
1523                 WREG32(MC_ARB_DRAM_TIMING2_3, mc_arb_dram_timing2);
1524                 WREG32_P(MC_ARB_BURST_TIME, STATE3(burst_time), ~STATE3_MASK);
1525                 break;
1526         default:
1527                 return -EINVAL;
1528         }
1529
1530         mc_cg_config = RREG32(MC_CG_CONFIG) | 0x0000000F;
1531         WREG32(MC_CG_CONFIG, mc_cg_config);
1532         WREG32_P(MC_ARB_CG, CG_ARB_REQ(arb_freq_dest), ~CG_ARB_REQ_MASK);
1533
1534         return 0;
1535 }
1536
1537 static int ni_init_arb_table_index(struct radeon_device *rdev)
1538 {
1539         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1540         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1541         u32 tmp;
1542         int ret;
1543
1544         ret = rv770_read_smc_sram_dword(rdev, ni_pi->arb_table_start,
1545                                         &tmp, pi->sram_end);
1546         if (ret)
1547                 return ret;
1548
1549         tmp &= 0x00FFFFFF;
1550         tmp |= ((u32)MC_CG_ARB_FREQ_F1) << 24;
1551
1552         return rv770_write_smc_sram_dword(rdev, ni_pi->arb_table_start,
1553                                           tmp, pi->sram_end);
1554 }
1555
1556 static int ni_initial_switch_from_arb_f0_to_f1(struct radeon_device *rdev)
1557 {
1558         return ni_copy_and_switch_arb_sets(rdev, MC_CG_ARB_FREQ_F0, MC_CG_ARB_FREQ_F1);
1559 }
1560
1561 static int ni_force_switch_to_arb_f0(struct radeon_device *rdev)
1562 {
1563         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1564         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1565         u32 tmp;
1566         int ret;
1567
1568         ret = rv770_read_smc_sram_dword(rdev, ni_pi->arb_table_start,
1569                                         &tmp, pi->sram_end);
1570         if (ret)
1571                 return ret;
1572
1573         tmp = (tmp >> 24) & 0xff;
1574
1575         if (tmp == MC_CG_ARB_FREQ_F0)
1576                 return 0;
1577
1578         return ni_copy_and_switch_arb_sets(rdev, tmp, MC_CG_ARB_FREQ_F0);
1579 }
1580
1581 static int ni_populate_memory_timing_parameters(struct radeon_device *rdev,
1582                                                 struct rv7xx_pl *pl,
1583                                                 SMC_NIslands_MCArbDramTimingRegisterSet *arb_regs)
1584 {
1585         u32 dram_timing;
1586         u32 dram_timing2;
1587
1588         arb_regs->mc_arb_rfsh_rate =
1589                 (u8)rv770_calculate_memory_refresh_rate(rdev, pl->sclk);
1590
1591
1592         radeon_atom_set_engine_dram_timings(rdev,
1593                                             pl->sclk,
1594                                             pl->mclk);
1595
1596         dram_timing = RREG32(MC_ARB_DRAM_TIMING);
1597         dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2);
1598
1599         arb_regs->mc_arb_dram_timing  = cpu_to_be32(dram_timing);
1600         arb_regs->mc_arb_dram_timing2 = cpu_to_be32(dram_timing2);
1601
1602         return 0;
1603 }
1604
1605 static int ni_do_program_memory_timing_parameters(struct radeon_device *rdev,
1606                                                   struct radeon_ps *radeon_state,
1607                                                   unsigned int first_arb_set)
1608 {
1609         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1610         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1611         struct ni_ps *state = ni_get_ps(radeon_state);
1612         SMC_NIslands_MCArbDramTimingRegisterSet arb_regs = { 0 };
1613         int i, ret = 0;
1614
1615         for (i = 0; i < state->performance_level_count; i++) {
1616                 ret = ni_populate_memory_timing_parameters(rdev, &state->performance_levels[i], &arb_regs);
1617                 if (ret)
1618                         break;
1619
1620                 ret = rv770_copy_bytes_to_smc(rdev,
1621                                               (u16)(ni_pi->arb_table_start +
1622                                                     offsetof(SMC_NIslands_MCArbDramTimingRegisters, data) +
1623                                                     sizeof(SMC_NIslands_MCArbDramTimingRegisterSet) * (first_arb_set + i)),
1624                                               (u8 *)&arb_regs,
1625                                               (u16)sizeof(SMC_NIslands_MCArbDramTimingRegisterSet),
1626                                               pi->sram_end);
1627                 if (ret)
1628                         break;
1629         }
1630         return ret;
1631 }
1632
1633 static int ni_program_memory_timing_parameters(struct radeon_device *rdev,
1634                                                struct radeon_ps *radeon_new_state)
1635 {
1636         return ni_do_program_memory_timing_parameters(rdev, radeon_new_state,
1637                                                       NISLANDS_DRIVER_STATE_ARB_INDEX);
1638 }
1639
1640 static void ni_populate_initial_mvdd_value(struct radeon_device *rdev,
1641                                            struct NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1642 {
1643         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1644
1645         voltage->index = eg_pi->mvdd_high_index;
1646         voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1647 }
1648
1649 static int ni_populate_smc_initial_state(struct radeon_device *rdev,
1650                                          struct radeon_ps *radeon_initial_state,
1651                                          NISLANDS_SMC_STATETABLE *table)
1652 {
1653         struct ni_ps *initial_state = ni_get_ps(radeon_initial_state);
1654         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1655         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1656         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1657         u32 reg;
1658         int ret;
1659
1660         table->initialState.levels[0].mclk.vMPLL_AD_FUNC_CNTL =
1661                 cpu_to_be32(ni_pi->clock_registers.mpll_ad_func_cntl);
1662         table->initialState.levels[0].mclk.vMPLL_AD_FUNC_CNTL_2 =
1663                 cpu_to_be32(ni_pi->clock_registers.mpll_ad_func_cntl_2);
1664         table->initialState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL =
1665                 cpu_to_be32(ni_pi->clock_registers.mpll_dq_func_cntl);
1666         table->initialState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL_2 =
1667                 cpu_to_be32(ni_pi->clock_registers.mpll_dq_func_cntl_2);
1668         table->initialState.levels[0].mclk.vMCLK_PWRMGT_CNTL =
1669                 cpu_to_be32(ni_pi->clock_registers.mclk_pwrmgt_cntl);
1670         table->initialState.levels[0].mclk.vDLL_CNTL =
1671                 cpu_to_be32(ni_pi->clock_registers.dll_cntl);
1672         table->initialState.levels[0].mclk.vMPLL_SS =
1673                 cpu_to_be32(ni_pi->clock_registers.mpll_ss1);
1674         table->initialState.levels[0].mclk.vMPLL_SS2 =
1675                 cpu_to_be32(ni_pi->clock_registers.mpll_ss2);
1676         table->initialState.levels[0].mclk.mclk_value =
1677                 cpu_to_be32(initial_state->performance_levels[0].mclk);
1678
1679         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL =
1680                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl);
1681         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 =
1682                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_2);
1683         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 =
1684                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_3);
1685         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_4 =
1686                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_4);
1687         table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM =
1688                 cpu_to_be32(ni_pi->clock_registers.cg_spll_spread_spectrum);
1689         table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM_2 =
1690                 cpu_to_be32(ni_pi->clock_registers.cg_spll_spread_spectrum_2);
1691         table->initialState.levels[0].sclk.sclk_value =
1692                 cpu_to_be32(initial_state->performance_levels[0].sclk);
1693         table->initialState.levels[0].arbRefreshState =
1694                 NISLANDS_INITIAL_STATE_ARB_INDEX;
1695
1696         table->initialState.levels[0].ACIndex = 0;
1697
1698         ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
1699                                         initial_state->performance_levels[0].vddc,
1700                                         &table->initialState.levels[0].vddc);
1701         if (!ret) {
1702                 u16 std_vddc;
1703
1704                 ret = ni_get_std_voltage_value(rdev,
1705                                                &table->initialState.levels[0].vddc,
1706                                                &std_vddc);
1707                 if (!ret)
1708                         ni_populate_std_voltage_value(rdev, std_vddc,
1709                                                       table->initialState.levels[0].vddc.index,
1710                                                       &table->initialState.levels[0].std_vddc);
1711         }
1712
1713         if (eg_pi->vddci_control)
1714                 ni_populate_voltage_value(rdev,
1715                                           &eg_pi->vddci_voltage_table,
1716                                           initial_state->performance_levels[0].vddci,
1717                                           &table->initialState.levels[0].vddci);
1718
1719         ni_populate_initial_mvdd_value(rdev, &table->initialState.levels[0].mvdd);
1720
1721         reg = CG_R(0xffff) | CG_L(0);
1722         table->initialState.levels[0].aT = cpu_to_be32(reg);
1723
1724         table->initialState.levels[0].bSP = cpu_to_be32(pi->dsp);
1725
1726         if (pi->boot_in_gen2)
1727                 table->initialState.levels[0].gen2PCIE = 1;
1728         else
1729                 table->initialState.levels[0].gen2PCIE = 0;
1730
1731         if (pi->mem_gddr5) {
1732                 table->initialState.levels[0].strobeMode =
1733                         cypress_get_strobe_mode_settings(rdev,
1734                                                          initial_state->performance_levels[0].mclk);
1735
1736                 if (initial_state->performance_levels[0].mclk > pi->mclk_edc_enable_threshold)
1737                         table->initialState.levels[0].mcFlags = NISLANDS_SMC_MC_EDC_RD_FLAG | NISLANDS_SMC_MC_EDC_WR_FLAG;
1738                 else
1739                         table->initialState.levels[0].mcFlags =  0;
1740         }
1741
1742         table->initialState.levelCount = 1;
1743
1744         table->initialState.flags |= PPSMC_SWSTATE_FLAG_DC;
1745
1746         table->initialState.levels[0].dpm2.MaxPS = 0;
1747         table->initialState.levels[0].dpm2.NearTDPDec = 0;
1748         table->initialState.levels[0].dpm2.AboveSafeInc = 0;
1749         table->initialState.levels[0].dpm2.BelowSafeInc = 0;
1750
1751         reg = MIN_POWER_MASK | MAX_POWER_MASK;
1752         table->initialState.levels[0].SQPowerThrottle = cpu_to_be32(reg);
1753
1754         reg = MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
1755         table->initialState.levels[0].SQPowerThrottle_2 = cpu_to_be32(reg);
1756
1757         return 0;
1758 }
1759
1760 static int ni_populate_smc_acpi_state(struct radeon_device *rdev,
1761                                       NISLANDS_SMC_STATETABLE *table)
1762 {
1763         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1764         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1765         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1766         u32 mpll_ad_func_cntl   = ni_pi->clock_registers.mpll_ad_func_cntl;
1767         u32 mpll_ad_func_cntl_2 = ni_pi->clock_registers.mpll_ad_func_cntl_2;
1768         u32 mpll_dq_func_cntl   = ni_pi->clock_registers.mpll_dq_func_cntl;
1769         u32 mpll_dq_func_cntl_2 = ni_pi->clock_registers.mpll_dq_func_cntl_2;
1770         u32 spll_func_cntl      = ni_pi->clock_registers.cg_spll_func_cntl;
1771         u32 spll_func_cntl_2    = ni_pi->clock_registers.cg_spll_func_cntl_2;
1772         u32 spll_func_cntl_3    = ni_pi->clock_registers.cg_spll_func_cntl_3;
1773         u32 spll_func_cntl_4    = ni_pi->clock_registers.cg_spll_func_cntl_4;
1774         u32 mclk_pwrmgt_cntl    = ni_pi->clock_registers.mclk_pwrmgt_cntl;
1775         u32 dll_cntl            = ni_pi->clock_registers.dll_cntl;
1776         u32 reg;
1777         int ret;
1778
1779         table->ACPIState = table->initialState;
1780
1781         table->ACPIState.flags &= ~PPSMC_SWSTATE_FLAG_DC;
1782
1783         if (pi->acpi_vddc) {
1784                 ret = ni_populate_voltage_value(rdev,
1785                                                 &eg_pi->vddc_voltage_table,
1786                                                 pi->acpi_vddc, &table->ACPIState.levels[0].vddc);
1787                 if (!ret) {
1788                         u16 std_vddc;
1789
1790                         ret = ni_get_std_voltage_value(rdev,
1791                                                        &table->ACPIState.levels[0].vddc, &std_vddc);
1792                         if (!ret)
1793                                 ni_populate_std_voltage_value(rdev, std_vddc,
1794                                                               table->ACPIState.levels[0].vddc.index,
1795                                                               &table->ACPIState.levels[0].std_vddc);
1796                 }
1797
1798                 if (pi->pcie_gen2) {
1799                         if (pi->acpi_pcie_gen2)
1800                                 table->ACPIState.levels[0].gen2PCIE = 1;
1801                         else
1802                                 table->ACPIState.levels[0].gen2PCIE = 0;
1803                 } else {
1804                         table->ACPIState.levels[0].gen2PCIE = 0;
1805                 }
1806         } else {
1807                 ret = ni_populate_voltage_value(rdev,
1808                                                 &eg_pi->vddc_voltage_table,
1809                                                 pi->min_vddc_in_table,
1810                                                 &table->ACPIState.levels[0].vddc);
1811                 if (!ret) {
1812                         u16 std_vddc;
1813
1814                         ret = ni_get_std_voltage_value(rdev,
1815                                                        &table->ACPIState.levels[0].vddc,
1816                                                        &std_vddc);
1817                         if (!ret)
1818                                 ni_populate_std_voltage_value(rdev, std_vddc,
1819                                                               table->ACPIState.levels[0].vddc.index,
1820                                                               &table->ACPIState.levels[0].std_vddc);
1821                 }
1822                 table->ACPIState.levels[0].gen2PCIE = 0;
1823         }
1824
1825         if (eg_pi->acpi_vddci) {
1826                 if (eg_pi->vddci_control)
1827                         ni_populate_voltage_value(rdev,
1828                                                   &eg_pi->vddci_voltage_table,
1829                                                   eg_pi->acpi_vddci,
1830                                                   &table->ACPIState.levels[0].vddci);
1831         }
1832
1833
1834         mpll_ad_func_cntl &= ~PDNB;
1835
1836         mpll_ad_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN;
1837
1838         if (pi->mem_gddr5)
1839                 mpll_dq_func_cntl &= ~PDNB;
1840         mpll_dq_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN | BYPASS;
1841
1842
1843         mclk_pwrmgt_cntl |= (MRDCKA0_RESET |
1844                              MRDCKA1_RESET |
1845                              MRDCKB0_RESET |
1846                              MRDCKB1_RESET |
1847                              MRDCKC0_RESET |
1848                              MRDCKC1_RESET |
1849                              MRDCKD0_RESET |
1850                              MRDCKD1_RESET);
1851
1852         mclk_pwrmgt_cntl &= ~(MRDCKA0_PDNB |
1853                               MRDCKA1_PDNB |
1854                               MRDCKB0_PDNB |
1855                               MRDCKB1_PDNB |
1856                               MRDCKC0_PDNB |
1857                               MRDCKC1_PDNB |
1858                               MRDCKD0_PDNB |
1859                               MRDCKD1_PDNB);
1860
1861         dll_cntl |= (MRDCKA0_BYPASS |
1862                      MRDCKA1_BYPASS |
1863                      MRDCKB0_BYPASS |
1864                      MRDCKB1_BYPASS |
1865                      MRDCKC0_BYPASS |
1866                      MRDCKC1_BYPASS |
1867                      MRDCKD0_BYPASS |
1868                      MRDCKD1_BYPASS);
1869
1870         spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
1871         spll_func_cntl_2 |= SCLK_MUX_SEL(4);
1872
1873         table->ACPIState.levels[0].mclk.vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
1874         table->ACPIState.levels[0].mclk.vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
1875         table->ACPIState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
1876         table->ACPIState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
1877         table->ACPIState.levels[0].mclk.vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
1878         table->ACPIState.levels[0].mclk.vDLL_CNTL = cpu_to_be32(dll_cntl);
1879
1880         table->ACPIState.levels[0].mclk.mclk_value = 0;
1881
1882         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL = cpu_to_be32(spll_func_cntl);
1883         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(spll_func_cntl_2);
1884         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(spll_func_cntl_3);
1885         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_4 = cpu_to_be32(spll_func_cntl_4);
1886
1887         table->ACPIState.levels[0].sclk.sclk_value = 0;
1888
1889         ni_populate_mvdd_value(rdev, 0, &table->ACPIState.levels[0].mvdd);
1890
1891         if (eg_pi->dynamic_ac_timing)
1892                 table->ACPIState.levels[0].ACIndex = 1;
1893
1894         table->ACPIState.levels[0].dpm2.MaxPS = 0;
1895         table->ACPIState.levels[0].dpm2.NearTDPDec = 0;
1896         table->ACPIState.levels[0].dpm2.AboveSafeInc = 0;
1897         table->ACPIState.levels[0].dpm2.BelowSafeInc = 0;
1898
1899         reg = MIN_POWER_MASK | MAX_POWER_MASK;
1900         table->ACPIState.levels[0].SQPowerThrottle = cpu_to_be32(reg);
1901
1902         reg = MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
1903         table->ACPIState.levels[0].SQPowerThrottle_2 = cpu_to_be32(reg);
1904
1905         return 0;
1906 }
1907
1908 static int ni_init_smc_table(struct radeon_device *rdev)
1909 {
1910         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1911         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1912         int ret;
1913         struct radeon_ps *radeon_boot_state = rdev->pm.dpm.boot_ps;
1914         NISLANDS_SMC_STATETABLE *table = &ni_pi->smc_statetable;
1915
1916         memset(table, 0, sizeof(NISLANDS_SMC_STATETABLE));
1917
1918         ni_populate_smc_voltage_tables(rdev, table);
1919
1920         switch (rdev->pm.int_thermal_type) {
1921         case THERMAL_TYPE_NI:
1922         case THERMAL_TYPE_EMC2103_WITH_INTERNAL:
1923                 table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_INTERNAL;
1924                 break;
1925         case THERMAL_TYPE_NONE:
1926                 table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_NONE;
1927                 break;
1928         default:
1929                 table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_EXTERNAL;
1930                 break;
1931         }
1932
1933         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC)
1934                 table->systemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC;
1935
1936         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_REGULATOR_HOT)
1937                 table->systemFlags |= PPSMC_SYSTEMFLAG_REGULATOR_HOT;
1938
1939         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC)
1940                 table->systemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC;
1941
1942         if (pi->mem_gddr5)
1943                 table->systemFlags |= PPSMC_SYSTEMFLAG_GDDR5;
1944
1945         ret = ni_populate_smc_initial_state(rdev, radeon_boot_state, table);
1946         if (ret)
1947                 return ret;
1948
1949         ret = ni_populate_smc_acpi_state(rdev, table);
1950         if (ret)
1951                 return ret;
1952
1953         table->driverState = table->initialState;
1954
1955         table->ULVState = table->initialState;
1956
1957         ret = ni_do_program_memory_timing_parameters(rdev, radeon_boot_state,
1958                                                      NISLANDS_INITIAL_STATE_ARB_INDEX);
1959         if (ret)
1960                 return ret;
1961
1962         return rv770_copy_bytes_to_smc(rdev, pi->state_table_start, (u8 *)table,
1963                                        sizeof(NISLANDS_SMC_STATETABLE), pi->sram_end);
1964 }
1965
1966 static int ni_calculate_sclk_params(struct radeon_device *rdev,
1967                                     u32 engine_clock,
1968                                     NISLANDS_SMC_SCLK_VALUE *sclk)
1969 {
1970         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1971         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1972         struct atom_clock_dividers dividers;
1973         u32 spll_func_cntl = ni_pi->clock_registers.cg_spll_func_cntl;
1974         u32 spll_func_cntl_2 = ni_pi->clock_registers.cg_spll_func_cntl_2;
1975         u32 spll_func_cntl_3 = ni_pi->clock_registers.cg_spll_func_cntl_3;
1976         u32 spll_func_cntl_4 = ni_pi->clock_registers.cg_spll_func_cntl_4;
1977         u32 cg_spll_spread_spectrum = ni_pi->clock_registers.cg_spll_spread_spectrum;
1978         u32 cg_spll_spread_spectrum_2 = ni_pi->clock_registers.cg_spll_spread_spectrum_2;
1979         u64 tmp;
1980         u32 reference_clock = rdev->clock.spll.reference_freq;
1981         u32 reference_divider;
1982         u32 fbdiv;
1983         int ret;
1984
1985         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1986                                              engine_clock, false, &dividers);
1987         if (ret)
1988                 return ret;
1989
1990         reference_divider = 1 + dividers.ref_div;
1991
1992
1993         tmp = (u64) engine_clock * reference_divider * dividers.post_div * 16834;
1994         do_div(tmp, reference_clock);
1995         fbdiv = (u32) tmp;
1996
1997         spll_func_cntl &= ~(SPLL_PDIV_A_MASK | SPLL_REF_DIV_MASK);
1998         spll_func_cntl |= SPLL_REF_DIV(dividers.ref_div);
1999         spll_func_cntl |= SPLL_PDIV_A(dividers.post_div);
2000
2001         spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
2002         spll_func_cntl_2 |= SCLK_MUX_SEL(2);
2003
2004         spll_func_cntl_3 &= ~SPLL_FB_DIV_MASK;
2005         spll_func_cntl_3 |= SPLL_FB_DIV(fbdiv);
2006         spll_func_cntl_3 |= SPLL_DITHEN;
2007
2008         if (pi->sclk_ss) {
2009                 struct radeon_atom_ss ss;
2010                 u32 vco_freq = engine_clock * dividers.post_div;
2011
2012                 if (radeon_atombios_get_asic_ss_info(rdev, &ss,
2013                                                      ASIC_INTERNAL_ENGINE_SS, vco_freq)) {
2014                         u32 clk_s = reference_clock * 5 / (reference_divider * ss.rate);
2015                         u32 clk_v = 4 * ss.percentage * fbdiv / (clk_s * 10000);
2016
2017                         cg_spll_spread_spectrum &= ~CLK_S_MASK;
2018                         cg_spll_spread_spectrum |= CLK_S(clk_s);
2019                         cg_spll_spread_spectrum |= SSEN;
2020
2021                         cg_spll_spread_spectrum_2 &= ~CLK_V_MASK;
2022                         cg_spll_spread_spectrum_2 |= CLK_V(clk_v);
2023                 }
2024         }
2025
2026         sclk->sclk_value = engine_clock;
2027         sclk->vCG_SPLL_FUNC_CNTL = spll_func_cntl;
2028         sclk->vCG_SPLL_FUNC_CNTL_2 = spll_func_cntl_2;
2029         sclk->vCG_SPLL_FUNC_CNTL_3 = spll_func_cntl_3;
2030         sclk->vCG_SPLL_FUNC_CNTL_4 = spll_func_cntl_4;
2031         sclk->vCG_SPLL_SPREAD_SPECTRUM = cg_spll_spread_spectrum;
2032         sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cg_spll_spread_spectrum_2;
2033
2034         return 0;
2035 }
2036
2037 static int ni_populate_sclk_value(struct radeon_device *rdev,
2038                                   u32 engine_clock,
2039                                   NISLANDS_SMC_SCLK_VALUE *sclk)
2040 {
2041         NISLANDS_SMC_SCLK_VALUE sclk_tmp;
2042         int ret;
2043
2044         ret = ni_calculate_sclk_params(rdev, engine_clock, &sclk_tmp);
2045         if (!ret) {
2046                 sclk->sclk_value = cpu_to_be32(sclk_tmp.sclk_value);
2047                 sclk->vCG_SPLL_FUNC_CNTL = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL);
2048                 sclk->vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_2);
2049                 sclk->vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_3);
2050                 sclk->vCG_SPLL_FUNC_CNTL_4 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_4);
2051                 sclk->vCG_SPLL_SPREAD_SPECTRUM = cpu_to_be32(sclk_tmp.vCG_SPLL_SPREAD_SPECTRUM);
2052                 sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cpu_to_be32(sclk_tmp.vCG_SPLL_SPREAD_SPECTRUM_2);
2053         }
2054
2055         return ret;
2056 }
2057
2058 static int ni_init_smc_spll_table(struct radeon_device *rdev)
2059 {
2060         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2061         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2062         SMC_NISLANDS_SPLL_DIV_TABLE *spll_table;
2063         NISLANDS_SMC_SCLK_VALUE sclk_params;
2064         u32 fb_div;
2065         u32 p_div;
2066         u32 clk_s;
2067         u32 clk_v;
2068         u32 sclk = 0;
2069         int i, ret;
2070         u32 tmp;
2071
2072         if (ni_pi->spll_table_start == 0)
2073                 return -EINVAL;
2074
2075         spll_table = kzalloc(sizeof(SMC_NISLANDS_SPLL_DIV_TABLE), GFP_KERNEL);
2076         if (spll_table == NULL)
2077                 return -ENOMEM;
2078
2079         for (i = 0; i < 256; i++) {
2080                 ret = ni_calculate_sclk_params(rdev, sclk, &sclk_params);
2081                 if (ret)
2082                         break;
2083
2084                 p_div = (sclk_params.vCG_SPLL_FUNC_CNTL & SPLL_PDIV_A_MASK) >> SPLL_PDIV_A_SHIFT;
2085                 fb_div = (sclk_params.vCG_SPLL_FUNC_CNTL_3 & SPLL_FB_DIV_MASK) >> SPLL_FB_DIV_SHIFT;
2086                 clk_s = (sclk_params.vCG_SPLL_SPREAD_SPECTRUM & CLK_S_MASK) >> CLK_S_SHIFT;
2087                 clk_v = (sclk_params.vCG_SPLL_SPREAD_SPECTRUM_2 & CLK_V_MASK) >> CLK_V_SHIFT;
2088
2089                 fb_div &= ~0x00001FFF;
2090                 fb_div >>= 1;
2091                 clk_v >>= 6;
2092
2093                 if (p_div & ~(SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT))
2094                         ret = -EINVAL;
2095
2096                 if (clk_s & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT))
2097                         ret = -EINVAL;
2098
2099                 if (clk_s & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT))
2100                         ret = -EINVAL;
2101
2102                 if (clk_v & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT))
2103                         ret = -EINVAL;
2104
2105                 if (ret)
2106                         break;
2107
2108                 tmp = ((fb_div << SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_MASK) |
2109                         ((p_div << SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_MASK);
2110                 spll_table->freq[i] = cpu_to_be32(tmp);
2111
2112                 tmp = ((clk_v << SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_MASK) |
2113                         ((clk_s << SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK);
2114                 spll_table->ss[i] = cpu_to_be32(tmp);
2115
2116                 sclk += 512;
2117         }
2118
2119         if (!ret)
2120                 ret = rv770_copy_bytes_to_smc(rdev, ni_pi->spll_table_start, (u8 *)spll_table,
2121                                               sizeof(SMC_NISLANDS_SPLL_DIV_TABLE), pi->sram_end);
2122
2123         kfree(spll_table);
2124
2125         return ret;
2126 }
2127
2128 static int ni_populate_mclk_value(struct radeon_device *rdev,
2129                                   u32 engine_clock,
2130                                   u32 memory_clock,
2131                                   NISLANDS_SMC_MCLK_VALUE *mclk,
2132                                   bool strobe_mode,
2133                                   bool dll_state_on)
2134 {
2135         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2136         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2137         u32 mpll_ad_func_cntl = ni_pi->clock_registers.mpll_ad_func_cntl;
2138         u32 mpll_ad_func_cntl_2 = ni_pi->clock_registers.mpll_ad_func_cntl_2;
2139         u32 mpll_dq_func_cntl = ni_pi->clock_registers.mpll_dq_func_cntl;
2140         u32 mpll_dq_func_cntl_2 = ni_pi->clock_registers.mpll_dq_func_cntl_2;
2141         u32 mclk_pwrmgt_cntl = ni_pi->clock_registers.mclk_pwrmgt_cntl;
2142         u32 dll_cntl = ni_pi->clock_registers.dll_cntl;
2143         u32 mpll_ss1 = ni_pi->clock_registers.mpll_ss1;
2144         u32 mpll_ss2 = ni_pi->clock_registers.mpll_ss2;
2145         struct atom_clock_dividers dividers;
2146         u32 ibias;
2147         u32 dll_speed;
2148         int ret;
2149         u32 mc_seq_misc7;
2150
2151         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_MEMORY_PLL_PARAM,
2152                                              memory_clock, strobe_mode, &dividers);
2153         if (ret)
2154                 return ret;
2155
2156         if (!strobe_mode) {
2157                 mc_seq_misc7 = RREG32(MC_SEQ_MISC7);
2158
2159                 if (mc_seq_misc7 & 0x8000000)
2160                         dividers.post_div = 1;
2161         }
2162
2163         ibias = cypress_map_clkf_to_ibias(rdev, dividers.whole_fb_div);
2164
2165         mpll_ad_func_cntl &= ~(CLKR_MASK |
2166                                YCLK_POST_DIV_MASK |
2167                                CLKF_MASK |
2168                                CLKFRAC_MASK |
2169                                IBIAS_MASK);
2170         mpll_ad_func_cntl |= CLKR(dividers.ref_div);
2171         mpll_ad_func_cntl |= YCLK_POST_DIV(dividers.post_div);
2172         mpll_ad_func_cntl |= CLKF(dividers.whole_fb_div);
2173         mpll_ad_func_cntl |= CLKFRAC(dividers.frac_fb_div);
2174         mpll_ad_func_cntl |= IBIAS(ibias);
2175
2176         if (dividers.vco_mode)
2177                 mpll_ad_func_cntl_2 |= VCO_MODE;
2178         else
2179                 mpll_ad_func_cntl_2 &= ~VCO_MODE;
2180
2181         if (pi->mem_gddr5) {
2182                 mpll_dq_func_cntl &= ~(CLKR_MASK |
2183                                        YCLK_POST_DIV_MASK |
2184                                        CLKF_MASK |
2185                                        CLKFRAC_MASK |
2186                                        IBIAS_MASK);
2187                 mpll_dq_func_cntl |= CLKR(dividers.ref_div);
2188                 mpll_dq_func_cntl |= YCLK_POST_DIV(dividers.post_div);
2189                 mpll_dq_func_cntl |= CLKF(dividers.whole_fb_div);
2190                 mpll_dq_func_cntl |= CLKFRAC(dividers.frac_fb_div);
2191                 mpll_dq_func_cntl |= IBIAS(ibias);
2192
2193                 if (strobe_mode)
2194                         mpll_dq_func_cntl &= ~PDNB;
2195                 else
2196                         mpll_dq_func_cntl |= PDNB;
2197
2198                 if (dividers.vco_mode)
2199                         mpll_dq_func_cntl_2 |= VCO_MODE;
2200                 else
2201                         mpll_dq_func_cntl_2 &= ~VCO_MODE;
2202         }
2203
2204         if (pi->mclk_ss) {
2205                 struct radeon_atom_ss ss;
2206                 u32 vco_freq = memory_clock * dividers.post_div;
2207
2208                 if (radeon_atombios_get_asic_ss_info(rdev, &ss,
2209                                                      ASIC_INTERNAL_MEMORY_SS, vco_freq)) {
2210                         u32 reference_clock = rdev->clock.mpll.reference_freq;
2211                         u32 decoded_ref = rv740_get_decoded_reference_divider(dividers.ref_div);
2212                         u32 clk_s = reference_clock * 5 / (decoded_ref * ss.rate);
2213                         u32 clk_v = ss.percentage *
2214                                 (0x4000 * dividers.whole_fb_div + 0x800 * dividers.frac_fb_div) / (clk_s * 625);
2215
2216                         mpll_ss1 &= ~CLKV_MASK;
2217                         mpll_ss1 |= CLKV(clk_v);
2218
2219                         mpll_ss2 &= ~CLKS_MASK;
2220                         mpll_ss2 |= CLKS(clk_s);
2221                 }
2222         }
2223
2224         dll_speed = rv740_get_dll_speed(pi->mem_gddr5,
2225                                         memory_clock);
2226
2227         mclk_pwrmgt_cntl &= ~DLL_SPEED_MASK;
2228         mclk_pwrmgt_cntl |= DLL_SPEED(dll_speed);
2229         if (dll_state_on)
2230                 mclk_pwrmgt_cntl |= (MRDCKA0_PDNB |
2231                                      MRDCKA1_PDNB |
2232                                      MRDCKB0_PDNB |
2233                                      MRDCKB1_PDNB |
2234                                      MRDCKC0_PDNB |
2235                                      MRDCKC1_PDNB |
2236                                      MRDCKD0_PDNB |
2237                                      MRDCKD1_PDNB);
2238         else
2239                 mclk_pwrmgt_cntl &= ~(MRDCKA0_PDNB |
2240                                       MRDCKA1_PDNB |
2241                                       MRDCKB0_PDNB |
2242                                       MRDCKB1_PDNB |
2243                                       MRDCKC0_PDNB |
2244                                       MRDCKC1_PDNB |
2245                                       MRDCKD0_PDNB |
2246                                       MRDCKD1_PDNB);
2247
2248
2249         mclk->mclk_value = cpu_to_be32(memory_clock);
2250         mclk->vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
2251         mclk->vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
2252         mclk->vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
2253         mclk->vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
2254         mclk->vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
2255         mclk->vDLL_CNTL = cpu_to_be32(dll_cntl);
2256         mclk->vMPLL_SS = cpu_to_be32(mpll_ss1);
2257         mclk->vMPLL_SS2 = cpu_to_be32(mpll_ss2);
2258
2259         return 0;
2260 }
2261
2262 static void ni_populate_smc_sp(struct radeon_device *rdev,
2263                                struct radeon_ps *radeon_state,
2264                                NISLANDS_SMC_SWSTATE *smc_state)
2265 {
2266         struct ni_ps *ps = ni_get_ps(radeon_state);
2267         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2268         int i;
2269
2270         for (i = 0; i < ps->performance_level_count - 1; i++)
2271                 smc_state->levels[i].bSP = cpu_to_be32(pi->dsp);
2272
2273         smc_state->levels[ps->performance_level_count - 1].bSP =
2274                 cpu_to_be32(pi->psp);
2275 }
2276
2277 static int ni_convert_power_level_to_smc(struct radeon_device *rdev,
2278                                          struct rv7xx_pl *pl,
2279                                          NISLANDS_SMC_HW_PERFORMANCE_LEVEL *level)
2280 {
2281         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2282         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2283         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2284         int ret;
2285         bool dll_state_on;
2286         u16 std_vddc;
2287         u32 tmp = RREG32(DC_STUTTER_CNTL);
2288
2289         level->gen2PCIE = pi->pcie_gen2 ?
2290                 ((pl->flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? 1 : 0) : 0;
2291
2292         ret = ni_populate_sclk_value(rdev, pl->sclk, &level->sclk);
2293         if (ret)
2294                 return ret;
2295
2296         level->mcFlags =  0;
2297         if (pi->mclk_stutter_mode_threshold &&
2298             (pl->mclk <= pi->mclk_stutter_mode_threshold) &&
2299             !eg_pi->uvd_enabled &&
2300             (tmp & DC_STUTTER_ENABLE_A) &&
2301             (tmp & DC_STUTTER_ENABLE_B))
2302                 level->mcFlags |= NISLANDS_SMC_MC_STUTTER_EN;
2303
2304         if (pi->mem_gddr5) {
2305                 if (pl->mclk > pi->mclk_edc_enable_threshold)
2306                         level->mcFlags |= NISLANDS_SMC_MC_EDC_RD_FLAG;
2307                 if (pl->mclk > eg_pi->mclk_edc_wr_enable_threshold)
2308                         level->mcFlags |= NISLANDS_SMC_MC_EDC_WR_FLAG;
2309
2310                 level->strobeMode = cypress_get_strobe_mode_settings(rdev, pl->mclk);
2311
2312                 if (level->strobeMode & NISLANDS_SMC_STROBE_ENABLE) {
2313                         if (cypress_get_mclk_frequency_ratio(rdev, pl->mclk, true) >=
2314                             ((RREG32(MC_SEQ_MISC7) >> 16) & 0xf))
2315                                 dll_state_on = ((RREG32(MC_SEQ_MISC5) >> 1) & 0x1) ? true : false;
2316                         else
2317                                 dll_state_on = ((RREG32(MC_SEQ_MISC6) >> 1) & 0x1) ? true : false;
2318                 } else {
2319                         dll_state_on = false;
2320                         if (pl->mclk > ni_pi->mclk_rtt_mode_threshold)
2321                                 level->mcFlags |= NISLANDS_SMC_MC_RTT_ENABLE;
2322                 }
2323
2324                 ret = ni_populate_mclk_value(rdev, pl->sclk, pl->mclk,
2325                                              &level->mclk,
2326                                              (level->strobeMode & NISLANDS_SMC_STROBE_ENABLE) != 0,
2327                                              dll_state_on);
2328         } else
2329                 ret = ni_populate_mclk_value(rdev, pl->sclk, pl->mclk, &level->mclk, 1, 1);
2330
2331         if (ret)
2332                 return ret;
2333
2334         ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
2335                                         pl->vddc, &level->vddc);
2336         if (ret)
2337                 return ret;
2338
2339         ret = ni_get_std_voltage_value(rdev, &level->vddc, &std_vddc);
2340         if (ret)
2341                 return ret;
2342
2343         ni_populate_std_voltage_value(rdev, std_vddc,
2344                                       level->vddc.index, &level->std_vddc);
2345
2346         if (eg_pi->vddci_control) {
2347                 ret = ni_populate_voltage_value(rdev, &eg_pi->vddci_voltage_table,
2348                                                 pl->vddci, &level->vddci);
2349                 if (ret)
2350                         return ret;
2351         }
2352
2353         ni_populate_mvdd_value(rdev, pl->mclk, &level->mvdd);
2354
2355         return ret;
2356 }
2357
2358 static int ni_populate_smc_t(struct radeon_device *rdev,
2359                              struct radeon_ps *radeon_state,
2360                              NISLANDS_SMC_SWSTATE *smc_state)
2361 {
2362         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2363         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2364         struct ni_ps *state = ni_get_ps(radeon_state);
2365         u32 a_t;
2366         u32 t_l, t_h;
2367         u32 high_bsp;
2368         int i, ret;
2369
2370         if (state->performance_level_count >= 9)
2371                 return -EINVAL;
2372
2373         if (state->performance_level_count < 2) {
2374                 a_t = CG_R(0xffff) | CG_L(0);
2375                 smc_state->levels[0].aT = cpu_to_be32(a_t);
2376                 return 0;
2377         }
2378
2379         smc_state->levels[0].aT = cpu_to_be32(0);
2380
2381         for (i = 0; i <= state->performance_level_count - 2; i++) {
2382                 if (eg_pi->uvd_enabled)
2383                         ret = r600_calculate_at(
2384                                 1000 * (i * (eg_pi->smu_uvd_hs ? 2 : 8) + 2),
2385                                 100 * R600_AH_DFLT,
2386                                 state->performance_levels[i + 1].sclk,
2387                                 state->performance_levels[i].sclk,
2388                                 &t_l,
2389                                 &t_h);
2390                 else
2391                         ret = r600_calculate_at(
2392                                 1000 * (i + 1),
2393                                 100 * R600_AH_DFLT,
2394                                 state->performance_levels[i + 1].sclk,
2395                                 state->performance_levels[i].sclk,
2396                                 &t_l,
2397                                 &t_h);
2398
2399                 if (ret) {
2400                         t_h = (i + 1) * 1000 - 50 * R600_AH_DFLT;
2401                         t_l = (i + 1) * 1000 + 50 * R600_AH_DFLT;
2402                 }
2403
2404                 a_t = be32_to_cpu(smc_state->levels[i].aT) & ~CG_R_MASK;
2405                 a_t |= CG_R(t_l * pi->bsp / 20000);
2406                 smc_state->levels[i].aT = cpu_to_be32(a_t);
2407
2408                 high_bsp = (i == state->performance_level_count - 2) ?
2409                         pi->pbsp : pi->bsp;
2410
2411                 a_t = CG_R(0xffff) | CG_L(t_h * high_bsp / 20000);
2412                 smc_state->levels[i + 1].aT = cpu_to_be32(a_t);
2413         }
2414
2415         return 0;
2416 }
2417
2418 static int ni_populate_power_containment_values(struct radeon_device *rdev,
2419                                                 struct radeon_ps *radeon_state,
2420                                                 NISLANDS_SMC_SWSTATE *smc_state)
2421 {
2422         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2423         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2424         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2425         struct ni_ps *state = ni_get_ps(radeon_state);
2426         u32 prev_sclk;
2427         u32 max_sclk;
2428         u32 min_sclk;
2429         int i, ret;
2430         u32 tdp_limit;
2431         u32 near_tdp_limit;
2432         u32 power_boost_limit;
2433         u8 max_ps_percent;
2434
2435         if (ni_pi->enable_power_containment == false)
2436                 return 0;
2437
2438         if (state->performance_level_count == 0)
2439                 return -EINVAL;
2440
2441         if (smc_state->levelCount != state->performance_level_count)
2442                 return -EINVAL;
2443
2444         ret = ni_calculate_adjusted_tdp_limits(rdev,
2445                                                false, /* ??? */
2446                                                rdev->pm.dpm.tdp_adjustment,
2447                                                &tdp_limit,
2448                                                &near_tdp_limit);
2449         if (ret)
2450                 return ret;
2451
2452         power_boost_limit = ni_calculate_power_boost_limit(rdev, radeon_state, near_tdp_limit);
2453
2454         ret = rv770_write_smc_sram_dword(rdev,
2455                                          pi->state_table_start +
2456                                          offsetof(NISLANDS_SMC_STATETABLE, dpm2Params) +
2457                                          offsetof(PP_NIslands_DPM2Parameters, PowerBoostLimit),
2458                                          ni_scale_power_for_smc(power_boost_limit, ni_get_smc_power_scaling_factor(rdev)),
2459                                          pi->sram_end);
2460         if (ret)
2461                 power_boost_limit = 0;
2462
2463         smc_state->levels[0].dpm2.MaxPS = 0;
2464         smc_state->levels[0].dpm2.NearTDPDec = 0;
2465         smc_state->levels[0].dpm2.AboveSafeInc = 0;
2466         smc_state->levels[0].dpm2.BelowSafeInc = 0;
2467         smc_state->levels[0].stateFlags |= power_boost_limit ? PPSMC_STATEFLAG_POWERBOOST : 0;
2468
2469         for (i = 1; i < state->performance_level_count; i++) {
2470                 prev_sclk = state->performance_levels[i-1].sclk;
2471                 max_sclk  = state->performance_levels[i].sclk;
2472                 max_ps_percent = (i != (state->performance_level_count - 1)) ?
2473                         NISLANDS_DPM2_MAXPS_PERCENT_M : NISLANDS_DPM2_MAXPS_PERCENT_H;
2474
2475                 if (max_sclk < prev_sclk)
2476                         return -EINVAL;
2477
2478                 if ((max_ps_percent == 0) || (prev_sclk == max_sclk) || eg_pi->uvd_enabled)
2479                         min_sclk = max_sclk;
2480                 else if (1 == i)
2481                         min_sclk = prev_sclk;
2482                 else
2483                         min_sclk = (prev_sclk * (u32)max_ps_percent) / 100;
2484
2485                 if (min_sclk < state->performance_levels[0].sclk)
2486                         min_sclk = state->performance_levels[0].sclk;
2487
2488                 if (min_sclk == 0)
2489                         return -EINVAL;
2490
2491                 smc_state->levels[i].dpm2.MaxPS =
2492                         (u8)((NISLANDS_DPM2_MAX_PULSE_SKIP * (max_sclk - min_sclk)) / max_sclk);
2493                 smc_state->levels[i].dpm2.NearTDPDec = NISLANDS_DPM2_NEAR_TDP_DEC;
2494                 smc_state->levels[i].dpm2.AboveSafeInc = NISLANDS_DPM2_ABOVE_SAFE_INC;
2495                 smc_state->levels[i].dpm2.BelowSafeInc = NISLANDS_DPM2_BELOW_SAFE_INC;
2496                 smc_state->levels[i].stateFlags |=
2497                         ((i != (state->performance_level_count - 1)) && power_boost_limit) ?
2498                         PPSMC_STATEFLAG_POWERBOOST : 0;
2499         }
2500
2501         return 0;
2502 }
2503
2504 static int ni_populate_sq_ramping_values(struct radeon_device *rdev,
2505                                          struct radeon_ps *radeon_state,
2506                                          NISLANDS_SMC_SWSTATE *smc_state)
2507 {
2508         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2509         struct ni_ps *state = ni_get_ps(radeon_state);
2510         u32 sq_power_throttle;
2511         u32 sq_power_throttle2;
2512         bool enable_sq_ramping = ni_pi->enable_sq_ramping;
2513         int i;
2514
2515         if (state->performance_level_count == 0)
2516                 return -EINVAL;
2517
2518         if (smc_state->levelCount != state->performance_level_count)
2519                 return -EINVAL;
2520
2521         if (rdev->pm.dpm.sq_ramping_threshold == 0)
2522                 return -EINVAL;
2523
2524         if (NISLANDS_DPM2_SQ_RAMP_MAX_POWER > (MAX_POWER_MASK >> MAX_POWER_SHIFT))
2525                 enable_sq_ramping = false;
2526
2527         if (NISLANDS_DPM2_SQ_RAMP_MIN_POWER > (MIN_POWER_MASK >> MIN_POWER_SHIFT))
2528                 enable_sq_ramping = false;
2529
2530         if (NISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA > (MAX_POWER_DELTA_MASK >> MAX_POWER_DELTA_SHIFT))
2531                 enable_sq_ramping = false;
2532
2533         if (NISLANDS_DPM2_SQ_RAMP_STI_SIZE > (STI_SIZE_MASK >> STI_SIZE_SHIFT))
2534                 enable_sq_ramping = false;
2535
2536         if (NISLANDS_DPM2_SQ_RAMP_LTI_RATIO <= (LTI_RATIO_MASK >> LTI_RATIO_SHIFT))
2537                 enable_sq_ramping = false;
2538
2539         for (i = 0; i < state->performance_level_count; i++) {
2540                 sq_power_throttle  = 0;
2541                 sq_power_throttle2 = 0;
2542
2543                 if ((state->performance_levels[i].sclk >= rdev->pm.dpm.sq_ramping_threshold) &&
2544                     enable_sq_ramping) {
2545                         sq_power_throttle |= MAX_POWER(NISLANDS_DPM2_SQ_RAMP_MAX_POWER);
2546                         sq_power_throttle |= MIN_POWER(NISLANDS_DPM2_SQ_RAMP_MIN_POWER);
2547                         sq_power_throttle2 |= MAX_POWER_DELTA(NISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA);
2548                         sq_power_throttle2 |= STI_SIZE(NISLANDS_DPM2_SQ_RAMP_STI_SIZE);
2549                         sq_power_throttle2 |= LTI_RATIO(NISLANDS_DPM2_SQ_RAMP_LTI_RATIO);
2550                 } else {
2551                         sq_power_throttle |= MAX_POWER_MASK | MIN_POWER_MASK;
2552                         sq_power_throttle2 |= MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
2553                 }
2554
2555                 smc_state->levels[i].SQPowerThrottle   = cpu_to_be32(sq_power_throttle);
2556                 smc_state->levels[i].SQPowerThrottle_2 = cpu_to_be32(sq_power_throttle2);
2557         }
2558
2559         return 0;
2560 }
2561
2562 static int ni_enable_power_containment(struct radeon_device *rdev,
2563                                        struct radeon_ps *radeon_new_state,
2564                                        bool enable)
2565 {
2566         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2567         PPSMC_Result smc_result;
2568         int ret = 0;
2569
2570         if (ni_pi->enable_power_containment) {
2571                 if (enable) {
2572                         if (!r600_is_uvd_state(radeon_new_state->class, radeon_new_state->class2)) {
2573                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_TDPClampingActive);
2574                                 if (smc_result != PPSMC_Result_OK) {
2575                                         ret = -EINVAL;
2576                                         ni_pi->pc_enabled = false;
2577                                 } else {
2578                                         ni_pi->pc_enabled = true;
2579                                 }
2580                         }
2581                 } else {
2582                         smc_result = rv770_send_msg_to_smc(rdev, PPSMC_TDPClampingInactive);
2583                         if (smc_result != PPSMC_Result_OK)
2584                                 ret = -EINVAL;
2585                         ni_pi->pc_enabled = false;
2586                 }
2587         }
2588
2589         return ret;
2590 }
2591
2592 static int ni_convert_power_state_to_smc(struct radeon_device *rdev,
2593                                          struct radeon_ps *radeon_state,
2594                                          NISLANDS_SMC_SWSTATE *smc_state)
2595 {
2596         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2597         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2598         struct ni_ps *state = ni_get_ps(radeon_state);
2599         int i, ret;
2600         u32 threshold = state->performance_levels[state->performance_level_count - 1].sclk * 100 / 100;
2601
2602         if (!(radeon_state->caps & ATOM_PPLIB_DISALLOW_ON_DC))
2603                 smc_state->flags |= PPSMC_SWSTATE_FLAG_DC;
2604
2605         smc_state->levelCount = 0;
2606
2607         if (state->performance_level_count > NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE)
2608                 return -EINVAL;
2609
2610         for (i = 0; i < state->performance_level_count; i++) {
2611                 ret = ni_convert_power_level_to_smc(rdev, &state->performance_levels[i],
2612                                                     &smc_state->levels[i]);
2613                 smc_state->levels[i].arbRefreshState =
2614                         (u8)(NISLANDS_DRIVER_STATE_ARB_INDEX + i);
2615
2616                 if (ret)
2617                         return ret;
2618
2619                 if (ni_pi->enable_power_containment)
2620                         smc_state->levels[i].displayWatermark =
2621                                 (state->performance_levels[i].sclk < threshold) ?
2622                                 PPSMC_DISPLAY_WATERMARK_LOW : PPSMC_DISPLAY_WATERMARK_HIGH;
2623                 else
2624                         smc_state->levels[i].displayWatermark = (i < 2) ?
2625                                 PPSMC_DISPLAY_WATERMARK_LOW : PPSMC_DISPLAY_WATERMARK_HIGH;
2626
2627                 if (eg_pi->dynamic_ac_timing)
2628                         smc_state->levels[i].ACIndex = NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT + i;
2629                 else
2630                         smc_state->levels[i].ACIndex = 0;
2631
2632                 smc_state->levelCount++;
2633         }
2634
2635         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_watermark_threshold,
2636                                       cpu_to_be32(threshold / 512));
2637
2638         ni_populate_smc_sp(rdev, radeon_state, smc_state);
2639
2640         ret = ni_populate_power_containment_values(rdev, radeon_state, smc_state);
2641         if (ret)
2642                 ni_pi->enable_power_containment = false;
2643
2644         ret = ni_populate_sq_ramping_values(rdev, radeon_state, smc_state);
2645         if (ret)
2646                 ni_pi->enable_sq_ramping = false;
2647
2648         return ni_populate_smc_t(rdev, radeon_state, smc_state);
2649 }
2650
2651 static int ni_upload_sw_state(struct radeon_device *rdev,
2652                               struct radeon_ps *radeon_new_state)
2653 {
2654         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2655         u16 address = pi->state_table_start +
2656                 offsetof(NISLANDS_SMC_STATETABLE, driverState);
2657         u16 state_size = sizeof(NISLANDS_SMC_SWSTATE) +
2658                 ((NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1) * sizeof(NISLANDS_SMC_HW_PERFORMANCE_LEVEL));
2659         int ret;
2660         NISLANDS_SMC_SWSTATE *smc_state = kzalloc(state_size, GFP_KERNEL);
2661
2662         if (smc_state == NULL)
2663                 return -ENOMEM;
2664
2665         ret = ni_convert_power_state_to_smc(rdev, radeon_new_state, smc_state);
2666         if (ret)
2667                 goto done;
2668
2669         ret = rv770_copy_bytes_to_smc(rdev, address, (u8 *)smc_state, state_size, pi->sram_end);
2670
2671 done:
2672         kfree(smc_state);
2673
2674         return ret;
2675 }
2676
2677 static int ni_set_mc_special_registers(struct radeon_device *rdev,
2678                                        struct ni_mc_reg_table *table)
2679 {
2680         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2681         u8 i, j, k;
2682         u32 temp_reg;
2683
2684         for (i = 0, j = table->last; i < table->last; i++) {
2685                 switch (table->mc_reg_address[i].s1) {
2686                 case MC_SEQ_MISC1 >> 2:
2687                         if (j >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2688                                 return -EINVAL;
2689                         temp_reg = RREG32(MC_PMG_CMD_EMRS);
2690                         table->mc_reg_address[j].s1 = MC_PMG_CMD_EMRS >> 2;
2691                         table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_EMRS_LP >> 2;
2692                         for (k = 0; k < table->num_entries; k++)
2693                                 table->mc_reg_table_entry[k].mc_data[j] =
2694                                         ((temp_reg & 0xffff0000)) |
2695                                         ((table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16);
2696                         j++;
2697                         if (j >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2698                                 return -EINVAL;
2699
2700                         temp_reg = RREG32(MC_PMG_CMD_MRS);
2701                         table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS >> 2;
2702                         table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS_LP >> 2;
2703                         for(k = 0; k < table->num_entries; k++) {
2704                                 table->mc_reg_table_entry[k].mc_data[j] =
2705                                         (temp_reg & 0xffff0000) |
2706                                         (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
2707                                 if (!pi->mem_gddr5)
2708                                         table->mc_reg_table_entry[k].mc_data[j] |= 0x100;
2709                         }
2710                         j++;
2711                         if (j > SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2712                                 return -EINVAL;
2713                         break;
2714                 case MC_SEQ_RESERVE_M >> 2:
2715                         temp_reg = RREG32(MC_PMG_CMD_MRS1);
2716                         table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS1 >> 2;
2717                         table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS1_LP >> 2;
2718                         for (k = 0; k < table->num_entries; k++)
2719                                 table->mc_reg_table_entry[k].mc_data[j] =
2720                                         (temp_reg & 0xffff0000) |
2721                                         (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
2722                         j++;
2723                         if (j > SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2724                                 return -EINVAL;
2725                         break;
2726                 default:
2727                         break;
2728                 }
2729         }
2730
2731         table->last = j;
2732
2733         return 0;
2734 }
2735
2736 static bool ni_check_s0_mc_reg_index(u16 in_reg, u16 *out_reg)
2737 {
2738         bool result = true;
2739
2740         switch (in_reg) {
2741         case  MC_SEQ_RAS_TIMING >> 2:
2742                 *out_reg = MC_SEQ_RAS_TIMING_LP >> 2;
2743                 break;
2744         case MC_SEQ_CAS_TIMING >> 2:
2745                 *out_reg = MC_SEQ_CAS_TIMING_LP >> 2;
2746                 break;
2747         case MC_SEQ_MISC_TIMING >> 2:
2748                 *out_reg = MC_SEQ_MISC_TIMING_LP >> 2;
2749                 break;
2750         case MC_SEQ_MISC_TIMING2 >> 2:
2751                 *out_reg = MC_SEQ_MISC_TIMING2_LP >> 2;
2752                 break;
2753         case MC_SEQ_RD_CTL_D0 >> 2:
2754                 *out_reg = MC_SEQ_RD_CTL_D0_LP >> 2;
2755                 break;
2756         case MC_SEQ_RD_CTL_D1 >> 2:
2757                 *out_reg = MC_SEQ_RD_CTL_D1_LP >> 2;
2758                 break;
2759         case MC_SEQ_WR_CTL_D0 >> 2:
2760                 *out_reg = MC_SEQ_WR_CTL_D0_LP >> 2;
2761                 break;
2762         case MC_SEQ_WR_CTL_D1 >> 2:
2763                 *out_reg = MC_SEQ_WR_CTL_D1_LP >> 2;
2764                 break;
2765         case MC_PMG_CMD_EMRS >> 2:
2766                 *out_reg = MC_SEQ_PMG_CMD_EMRS_LP >> 2;
2767                 break;
2768         case MC_PMG_CMD_MRS >> 2:
2769                 *out_reg = MC_SEQ_PMG_CMD_MRS_LP >> 2;
2770                 break;
2771         case MC_PMG_CMD_MRS1 >> 2:
2772                 *out_reg = MC_SEQ_PMG_CMD_MRS1_LP >> 2;
2773                 break;
2774         case MC_SEQ_PMG_TIMING >> 2:
2775                 *out_reg = MC_SEQ_PMG_TIMING_LP >> 2;
2776                 break;
2777         case MC_PMG_CMD_MRS2 >> 2:
2778                 *out_reg = MC_SEQ_PMG_CMD_MRS2_LP >> 2;
2779                 break;
2780         default:
2781                 result = false;
2782                 break;
2783         }
2784
2785         return result;
2786 }
2787
2788 static void ni_set_valid_flag(struct ni_mc_reg_table *table)
2789 {
2790         u8 i, j;
2791
2792         for (i = 0; i < table->last; i++) {
2793                 for (j = 1; j < table->num_entries; j++) {
2794                         if (table->mc_reg_table_entry[j-1].mc_data[i] != table->mc_reg_table_entry[j].mc_data[i]) {
2795                                 table->valid_flag |= 1 << i;
2796                                 break;
2797                         }
2798                 }
2799         }
2800 }
2801
2802 static void ni_set_s0_mc_reg_index(struct ni_mc_reg_table *table)
2803 {
2804         u32 i;
2805         u16 address;
2806
2807         for (i = 0; i < table->last; i++)
2808                 table->mc_reg_address[i].s0 =
2809                         ni_check_s0_mc_reg_index(table->mc_reg_address[i].s1, &address) ?
2810                         address : table->mc_reg_address[i].s1;
2811 }
2812
2813 static int ni_copy_vbios_mc_reg_table(struct atom_mc_reg_table *table,
2814                                       struct ni_mc_reg_table *ni_table)
2815 {
2816         u8 i, j;
2817
2818         if (table->last > SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2819                 return -EINVAL;
2820         if (table->num_entries > MAX_AC_TIMING_ENTRIES)
2821                 return -EINVAL;
2822
2823         for (i = 0; i < table->last; i++)
2824                 ni_table->mc_reg_address[i].s1 = table->mc_reg_address[i].s1;
2825         ni_table->last = table->last;
2826
2827         for (i = 0; i < table->num_entries; i++) {
2828                 ni_table->mc_reg_table_entry[i].mclk_max =
2829                         table->mc_reg_table_entry[i].mclk_max;
2830                 for (j = 0; j < table->last; j++)
2831                         ni_table->mc_reg_table_entry[i].mc_data[j] =
2832                                 table->mc_reg_table_entry[i].mc_data[j];
2833         }
2834         ni_table->num_entries = table->num_entries;
2835
2836         return 0;
2837 }
2838
2839 static int ni_initialize_mc_reg_table(struct radeon_device *rdev)
2840 {
2841         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2842         int ret;
2843         struct atom_mc_reg_table *table;
2844         struct ni_mc_reg_table *ni_table = &ni_pi->mc_reg_table;
2845         u8 module_index = rv770_get_memory_module_index(rdev);
2846
2847         table = kzalloc(sizeof(struct atom_mc_reg_table), GFP_KERNEL);
2848         if (!table)
2849                 return -ENOMEM;
2850
2851         WREG32(MC_SEQ_RAS_TIMING_LP, RREG32(MC_SEQ_RAS_TIMING));
2852         WREG32(MC_SEQ_CAS_TIMING_LP, RREG32(MC_SEQ_CAS_TIMING));
2853         WREG32(MC_SEQ_MISC_TIMING_LP, RREG32(MC_SEQ_MISC_TIMING));
2854         WREG32(MC_SEQ_MISC_TIMING2_LP, RREG32(MC_SEQ_MISC_TIMING2));
2855         WREG32(MC_SEQ_PMG_CMD_EMRS_LP, RREG32(MC_PMG_CMD_EMRS));
2856         WREG32(MC_SEQ_PMG_CMD_MRS_LP, RREG32(MC_PMG_CMD_MRS));
2857         WREG32(MC_SEQ_PMG_CMD_MRS1_LP, RREG32(MC_PMG_CMD_MRS1));
2858         WREG32(MC_SEQ_WR_CTL_D0_LP, RREG32(MC_SEQ_WR_CTL_D0));
2859         WREG32(MC_SEQ_WR_CTL_D1_LP, RREG32(MC_SEQ_WR_CTL_D1));
2860         WREG32(MC_SEQ_RD_CTL_D0_LP, RREG32(MC_SEQ_RD_CTL_D0));
2861         WREG32(MC_SEQ_RD_CTL_D1_LP, RREG32(MC_SEQ_RD_CTL_D1));
2862         WREG32(MC_SEQ_PMG_TIMING_LP, RREG32(MC_SEQ_PMG_TIMING));
2863         WREG32(MC_SEQ_PMG_CMD_MRS2_LP, RREG32(MC_PMG_CMD_MRS2));
2864
2865         ret = radeon_atom_init_mc_reg_table(rdev, module_index, table);
2866
2867         if (ret)
2868                 goto init_mc_done;
2869
2870         ret = ni_copy_vbios_mc_reg_table(table, ni_table);
2871
2872         if (ret)
2873                 goto init_mc_done;
2874
2875         ni_set_s0_mc_reg_index(ni_table);
2876
2877         ret = ni_set_mc_special_registers(rdev, ni_table);
2878
2879         if (ret)
2880                 goto init_mc_done;
2881
2882         ni_set_valid_flag(ni_table);
2883
2884 init_mc_done:
2885         kfree(table);
2886
2887         return ret;
2888 }
2889
2890 static void ni_populate_mc_reg_addresses(struct radeon_device *rdev,
2891                                          SMC_NIslands_MCRegisters *mc_reg_table)
2892 {
2893         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2894         u32 i, j;
2895
2896         for (i = 0, j = 0; j < ni_pi->mc_reg_table.last; j++) {
2897                 if (ni_pi->mc_reg_table.valid_flag & (1 << j)) {
2898                         if (i >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2899                                 break;
2900                         mc_reg_table->address[i].s0 =
2901                                 cpu_to_be16(ni_pi->mc_reg_table.mc_reg_address[j].s0);
2902                         mc_reg_table->address[i].s1 =
2903                                 cpu_to_be16(ni_pi->mc_reg_table.mc_reg_address[j].s1);
2904                         i++;
2905                 }
2906         }
2907         mc_reg_table->last = (u8)i;
2908 }
2909
2910
2911 static void ni_convert_mc_registers(struct ni_mc_reg_entry *entry,
2912                                     SMC_NIslands_MCRegisterSet *data,
2913                                     u32 num_entries, u32 valid_flag)
2914 {
2915         u32 i, j;
2916
2917         for (i = 0, j = 0; j < num_entries; j++) {
2918                 if (valid_flag & (1 << j)) {
2919                         data->value[i] = cpu_to_be32(entry->mc_data[j]);
2920                         i++;
2921                 }
2922         }
2923 }
2924
2925 static void ni_convert_mc_reg_table_entry_to_smc(struct radeon_device *rdev,
2926                                                  struct rv7xx_pl *pl,
2927                                                  SMC_NIslands_MCRegisterSet *mc_reg_table_data)
2928 {
2929         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2930         u32 i = 0;
2931
2932         for (i = 0; i < ni_pi->mc_reg_table.num_entries; i++) {
2933                 if (pl->mclk <= ni_pi->mc_reg_table.mc_reg_table_entry[i].mclk_max)
2934                         break;
2935         }
2936
2937         if ((i == ni_pi->mc_reg_table.num_entries) && (i > 0))
2938                 --i;
2939
2940         ni_convert_mc_registers(&ni_pi->mc_reg_table.mc_reg_table_entry[i],
2941                                 mc_reg_table_data,
2942                                 ni_pi->mc_reg_table.last,
2943                                 ni_pi->mc_reg_table.valid_flag);
2944 }
2945
2946 static void ni_convert_mc_reg_table_to_smc(struct radeon_device *rdev,
2947                                            struct radeon_ps *radeon_state,
2948                                            SMC_NIslands_MCRegisters *mc_reg_table)
2949 {
2950         struct ni_ps *state = ni_get_ps(radeon_state);
2951         int i;
2952
2953         for (i = 0; i < state->performance_level_count; i++) {
2954                 ni_convert_mc_reg_table_entry_to_smc(rdev,
2955                                                      &state->performance_levels[i],
2956                                                      &mc_reg_table->data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT + i]);
2957         }
2958 }
2959
2960 static int ni_populate_mc_reg_table(struct radeon_device *rdev,
2961                                     struct radeon_ps *radeon_boot_state)
2962 {
2963         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2964         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2965         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2966         struct ni_ps *boot_state = ni_get_ps(radeon_boot_state);
2967         SMC_NIslands_MCRegisters *mc_reg_table = &ni_pi->smc_mc_reg_table;
2968
2969         memset(mc_reg_table, 0, sizeof(SMC_NIslands_MCRegisters));
2970
2971         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_seq_index, 1);
2972
2973         ni_populate_mc_reg_addresses(rdev, mc_reg_table);
2974
2975         ni_convert_mc_reg_table_entry_to_smc(rdev, &boot_state->performance_levels[0],
2976                                              &mc_reg_table->data[0]);
2977
2978         ni_convert_mc_registers(&ni_pi->mc_reg_table.mc_reg_table_entry[0],
2979                                 &mc_reg_table->data[1],
2980                                 ni_pi->mc_reg_table.last,
2981                                 ni_pi->mc_reg_table.valid_flag);
2982
2983         ni_convert_mc_reg_table_to_smc(rdev, radeon_boot_state, mc_reg_table);
2984
2985         return rv770_copy_bytes_to_smc(rdev, eg_pi->mc_reg_table_start,
2986                                        (u8 *)mc_reg_table,
2987                                        sizeof(SMC_NIslands_MCRegisters),
2988                                        pi->sram_end);
2989 }
2990
2991 static int ni_upload_mc_reg_table(struct radeon_device *rdev,
2992                                   struct radeon_ps *radeon_new_state)
2993 {
2994         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2995         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2996         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2997         struct ni_ps *ni_new_state = ni_get_ps(radeon_new_state);
2998         SMC_NIslands_MCRegisters *mc_reg_table = &ni_pi->smc_mc_reg_table;
2999         u16 address;
3000
3001         memset(mc_reg_table, 0, sizeof(SMC_NIslands_MCRegisters));
3002
3003         ni_convert_mc_reg_table_to_smc(rdev, radeon_new_state, mc_reg_table);
3004
3005         address = eg_pi->mc_reg_table_start +
3006                 (u16)offsetof(SMC_NIslands_MCRegisters, data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT]);
3007
3008         return rv770_copy_bytes_to_smc(rdev, address,
3009                                        (u8 *)&mc_reg_table->data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT],
3010                                        sizeof(SMC_NIslands_MCRegisterSet) * ni_new_state->performance_level_count,
3011                                        pi->sram_end);
3012 }
3013
3014 static int ni_init_driver_calculated_leakage_table(struct radeon_device *rdev,
3015                                                    PP_NIslands_CACTABLES *cac_tables)
3016 {
3017         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3018         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3019         u32 leakage = 0;
3020         unsigned int i, j, table_size;
3021         s32 t;
3022         u32 smc_leakage, max_leakage = 0;
3023         u32 scaling_factor;
3024
3025         table_size = eg_pi->vddc_voltage_table.count;
3026
3027         if (SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES < table_size)
3028                 table_size = SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES;
3029
3030         scaling_factor = ni_get_smc_power_scaling_factor(rdev);
3031
3032         for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++) {
3033                 for (j = 0; j < table_size; j++) {
3034                         t = (1000 * ((i + 1) * 8));
3035
3036                         if (t < ni_pi->cac_data.leakage_minimum_temperature)
3037                                 t = ni_pi->cac_data.leakage_minimum_temperature;
3038
3039                         ni_calculate_leakage_for_v_and_t(rdev,
3040                                                          &ni_pi->cac_data.leakage_coefficients,
3041                                                          eg_pi->vddc_voltage_table.entries[j].value,
3042                                                          t,
3043                                                          ni_pi->cac_data.i_leakage,
3044                                                          &leakage);
3045
3046                         smc_leakage = ni_scale_power_for_smc(leakage, scaling_factor) / 1000;
3047                         if (smc_leakage > max_leakage)
3048                                 max_leakage = smc_leakage;
3049
3050                         cac_tables->cac_lkge_lut[i][j] = cpu_to_be32(smc_leakage);
3051                 }
3052         }
3053
3054         for (j = table_size; j < SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; j++) {
3055                 for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
3056                         cac_tables->cac_lkge_lut[i][j] = cpu_to_be32(max_leakage);
3057         }
3058         return 0;
3059 }
3060
3061 static int ni_init_simplified_leakage_table(struct radeon_device *rdev,
3062                                             PP_NIslands_CACTABLES *cac_tables)
3063 {
3064         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3065         struct radeon_cac_leakage_table *leakage_table =
3066                 &rdev->pm.dpm.dyn_state.cac_leakage_table;
3067         u32 i, j, table_size;
3068         u32 smc_leakage, max_leakage = 0;
3069         u32 scaling_factor;
3070
3071         if (!leakage_table)
3072                 return -EINVAL;
3073
3074         table_size = leakage_table->count;
3075
3076         if (eg_pi->vddc_voltage_table.count != table_size)
3077                 table_size = (eg_pi->vddc_voltage_table.count < leakage_table->count) ?
3078                         eg_pi->vddc_voltage_table.count : leakage_table->count;
3079
3080         if (SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES < table_size)
3081                 table_size = SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES;
3082
3083         if (table_size == 0)
3084                 return -EINVAL;
3085
3086         scaling_factor = ni_get_smc_power_scaling_factor(rdev);
3087
3088         for (j = 0; j < table_size; j++) {
3089                 smc_leakage = leakage_table->entries[j].leakage;
3090
3091                 if (smc_leakage > max_leakage)
3092                         max_leakage = smc_leakage;
3093
3094                 for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
3095                         cac_tables->cac_lkge_lut[i][j] =
3096                                 cpu_to_be32(ni_scale_power_for_smc(smc_leakage, scaling_factor));
3097         }
3098
3099         for (j = table_size; j < SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; j++) {
3100                 for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
3101                         cac_tables->cac_lkge_lut[i][j] =
3102                                 cpu_to_be32(ni_scale_power_for_smc(max_leakage, scaling_factor));
3103         }
3104         return 0;
3105 }
3106
3107 static int ni_initialize_smc_cac_tables(struct radeon_device *rdev)
3108 {
3109         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3110         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3111         PP_NIslands_CACTABLES *cac_tables = NULL;
3112         int i, ret;
3113         u32 reg;
3114
3115         if (ni_pi->enable_cac == false)
3116                 return 0;
3117
3118         cac_tables = kzalloc(sizeof(PP_NIslands_CACTABLES), GFP_KERNEL);
3119         if (!cac_tables)
3120                 return -ENOMEM;
3121
3122         reg = RREG32(CG_CAC_CTRL) & ~(TID_CNT_MASK | TID_UNIT_MASK);
3123         reg |= (TID_CNT(ni_pi->cac_weights->tid_cnt) |
3124                 TID_UNIT(ni_pi->cac_weights->tid_unit));
3125         WREG32(CG_CAC_CTRL, reg);
3126
3127         for (i = 0; i < NISLANDS_DCCAC_MAX_LEVELS; i++)
3128                 ni_pi->dc_cac_table[i] = ni_pi->cac_weights->dc_cac[i];
3129
3130         for (i = 0; i < SMC_NISLANDS_BIF_LUT_NUM_OF_ENTRIES; i++)
3131                 cac_tables->cac_bif_lut[i] = ni_pi->cac_weights->pcie_cac[i];
3132
3133         ni_pi->cac_data.i_leakage = rdev->pm.dpm.cac_leakage;
3134         ni_pi->cac_data.pwr_const = 0;
3135         ni_pi->cac_data.dc_cac_value = ni_pi->dc_cac_table[NISLANDS_DCCAC_LEVEL_0];
3136         ni_pi->cac_data.bif_cac_value = 0;
3137         ni_pi->cac_data.mc_wr_weight = ni_pi->cac_weights->mc_write_weight;
3138         ni_pi->cac_data.mc_rd_weight = ni_pi->cac_weights->mc_read_weight;
3139         ni_pi->cac_data.allow_ovrflw = 0;
3140         ni_pi->cac_data.l2num_win_tdp = ni_pi->lta_window_size;
3141         ni_pi->cac_data.num_win_tdp = 0;
3142         ni_pi->cac_data.lts_truncate_n = ni_pi->lts_truncate;
3143
3144         if (ni_pi->driver_calculate_cac_leakage)
3145                 ret = ni_init_driver_calculated_leakage_table(rdev, cac_tables);
3146         else
3147                 ret = ni_init_simplified_leakage_table(rdev, cac_tables);
3148
3149         if (ret)
3150                 goto done_free;
3151
3152         cac_tables->pwr_const      = cpu_to_be32(ni_pi->cac_data.pwr_const);
3153         cac_tables->dc_cacValue    = cpu_to_be32(ni_pi->cac_data.dc_cac_value);
3154         cac_tables->bif_cacValue   = cpu_to_be32(ni_pi->cac_data.bif_cac_value);
3155         cac_tables->AllowOvrflw    = ni_pi->cac_data.allow_ovrflw;
3156         cac_tables->MCWrWeight     = ni_pi->cac_data.mc_wr_weight;
3157         cac_tables->MCRdWeight     = ni_pi->cac_data.mc_rd_weight;
3158         cac_tables->numWin_TDP     = ni_pi->cac_data.num_win_tdp;
3159         cac_tables->l2numWin_TDP   = ni_pi->cac_data.l2num_win_tdp;
3160         cac_tables->lts_truncate_n = ni_pi->cac_data.lts_truncate_n;
3161
3162         ret = rv770_copy_bytes_to_smc(rdev, ni_pi->cac_table_start, (u8 *)cac_tables,
3163                                       sizeof(PP_NIslands_CACTABLES), pi->sram_end);
3164
3165 done_free:
3166         if (ret) {
3167                 ni_pi->enable_cac = false;
3168                 ni_pi->enable_power_containment = false;
3169         }
3170
3171         kfree(cac_tables);
3172
3173         return 0;
3174 }
3175
3176 static int ni_initialize_hardware_cac_manager(struct radeon_device *rdev)
3177 {
3178         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3179         u32 reg;
3180
3181         if (!ni_pi->enable_cac ||
3182             !ni_pi->cac_configuration_required)
3183                 return 0;
3184
3185         if (ni_pi->cac_weights == NULL)
3186                 return -EINVAL;
3187
3188         reg = RREG32_CG(CG_CAC_REGION_1_WEIGHT_0) & ~(WEIGHT_TCP_SIG0_MASK |
3189                                                       WEIGHT_TCP_SIG1_MASK |
3190                                                       WEIGHT_TA_SIG_MASK);
3191         reg |= (WEIGHT_TCP_SIG0(ni_pi->cac_weights->weight_tcp_sig0) |
3192                 WEIGHT_TCP_SIG1(ni_pi->cac_weights->weight_tcp_sig1) |
3193                 WEIGHT_TA_SIG(ni_pi->cac_weights->weight_ta_sig));
3194         WREG32_CG(CG_CAC_REGION_1_WEIGHT_0, reg);
3195
3196         reg = RREG32_CG(CG_CAC_REGION_1_WEIGHT_1) & ~(WEIGHT_TCC_EN0_MASK |
3197                                                       WEIGHT_TCC_EN1_MASK |
3198                                                       WEIGHT_TCC_EN2_MASK);
3199         reg |= (WEIGHT_TCC_EN0(ni_pi->cac_weights->weight_tcc_en0) |
3200                 WEIGHT_TCC_EN1(ni_pi->cac_weights->weight_tcc_en1) |
3201                 WEIGHT_TCC_EN2(ni_pi->cac_weights->weight_tcc_en2));
3202         WREG32_CG(CG_CAC_REGION_1_WEIGHT_1, reg);
3203
3204         reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_0) & ~(WEIGHT_CB_EN0_MASK |
3205                                                       WEIGHT_CB_EN1_MASK |
3206                                                       WEIGHT_CB_EN2_MASK |
3207                                                       WEIGHT_CB_EN3_MASK);
3208         reg |= (WEIGHT_CB_EN0(ni_pi->cac_weights->weight_cb_en0) |
3209                 WEIGHT_CB_EN1(ni_pi->cac_weights->weight_cb_en1) |
3210                 WEIGHT_CB_EN2(ni_pi->cac_weights->weight_cb_en2) |
3211                 WEIGHT_CB_EN3(ni_pi->cac_weights->weight_cb_en3));
3212         WREG32_CG(CG_CAC_REGION_2_WEIGHT_0, reg);
3213
3214         reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_1) & ~(WEIGHT_DB_SIG0_MASK |
3215                                                       WEIGHT_DB_SIG1_MASK |
3216                                                       WEIGHT_DB_SIG2_MASK |
3217                                                       WEIGHT_DB_SIG3_MASK);
3218         reg |= (WEIGHT_DB_SIG0(ni_pi->cac_weights->weight_db_sig0) |
3219                 WEIGHT_DB_SIG1(ni_pi->cac_weights->weight_db_sig1) |
3220                 WEIGHT_DB_SIG2(ni_pi->cac_weights->weight_db_sig2) |
3221                 WEIGHT_DB_SIG3(ni_pi->cac_weights->weight_db_sig3));
3222         WREG32_CG(CG_CAC_REGION_2_WEIGHT_1, reg);
3223
3224         reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_2) & ~(WEIGHT_SXM_SIG0_MASK |
3225                                                       WEIGHT_SXM_SIG1_MASK |
3226                                                       WEIGHT_SXM_SIG2_MASK |
3227                                                       WEIGHT_SXS_SIG0_MASK |
3228                                                       WEIGHT_SXS_SIG1_MASK);
3229         reg |= (WEIGHT_SXM_SIG0(ni_pi->cac_weights->weight_sxm_sig0) |
3230                 WEIGHT_SXM_SIG1(ni_pi->cac_weights->weight_sxm_sig1) |
3231                 WEIGHT_SXM_SIG2(ni_pi->cac_weights->weight_sxm_sig2) |
3232                 WEIGHT_SXS_SIG0(ni_pi->cac_weights->weight_sxs_sig0) |
3233                 WEIGHT_SXS_SIG1(ni_pi->cac_weights->weight_sxs_sig1));
3234         WREG32_CG(CG_CAC_REGION_2_WEIGHT_2, reg);
3235
3236         reg = RREG32_CG(CG_CAC_REGION_3_WEIGHT_0) & ~(WEIGHT_XBR_0_MASK |
3237                                                       WEIGHT_XBR_1_MASK |
3238                                                       WEIGHT_XBR_2_MASK |
3239                                                       WEIGHT_SPI_SIG0_MASK);
3240         reg |= (WEIGHT_XBR_0(ni_pi->cac_weights->weight_xbr_0) |
3241                 WEIGHT_XBR_1(ni_pi->cac_weights->weight_xbr_1) |
3242                 WEIGHT_XBR_2(ni_pi->cac_weights->weight_xbr_2) |
3243                 WEIGHT_SPI_SIG0(ni_pi->cac_weights->weight_spi_sig0));
3244         WREG32_CG(CG_CAC_REGION_3_WEIGHT_0, reg);
3245
3246         reg = RREG32_CG(CG_CAC_REGION_3_WEIGHT_1) & ~(WEIGHT_SPI_SIG1_MASK |
3247                                                       WEIGHT_SPI_SIG2_MASK |
3248                                                       WEIGHT_SPI_SIG3_MASK |
3249                                                       WEIGHT_SPI_SIG4_MASK |
3250                                                       WEIGHT_SPI_SIG5_MASK);
3251         reg |= (WEIGHT_SPI_SIG1(ni_pi->cac_weights->weight_spi_sig1) |
3252                 WEIGHT_SPI_SIG2(ni_pi->cac_weights->weight_spi_sig2) |
3253                 WEIGHT_SPI_SIG3(ni_pi->cac_weights->weight_spi_sig3) |
3254                 WEIGHT_SPI_SIG4(ni_pi->cac_weights->weight_spi_sig4) |
3255                 WEIGHT_SPI_SIG5(ni_pi->cac_weights->weight_spi_sig5));
3256         WREG32_CG(CG_CAC_REGION_3_WEIGHT_1, reg);
3257
3258         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_0) & ~(WEIGHT_LDS_SIG0_MASK |
3259                                                       WEIGHT_LDS_SIG1_MASK |
3260                                                       WEIGHT_SC_MASK);
3261         reg |= (WEIGHT_LDS_SIG0(ni_pi->cac_weights->weight_lds_sig0) |
3262                 WEIGHT_LDS_SIG1(ni_pi->cac_weights->weight_lds_sig1) |
3263                 WEIGHT_SC(ni_pi->cac_weights->weight_sc));
3264         WREG32_CG(CG_CAC_REGION_4_WEIGHT_0, reg);
3265
3266         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_1) & ~(WEIGHT_BIF_MASK |
3267                                                       WEIGHT_CP_MASK |
3268                                                       WEIGHT_PA_SIG0_MASK |
3269                                                       WEIGHT_PA_SIG1_MASK |
3270                                                       WEIGHT_VGT_SIG0_MASK);
3271         reg |= (WEIGHT_BIF(ni_pi->cac_weights->weight_bif) |
3272                 WEIGHT_CP(ni_pi->cac_weights->weight_cp) |
3273                 WEIGHT_PA_SIG0(ni_pi->cac_weights->weight_pa_sig0) |
3274                 WEIGHT_PA_SIG1(ni_pi->cac_weights->weight_pa_sig1) |
3275                 WEIGHT_VGT_SIG0(ni_pi->cac_weights->weight_vgt_sig0));
3276         WREG32_CG(CG_CAC_REGION_4_WEIGHT_1, reg);
3277
3278         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_2) & ~(WEIGHT_VGT_SIG1_MASK |
3279                                                       WEIGHT_VGT_SIG2_MASK |
3280                                                       WEIGHT_DC_SIG0_MASK |
3281                                                       WEIGHT_DC_SIG1_MASK |
3282                                                       WEIGHT_DC_SIG2_MASK);
3283         reg |= (WEIGHT_VGT_SIG1(ni_pi->cac_weights->weight_vgt_sig1) |
3284                 WEIGHT_VGT_SIG2(ni_pi->cac_weights->weight_vgt_sig2) |
3285                 WEIGHT_DC_SIG0(ni_pi->cac_weights->weight_dc_sig0) |
3286                 WEIGHT_DC_SIG1(ni_pi->cac_weights->weight_dc_sig1) |
3287                 WEIGHT_DC_SIG2(ni_pi->cac_weights->weight_dc_sig2));
3288         WREG32_CG(CG_CAC_REGION_4_WEIGHT_2, reg);
3289
3290         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_3) & ~(WEIGHT_DC_SIG3_MASK |
3291                                                       WEIGHT_UVD_SIG0_MASK |
3292                                                       WEIGHT_UVD_SIG1_MASK |
3293                                                       WEIGHT_SPARE0_MASK |
3294                                                       WEIGHT_SPARE1_MASK);
3295         reg |= (WEIGHT_DC_SIG3(ni_pi->cac_weights->weight_dc_sig3) |
3296                 WEIGHT_UVD_SIG0(ni_pi->cac_weights->weight_uvd_sig0) |
3297                 WEIGHT_UVD_SIG1(ni_pi->cac_weights->weight_uvd_sig1) |
3298                 WEIGHT_SPARE0(ni_pi->cac_weights->weight_spare0) |
3299                 WEIGHT_SPARE1(ni_pi->cac_weights->weight_spare1));
3300         WREG32_CG(CG_CAC_REGION_4_WEIGHT_3, reg);
3301
3302         reg = RREG32_CG(CG_CAC_REGION_5_WEIGHT_0) & ~(WEIGHT_SQ_VSP_MASK |
3303                                                       WEIGHT_SQ_VSP0_MASK);
3304         reg |= (WEIGHT_SQ_VSP(ni_pi->cac_weights->weight_sq_vsp) |
3305                 WEIGHT_SQ_VSP0(ni_pi->cac_weights->weight_sq_vsp0));
3306         WREG32_CG(CG_CAC_REGION_5_WEIGHT_0, reg);
3307
3308         reg = RREG32_CG(CG_CAC_REGION_5_WEIGHT_1) & ~(WEIGHT_SQ_GPR_MASK);
3309         reg |= WEIGHT_SQ_GPR(ni_pi->cac_weights->weight_sq_gpr);
3310         WREG32_CG(CG_CAC_REGION_5_WEIGHT_1, reg);
3311
3312         reg = RREG32_CG(CG_CAC_REGION_4_OVERRIDE_4) & ~(OVR_MODE_SPARE_0_MASK |
3313                                                         OVR_VAL_SPARE_0_MASK |
3314                                                         OVR_MODE_SPARE_1_MASK |
3315                                                         OVR_VAL_SPARE_1_MASK);
3316         reg |= (OVR_MODE_SPARE_0(ni_pi->cac_weights->ovr_mode_spare_0) |
3317                 OVR_VAL_SPARE_0(ni_pi->cac_weights->ovr_val_spare_0) |
3318                 OVR_MODE_SPARE_1(ni_pi->cac_weights->ovr_mode_spare_1) |
3319                 OVR_VAL_SPARE_1(ni_pi->cac_weights->ovr_val_spare_1));
3320         WREG32_CG(CG_CAC_REGION_4_OVERRIDE_4, reg);
3321
3322         reg = RREG32(SQ_CAC_THRESHOLD) & ~(VSP_MASK |
3323                                            VSP0_MASK |
3324                                            GPR_MASK);
3325         reg |= (VSP(ni_pi->cac_weights->vsp) |
3326                 VSP0(ni_pi->cac_weights->vsp0) |
3327                 GPR(ni_pi->cac_weights->gpr));
3328         WREG32(SQ_CAC_THRESHOLD, reg);
3329
3330         reg = (MCDW_WR_ENABLE |
3331                MCDX_WR_ENABLE |
3332                MCDY_WR_ENABLE |
3333                MCDZ_WR_ENABLE |
3334                INDEX(0x09D4));
3335         WREG32(MC_CG_CONFIG, reg);
3336
3337         reg = (READ_WEIGHT(ni_pi->cac_weights->mc_read_weight) |
3338                WRITE_WEIGHT(ni_pi->cac_weights->mc_write_weight) |
3339                ALLOW_OVERFLOW);
3340         WREG32(MC_CG_DATAPORT, reg);
3341
3342         return 0;
3343 }
3344
3345 static int ni_enable_smc_cac(struct radeon_device *rdev,
3346                              struct radeon_ps *radeon_new_state,
3347                              bool enable)
3348 {
3349         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3350         int ret = 0;
3351         PPSMC_Result smc_result;
3352
3353         if (ni_pi->enable_cac) {
3354                 if (enable) {
3355                         if (!r600_is_uvd_state(radeon_new_state->class, radeon_new_state->class2)) {
3356                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_CollectCAC_PowerCorreln);
3357
3358                                 if (ni_pi->support_cac_long_term_average) {
3359                                         smc_result = rv770_send_msg_to_smc(rdev, PPSMC_CACLongTermAvgEnable);
3360                                         if (PPSMC_Result_OK != smc_result)
3361                                                 ni_pi->support_cac_long_term_average = false;
3362                                 }
3363
3364                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_EnableCac);
3365                                 if (PPSMC_Result_OK != smc_result)
3366                                         ret = -EINVAL;
3367
3368                                 ni_pi->cac_enabled = (PPSMC_Result_OK == smc_result) ? true : false;
3369                         }
3370                 } else if (ni_pi->cac_enabled) {
3371                         smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_DisableCac);
3372
3373                         ni_pi->cac_enabled = false;
3374
3375                         if (ni_pi->support_cac_long_term_average) {
3376                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_CACLongTermAvgDisable);
3377                                 if (PPSMC_Result_OK != smc_result)
3378                                         ni_pi->support_cac_long_term_average = false;
3379                         }
3380                 }
3381         }
3382
3383         return ret;
3384 }
3385
3386 static int ni_pcie_performance_request(struct radeon_device *rdev,
3387                                        u8 perf_req, bool advertise)
3388 {
3389         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3390
3391 #if defined(CONFIG_ACPI)
3392         if ((perf_req == PCIE_PERF_REQ_PECI_GEN1) ||
3393             (perf_req == PCIE_PERF_REQ_PECI_GEN2)) {
3394                 if (eg_pi->pcie_performance_request_registered == false)
3395                         radeon_acpi_pcie_notify_device_ready(rdev);
3396                 eg_pi->pcie_performance_request_registered = true;
3397                 return radeon_acpi_pcie_performance_request(rdev, perf_req, advertise);
3398         } else if ((perf_req == PCIE_PERF_REQ_REMOVE_REGISTRY) &&
3399                    eg_pi->pcie_performance_request_registered) {
3400                 eg_pi->pcie_performance_request_registered = false;
3401                 return radeon_acpi_pcie_performance_request(rdev, perf_req, advertise);
3402         }
3403 #endif
3404         return 0;
3405 }
3406
3407 static int ni_advertise_gen2_capability(struct radeon_device *rdev)
3408 {
3409         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3410         u32 tmp;
3411
3412         tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3413
3414         if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
3415             (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2))
3416                 pi->pcie_gen2 = true;
3417         else
3418                 pi->pcie_gen2 = false;
3419
3420         if (!pi->pcie_gen2)
3421                 ni_pcie_performance_request(rdev, PCIE_PERF_REQ_PECI_GEN2, true);
3422
3423         return 0;
3424 }
3425
3426 static void ni_enable_bif_dynamic_pcie_gen2(struct radeon_device *rdev,
3427                                             bool enable)
3428 {
3429         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3430         u32 tmp, bif;
3431
3432         tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3433
3434         if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
3435             (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
3436                 if (enable) {
3437                         if (!pi->boot_in_gen2) {
3438                                 bif = RREG32(CG_BIF_REQ_AND_RSP) & ~CG_CLIENT_REQ_MASK;
3439                                 bif |= CG_CLIENT_REQ(0xd);
3440                                 WREG32(CG_BIF_REQ_AND_RSP, bif);
3441                         }
3442                         tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
3443                         tmp |= LC_HW_VOLTAGE_IF_CONTROL(1);
3444                         tmp |= LC_GEN2_EN_STRAP;
3445
3446                         tmp |= LC_CLR_FAILED_SPD_CHANGE_CNT;
3447                         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
3448                         udelay(10);
3449                         tmp &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
3450                         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
3451                 } else {
3452                         if (!pi->boot_in_gen2) {
3453                                 bif = RREG32(CG_BIF_REQ_AND_RSP) & ~CG_CLIENT_REQ_MASK;
3454                                 bif |= CG_CLIENT_REQ(0xd);
3455                                 WREG32(CG_BIF_REQ_AND_RSP, bif);
3456
3457                                 tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
3458                                 tmp &= ~LC_GEN2_EN_STRAP;
3459                         }
3460                         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
3461                 }
3462         }
3463 }
3464
3465 static void ni_enable_dynamic_pcie_gen2(struct radeon_device *rdev,
3466                                         bool enable)
3467 {
3468         ni_enable_bif_dynamic_pcie_gen2(rdev, enable);
3469
3470         if (enable)
3471                 WREG32_P(GENERAL_PWRMGT, ENABLE_GEN2PCIE, ~ENABLE_GEN2PCIE);
3472         else
3473                 WREG32_P(GENERAL_PWRMGT, 0, ~ENABLE_GEN2PCIE);
3474 }
3475
3476 void ni_dpm_setup_asic(struct radeon_device *rdev)
3477 {
3478         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3479
3480         ni_read_clock_registers(rdev);
3481         btc_read_arb_registers(rdev);
3482         rv770_get_memory_type(rdev);
3483         if (eg_pi->pcie_performance_request)
3484                 ni_advertise_gen2_capability(rdev);
3485         rv770_get_pcie_gen2_status(rdev);
3486         rv770_enable_acpi_pm(rdev);
3487 }
3488
3489 static void ni_update_current_ps(struct radeon_device *rdev,
3490                                  struct radeon_ps *rps)
3491 {
3492         struct ni_ps *new_ps = ni_get_ps(rps);
3493         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3494         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3495
3496         eg_pi->current_rps = *rps;
3497         ni_pi->current_ps = *new_ps;
3498         eg_pi->current_rps.ps_priv = &ni_pi->current_ps;
3499 }
3500
3501 static void ni_update_requested_ps(struct radeon_device *rdev,
3502                                    struct radeon_ps *rps)
3503 {
3504         struct ni_ps *new_ps = ni_get_ps(rps);
3505         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3506         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3507
3508         eg_pi->requested_rps = *rps;
3509         ni_pi->requested_ps = *new_ps;
3510         eg_pi->requested_rps.ps_priv = &ni_pi->requested_ps;
3511 }
3512
3513 int ni_dpm_enable(struct radeon_device *rdev)
3514 {
3515         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3516         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3517         struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
3518
3519         if (pi->gfx_clock_gating)
3520                 ni_cg_clockgating_default(rdev);
3521         if (btc_dpm_enabled(rdev))
3522                 return -EINVAL;
3523         if (pi->mg_clock_gating)
3524                 ni_mg_clockgating_default(rdev);
3525         if (eg_pi->ls_clock_gating)
3526                 ni_ls_clockgating_default(rdev);
3527         if (pi->voltage_control) {
3528                 rv770_enable_voltage_control(rdev, true);
3529                 cypress_construct_voltage_tables(rdev);
3530         }
3531         if (eg_pi->dynamic_ac_timing)
3532                 ni_initialize_mc_reg_table(rdev);
3533         if (pi->dynamic_ss)
3534                 cypress_enable_spread_spectrum(rdev, true);
3535         if (pi->thermal_protection)
3536                 rv770_enable_thermal_protection(rdev, true);
3537         rv770_setup_bsp(rdev);
3538         rv770_program_git(rdev);
3539         rv770_program_tp(rdev);
3540         rv770_program_tpp(rdev);
3541         rv770_program_sstp(rdev);
3542         cypress_enable_display_gap(rdev);
3543         rv770_program_vc(rdev);
3544         if (pi->dynamic_pcie_gen2)
3545                 ni_enable_dynamic_pcie_gen2(rdev, true);
3546         if (rv770_upload_firmware(rdev))
3547                 return -EINVAL;
3548         ni_process_firmware_header(rdev);
3549         ni_initial_switch_from_arb_f0_to_f1(rdev);
3550         ni_init_smc_table(rdev);
3551         ni_init_smc_spll_table(rdev);
3552         ni_init_arb_table_index(rdev);
3553         if (eg_pi->dynamic_ac_timing)
3554                 ni_populate_mc_reg_table(rdev, boot_ps);
3555         ni_initialize_smc_cac_tables(rdev);
3556         ni_initialize_hardware_cac_manager(rdev);
3557         ni_populate_smc_tdp_limits(rdev, boot_ps);
3558         ni_program_response_times(rdev);
3559         r7xx_start_smc(rdev);
3560         cypress_notify_smc_display_change(rdev, false);
3561         cypress_enable_sclk_control(rdev, true);
3562         if (eg_pi->memory_transition)
3563                 cypress_enable_mclk_control(rdev, true);
3564         cypress_start_dpm(rdev);
3565         if (pi->gfx_clock_gating)
3566                 ni_gfx_clockgating_enable(rdev, true);
3567         if (pi->mg_clock_gating)
3568                 ni_mg_clockgating_enable(rdev, true);
3569         if (eg_pi->ls_clock_gating)
3570                 ni_ls_clockgating_enable(rdev, true);
3571
3572         if (rdev->irq.installed &&
3573             r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
3574                 PPSMC_Result result;
3575
3576                 rv770_set_thermal_temperature_range(rdev, R600_TEMP_RANGE_MIN, 0xff * 1000);
3577                 rdev->irq.dpm_thermal = true;
3578                 radeon_irq_set(rdev);
3579                 result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_EnableThermalInterrupt);
3580
3581                 if (result != PPSMC_Result_OK)
3582                         DRM_DEBUG_KMS("Could not enable thermal interrupts.\n");
3583         }
3584
3585         rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, true);
3586
3587         ni_update_current_ps(rdev, boot_ps);
3588
3589         return 0;
3590 }
3591
3592 void ni_dpm_disable(struct radeon_device *rdev)
3593 {
3594         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3595         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3596         struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
3597
3598         if (!btc_dpm_enabled(rdev))
3599                 return;
3600         rv770_clear_vc(rdev);
3601         if (pi->thermal_protection)
3602                 rv770_enable_thermal_protection(rdev, false);
3603         ni_enable_power_containment(rdev, boot_ps, false);
3604         ni_enable_smc_cac(rdev, boot_ps, false);
3605         cypress_enable_spread_spectrum(rdev, false);
3606         rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, false);
3607         if (pi->dynamic_pcie_gen2)
3608                 ni_enable_dynamic_pcie_gen2(rdev, false);
3609
3610         if (rdev->irq.installed &&
3611             r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
3612                 rdev->irq.dpm_thermal = false;
3613                 radeon_irq_set(rdev);
3614         }
3615
3616         if (pi->gfx_clock_gating)
3617                 ni_gfx_clockgating_enable(rdev, false);
3618         if (pi->mg_clock_gating)
3619                 ni_mg_clockgating_enable(rdev, false);
3620         if (eg_pi->ls_clock_gating)
3621                 ni_ls_clockgating_enable(rdev, false);
3622         ni_stop_dpm(rdev);
3623         btc_reset_to_default(rdev);
3624         ni_stop_smc(rdev);
3625         ni_force_switch_to_arb_f0(rdev);
3626
3627         ni_update_current_ps(rdev, boot_ps);
3628 }
3629
3630 int ni_power_control_set_level(struct radeon_device *rdev)
3631 {
3632         struct radeon_ps *new_ps = rdev->pm.dpm.requested_ps;
3633
3634         ni_restrict_performance_levels_before_switch(rdev);
3635         rv770_halt_smc(rdev);
3636         ni_populate_smc_tdp_limits(rdev, new_ps);
3637         rv770_resume_smc(rdev);
3638         rv770_set_sw_state(rdev);
3639
3640         return 0;
3641 }
3642
3643 int ni_dpm_pre_set_power_state(struct radeon_device *rdev)
3644 {
3645         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3646         struct radeon_ps requested_ps = *rdev->pm.dpm.requested_ps;
3647         struct radeon_ps *new_ps = &requested_ps;
3648
3649         ni_update_requested_ps(rdev, new_ps);
3650
3651         ni_apply_state_adjust_rules(rdev, &eg_pi->requested_rps);
3652
3653         return 0;
3654 }
3655
3656 int ni_dpm_set_power_state(struct radeon_device *rdev)
3657 {
3658         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3659         struct radeon_ps *new_ps = &eg_pi->requested_rps;
3660         struct radeon_ps *old_ps = &eg_pi->current_rps;
3661         int ret;
3662
3663         ni_restrict_performance_levels_before_switch(rdev);
3664         rv770_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps);
3665         ni_enable_power_containment(rdev, new_ps, false);
3666         ni_enable_smc_cac(rdev, new_ps, false);
3667         rv770_halt_smc(rdev);
3668         if (eg_pi->smu_uvd_hs)
3669                 btc_notify_uvd_to_smc(rdev, new_ps);
3670         ni_upload_sw_state(rdev, new_ps);
3671         if (eg_pi->dynamic_ac_timing)
3672                 ni_upload_mc_reg_table(rdev, new_ps);
3673         ret = ni_program_memory_timing_parameters(rdev, new_ps);
3674         if (ret)
3675                 return ret;
3676         ni_populate_smc_tdp_limits(rdev, new_ps);
3677         rv770_resume_smc(rdev);
3678         rv770_set_sw_state(rdev);
3679         rv770_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps);
3680         ni_enable_smc_cac(rdev, new_ps, true);
3681         ni_enable_power_containment(rdev, new_ps, true);
3682
3683 #if 0
3684         /* XXX */
3685         ni_unrestrict_performance_levels_after_switch(rdev);
3686 #endif
3687
3688         return 0;
3689 }
3690
3691 void ni_dpm_post_set_power_state(struct radeon_device *rdev)
3692 {
3693         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3694         struct radeon_ps *new_ps = &eg_pi->requested_rps;
3695
3696         ni_update_current_ps(rdev, new_ps);
3697 }
3698
3699 void ni_dpm_reset_asic(struct radeon_device *rdev)
3700 {
3701         ni_restrict_performance_levels_before_switch(rdev);
3702         rv770_set_boot_state(rdev);
3703 }
3704
3705 union power_info {
3706         struct _ATOM_POWERPLAY_INFO info;
3707         struct _ATOM_POWERPLAY_INFO_V2 info_2;
3708         struct _ATOM_POWERPLAY_INFO_V3 info_3;
3709         struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
3710         struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
3711         struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
3712 };
3713
3714 union pplib_clock_info {
3715         struct _ATOM_PPLIB_R600_CLOCK_INFO r600;
3716         struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780;
3717         struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
3718         struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
3719 };
3720
3721 union pplib_power_state {
3722         struct _ATOM_PPLIB_STATE v1;
3723         struct _ATOM_PPLIB_STATE_V2 v2;
3724 };
3725
3726 static void ni_parse_pplib_non_clock_info(struct radeon_device *rdev,
3727                                           struct radeon_ps *rps,
3728                                           struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info,
3729                                           u8 table_rev)
3730 {
3731         rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings);
3732         rps->class = le16_to_cpu(non_clock_info->usClassification);
3733         rps->class2 = le16_to_cpu(non_clock_info->usClassification2);
3734
3735         if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) {
3736                 rps->vclk = le32_to_cpu(non_clock_info->ulVCLK);
3737                 rps->dclk = le32_to_cpu(non_clock_info->ulDCLK);
3738         } else if (r600_is_uvd_state(rps->class, rps->class2)) {
3739                 rps->vclk = RV770_DEFAULT_VCLK_FREQ;
3740                 rps->dclk = RV770_DEFAULT_DCLK_FREQ;
3741         } else {
3742                 rps->vclk = 0;
3743                 rps->dclk = 0;
3744         }
3745
3746         if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT)
3747                 rdev->pm.dpm.boot_ps = rps;
3748         if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
3749                 rdev->pm.dpm.uvd_ps = rps;
3750 }
3751
3752 static void ni_parse_pplib_clock_info(struct radeon_device *rdev,
3753                                       struct radeon_ps *rps, int index,
3754                                       union pplib_clock_info *clock_info)
3755 {
3756         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3757         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3758         struct ni_ps *ps = ni_get_ps(rps);
3759         u16 vddc;
3760         struct rv7xx_pl *pl = &ps->performance_levels[index];
3761
3762         ps->performance_level_count = index + 1;
3763
3764         pl->sclk = le16_to_cpu(clock_info->evergreen.usEngineClockLow);
3765         pl->sclk |= clock_info->evergreen.ucEngineClockHigh << 16;
3766         pl->mclk = le16_to_cpu(clock_info->evergreen.usMemoryClockLow);
3767         pl->mclk |= clock_info->evergreen.ucMemoryClockHigh << 16;
3768
3769         pl->vddc = le16_to_cpu(clock_info->evergreen.usVDDC);
3770         pl->vddci = le16_to_cpu(clock_info->evergreen.usVDDCI);
3771         pl->flags = le32_to_cpu(clock_info->evergreen.ulFlags);
3772
3773         /* patch up vddc if necessary */
3774         if (pl->vddc == 0xff01) {
3775                 if (radeon_atom_get_max_vddc(rdev, 0, 0, &vddc) == 0)
3776                         pl->vddc = vddc;
3777         }
3778
3779         if (rps->class & ATOM_PPLIB_CLASSIFICATION_ACPI) {
3780                 pi->acpi_vddc = pl->vddc;
3781                 eg_pi->acpi_vddci = pl->vddci;
3782                 if (ps->performance_levels[0].flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2)
3783                         pi->acpi_pcie_gen2 = true;
3784                 else
3785                         pi->acpi_pcie_gen2 = false;
3786         }
3787
3788         if (rps->class2 & ATOM_PPLIB_CLASSIFICATION2_ULV) {
3789                 eg_pi->ulv.supported = true;
3790                 eg_pi->ulv.pl = pl;
3791         }
3792
3793         if (pi->min_vddc_in_table > pl->vddc)
3794                 pi->min_vddc_in_table = pl->vddc;
3795
3796         if (pi->max_vddc_in_table < pl->vddc)
3797                 pi->max_vddc_in_table = pl->vddc;
3798
3799         /* patch up boot state */
3800         if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) {
3801                 u16 vddc, vddci;
3802                 radeon_atombios_get_default_voltages(rdev, &vddc, &vddci);
3803                 pl->mclk = rdev->clock.default_mclk;
3804                 pl->sclk = rdev->clock.default_sclk;
3805                 pl->vddc = vddc;
3806                 pl->vddci = vddci;
3807         }
3808
3809         if ((rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) ==
3810             ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE) {
3811                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk = pl->sclk;
3812                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.mclk = pl->mclk;
3813                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddc = pl->vddc;
3814                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddci = pl->vddci;
3815         }
3816 }
3817
3818 static int ni_parse_power_table(struct radeon_device *rdev)
3819 {
3820         struct radeon_mode_info *mode_info = &rdev->mode_info;
3821         struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
3822         union pplib_power_state *power_state;
3823         int i, j;
3824         union pplib_clock_info *clock_info;
3825         union power_info *power_info;
3826         int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
3827         u16 data_offset;
3828         u8 frev, crev;
3829         struct ni_ps *ps;
3830
3831         if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
3832                                    &frev, &crev, &data_offset))
3833                 return -EINVAL;
3834         power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
3835
3836         rdev->pm.dpm.ps = kzalloc(sizeof(struct radeon_ps) *
3837                                   power_info->pplib.ucNumStates, GFP_KERNEL);
3838         if (!rdev->pm.dpm.ps)
3839                 return -ENOMEM;
3840         rdev->pm.dpm.platform_caps = le32_to_cpu(power_info->pplib.ulPlatformCaps);
3841         rdev->pm.dpm.backbias_response_time = le16_to_cpu(power_info->pplib.usBackbiasTime);
3842         rdev->pm.dpm.voltage_response_time = le16_to_cpu(power_info->pplib.usVoltageTime);
3843
3844         for (i = 0; i < power_info->pplib.ucNumStates; i++) {
3845                 power_state = (union pplib_power_state *)
3846                         (mode_info->atom_context->bios + data_offset +
3847                          le16_to_cpu(power_info->pplib.usStateArrayOffset) +
3848                          i * power_info->pplib.ucStateEntrySize);
3849                 non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
3850                         (mode_info->atom_context->bios + data_offset +
3851                          le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset) +
3852                          (power_state->v1.ucNonClockStateIndex *
3853                           power_info->pplib.ucNonClockSize));
3854                 if (power_info->pplib.ucStateEntrySize - 1) {
3855                         ps = kzalloc(sizeof(struct ni_ps), GFP_KERNEL);
3856                         if (ps == NULL) {
3857                                 kfree(rdev->pm.dpm.ps);
3858                                 return -ENOMEM;
3859                         }
3860                         rdev->pm.dpm.ps[i].ps_priv = ps;
3861                         ni_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i],
3862                                                          non_clock_info,
3863                                                          power_info->pplib.ucNonClockSize);
3864                         for (j = 0; j < (power_info->pplib.ucStateEntrySize - 1); j++) {
3865                                 clock_info = (union pplib_clock_info *)
3866                                         (mode_info->atom_context->bios + data_offset +
3867                                          le16_to_cpu(power_info->pplib.usClockInfoArrayOffset) +
3868                                          (power_state->v1.ucClockStateIndices[j] *
3869                                           power_info->pplib.ucClockInfoSize));
3870                                 ni_parse_pplib_clock_info(rdev,
3871                                                           &rdev->pm.dpm.ps[i], j,
3872                                                           clock_info);
3873                         }
3874                 }
3875         }
3876         rdev->pm.dpm.num_ps = power_info->pplib.ucNumStates;
3877         return 0;
3878 }
3879
3880 int ni_dpm_init(struct radeon_device *rdev)
3881 {
3882         struct rv7xx_power_info *pi;
3883         struct evergreen_power_info *eg_pi;
3884         struct ni_power_info *ni_pi;
3885         int index = GetIndexIntoMasterTable(DATA, ASIC_InternalSS_Info);
3886         u16 data_offset, size;
3887         u8 frev, crev;
3888         struct atom_clock_dividers dividers;
3889         int ret;
3890
3891         ni_pi = kzalloc(sizeof(struct ni_power_info), GFP_KERNEL);
3892         if (ni_pi == NULL)
3893                 return -ENOMEM;
3894         rdev->pm.dpm.priv = ni_pi;
3895         eg_pi = &ni_pi->eg;
3896         pi = &eg_pi->rv7xx;
3897
3898         rv770_get_max_vddc(rdev);
3899
3900         eg_pi->ulv.supported = false;
3901         pi->acpi_vddc = 0;
3902         eg_pi->acpi_vddci = 0;
3903         pi->min_vddc_in_table = 0;
3904         pi->max_vddc_in_table = 0;
3905
3906         ret = ni_parse_power_table(rdev);
3907         if (ret)
3908                 return ret;
3909         ret = r600_parse_extended_power_table(rdev);
3910         if (ret)
3911                 return ret;
3912
3913         ni_patch_dependency_tables_based_on_leakage(rdev);
3914
3915         if (rdev->pm.dpm.voltage_response_time == 0)
3916                 rdev->pm.dpm.voltage_response_time = R600_VOLTAGERESPONSETIME_DFLT;
3917         if (rdev->pm.dpm.backbias_response_time == 0)
3918                 rdev->pm.dpm.backbias_response_time = R600_BACKBIASRESPONSETIME_DFLT;
3919
3920         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
3921                                              0, false, &dividers);
3922         if (ret)
3923                 pi->ref_div = dividers.ref_div + 1;
3924         else
3925                 pi->ref_div = R600_REFERENCEDIVIDER_DFLT;
3926
3927         pi->rlp = RV770_RLP_DFLT;
3928         pi->rmp = RV770_RMP_DFLT;
3929         pi->lhp = RV770_LHP_DFLT;
3930         pi->lmp = RV770_LMP_DFLT;
3931
3932         eg_pi->ats[0].rlp = RV770_RLP_DFLT;
3933         eg_pi->ats[0].rmp = RV770_RMP_DFLT;
3934         eg_pi->ats[0].lhp = RV770_LHP_DFLT;
3935         eg_pi->ats[0].lmp = RV770_LMP_DFLT;
3936
3937         eg_pi->ats[1].rlp = BTC_RLP_UVD_DFLT;
3938         eg_pi->ats[1].rmp = BTC_RMP_UVD_DFLT;
3939         eg_pi->ats[1].lhp = BTC_LHP_UVD_DFLT;
3940         eg_pi->ats[1].lmp = BTC_LMP_UVD_DFLT;
3941
3942         eg_pi->smu_uvd_hs = true;
3943
3944         if (rdev->pdev->device == 0x6707) {
3945                 pi->mclk_strobe_mode_threshold = 55000;
3946                 pi->mclk_edc_enable_threshold = 55000;
3947                 eg_pi->mclk_edc_wr_enable_threshold = 55000;
3948         } else {
3949                 pi->mclk_strobe_mode_threshold = 40000;
3950                 pi->mclk_edc_enable_threshold = 40000;
3951                 eg_pi->mclk_edc_wr_enable_threshold = 40000;
3952         }
3953         ni_pi->mclk_rtt_mode_threshold = eg_pi->mclk_edc_wr_enable_threshold;
3954
3955         pi->voltage_control =
3956                 radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, 0);
3957
3958         pi->mvdd_control =
3959                 radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_MVDDC, 0);
3960
3961         eg_pi->vddci_control =
3962                 radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDCI, 0);
3963
3964         if (atom_parse_data_header(rdev->mode_info.atom_context, index, &size,
3965                                    &frev, &crev, &data_offset)) {
3966                 pi->sclk_ss = true;
3967                 pi->mclk_ss = true;
3968                 pi->dynamic_ss = true;
3969         } else {
3970                 pi->sclk_ss = false;
3971                 pi->mclk_ss = false;
3972                 pi->dynamic_ss = true;
3973         }
3974
3975         pi->asi = RV770_ASI_DFLT;
3976         pi->pasi = CYPRESS_HASI_DFLT;
3977         pi->vrc = CYPRESS_VRC_DFLT;
3978
3979         pi->power_gating = false;
3980
3981         pi->gfx_clock_gating = true;
3982
3983         pi->mg_clock_gating = true;
3984         pi->mgcgtssm = true;
3985         eg_pi->ls_clock_gating = false;
3986         eg_pi->sclk_deep_sleep = false;
3987
3988         pi->dynamic_pcie_gen2 = true;
3989
3990         if (pi->gfx_clock_gating &&
3991             (rdev->pm.int_thermal_type != THERMAL_TYPE_NONE))
3992                 pi->thermal_protection = true;
3993         else
3994                 pi->thermal_protection = false;
3995
3996         pi->display_gap = true;
3997
3998         pi->dcodt = true;
3999
4000         pi->ulps = true;
4001
4002         eg_pi->dynamic_ac_timing = true;
4003         eg_pi->abm = true;
4004         eg_pi->mcls = true;
4005         eg_pi->light_sleep = true;
4006         eg_pi->memory_transition = true;
4007 #if defined(CONFIG_ACPI)
4008         eg_pi->pcie_performance_request =
4009                 radeon_acpi_is_pcie_performance_request_supported(rdev);
4010 #else
4011         eg_pi->pcie_performance_request = false;
4012 #endif
4013
4014         eg_pi->dll_default_on = false;
4015
4016         eg_pi->sclk_deep_sleep = false;
4017
4018         pi->mclk_stutter_mode_threshold = 0;
4019
4020         pi->sram_end = SMC_RAM_END;
4021
4022         rdev->pm.dpm.dyn_state.mclk_sclk_ratio = 3;
4023         rdev->pm.dpm.dyn_state.vddc_vddci_delta = 200;
4024         rdev->pm.dpm.dyn_state.min_vddc_for_pcie_gen2 = 900;
4025         rdev->pm.dpm.dyn_state.valid_sclk_values.count = ARRAY_SIZE(btc_valid_sclk);
4026         rdev->pm.dpm.dyn_state.valid_sclk_values.values = btc_valid_sclk;
4027         rdev->pm.dpm.dyn_state.valid_mclk_values.count = 0;
4028         rdev->pm.dpm.dyn_state.valid_mclk_values.values = NULL;
4029         rdev->pm.dpm.dyn_state.sclk_mclk_delta = 12500;
4030
4031         ni_pi->cac_data.leakage_coefficients.at = 516;
4032         ni_pi->cac_data.leakage_coefficients.bt = 18;
4033         ni_pi->cac_data.leakage_coefficients.av = 51;
4034         ni_pi->cac_data.leakage_coefficients.bv = 2957;
4035
4036         switch (rdev->pdev->device) {
4037         case 0x6700:
4038         case 0x6701:
4039         case 0x6702:
4040         case 0x6703:
4041         case 0x6718:
4042                 ni_pi->cac_weights = &cac_weights_cayman_xt;
4043                 break;
4044         case 0x6705:
4045         case 0x6719:
4046         case 0x671D:
4047         case 0x671C:
4048         default:
4049                 ni_pi->cac_weights = &cac_weights_cayman_pro;
4050                 break;
4051         case 0x6704:
4052         case 0x6706:
4053         case 0x6707:
4054         case 0x6708:
4055         case 0x6709:
4056                 ni_pi->cac_weights = &cac_weights_cayman_le;
4057                 break;
4058         }
4059
4060         if (ni_pi->cac_weights->enable_power_containment_by_default) {
4061                 ni_pi->enable_power_containment = true;
4062                 ni_pi->enable_cac = true;
4063                 ni_pi->enable_sq_ramping = true;
4064         } else {
4065                 ni_pi->enable_power_containment = false;
4066                 ni_pi->enable_cac = false;
4067                 ni_pi->enable_sq_ramping = false;
4068         }
4069
4070         ni_pi->driver_calculate_cac_leakage = false;
4071         ni_pi->cac_configuration_required = true;
4072
4073         if (ni_pi->cac_configuration_required) {
4074                 ni_pi->support_cac_long_term_average = true;
4075                 ni_pi->lta_window_size = ni_pi->cac_weights->l2_lta_window_size;
4076                 ni_pi->lts_truncate = ni_pi->cac_weights->lts_truncate;
4077         } else {
4078                 ni_pi->support_cac_long_term_average = false;
4079                 ni_pi->lta_window_size = 0;
4080                 ni_pi->lts_truncate = 0;
4081         }
4082
4083         ni_pi->use_power_boost_limit = true;
4084
4085         return 0;
4086 }
4087
4088 void ni_dpm_fini(struct radeon_device *rdev)
4089 {
4090         int i;
4091
4092         for (i = 0; i < rdev->pm.dpm.num_ps; i++) {
4093                 kfree(rdev->pm.dpm.ps[i].ps_priv);
4094         }
4095         kfree(rdev->pm.dpm.ps);
4096         kfree(rdev->pm.dpm.priv);
4097         r600_free_extended_power_table(rdev);
4098 }
4099
4100 void ni_dpm_print_power_state(struct radeon_device *rdev,
4101                               struct radeon_ps *rps)
4102 {
4103         struct ni_ps *ps = ni_get_ps(rps);
4104         struct rv7xx_pl *pl;
4105         int i;
4106
4107         r600_dpm_print_class_info(rps->class, rps->class2);
4108         r600_dpm_print_cap_info(rps->caps);
4109         printk("\tuvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
4110         for (i = 0; i < ps->performance_level_count; i++) {
4111                 pl = &ps->performance_levels[i];
4112                 printk("\t\tpower level 0    sclk: %u mclk: %u vddc: %u vddci: %u\n",
4113                        pl->sclk, pl->mclk, pl->vddc, pl->vddci);
4114         }
4115         r600_dpm_print_ps_status(rdev, rps);
4116 }
4117
4118 u32 ni_dpm_get_sclk(struct radeon_device *rdev, bool low)
4119 {
4120         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4121         struct ni_ps *requested_state = ni_get_ps(&eg_pi->requested_rps);
4122
4123         if (low)
4124                 return requested_state->performance_levels[0].sclk;
4125         else
4126                 return requested_state->performance_levels[requested_state->performance_level_count - 1].sclk;
4127 }
4128
4129 u32 ni_dpm_get_mclk(struct radeon_device *rdev, bool low)
4130 {
4131         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4132         struct ni_ps *requested_state = ni_get_ps(&eg_pi->requested_rps);
4133
4134         if (low)
4135                 return requested_state->performance_levels[0].mclk;
4136         else
4137                 return requested_state->performance_levels[requested_state->performance_level_count - 1].mclk;
4138 }
4139