drivers/gpu/drm/radeon/ni_dpm.c

   1 /*
   2  * Copyright 2012 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include "drmP.h"
  25 #include "radeon.h"
  26 #include "nid.h"
  27 #include "r600_dpm.h"
  28 #include "ni_dpm.h"
  29 #include "atom.h"
  30 #include <linux/math64.h>
  31 #include <linux/seq_file.h>
  32
  33 #define MC_CG_ARB_FREQ_F0           0x0a
  34 #define MC_CG_ARB_FREQ_F1           0x0b
  35 #define MC_CG_ARB_FREQ_F2           0x0c
  36 #define MC_CG_ARB_FREQ_F3           0x0d
  37
  38 #define SMC_RAM_END 0xC000
  39
  40 static const struct ni_cac_weights cac_weights_cayman_xt =
  41 {
  42         0x15,
  43         0x2,
  44         0x19,
  45         0x2,
  46         0x8,
  47         0x14,
  48         0x2,
  49         0x16,
  50         0xE,
  51         0x17,
  52         0x13,
  53         0x2B,
  54         0x10,
  55         0x7,
  56         0x5,
  57         0x5,
  58         0x5,
  59         0x2,
  60         0x3,
  61         0x9,
  62         0x10,
  63         0x10,
  64         0x2B,
  65         0xA,
  66         0x9,
  67         0x4,
  68         0xD,
  69         0xD,
  70         0x3E,
  71         0x18,
  72         0x14,
  73         0,
  74         0x3,
  75         0x3,
  76         0x5,
  77         0,
  78         0x2,
  79         0,
  80         0,
  81         0,
  82         0,
  83         0,
  84         0,
  85         0,
  86         0,
  87         0,
  88         0x1CC,
  89         0,
  90         0x164,
  91         1,
  92         1,
  93         1,
  94         1,
  95         12,
  96         12,
  97         12,
  98         0x12,
  99         0x1F,
 100         132,
 101         5,
 102         7,
 103         0,
 104         { 0, 0, 0, 0, 0, 0, 0, 0 },
 105         { 0, 0, 0, 0 },
 106         true
 107 };
 108
 109 static const struct ni_cac_weights cac_weights_cayman_pro =
 110 {
 111         0x16,
 112         0x4,
 113         0x10,
 114         0x2,
 115         0xA,
 116         0x16,
 117         0x2,
 118         0x18,
 119         0x10,
 120         0x1A,
 121         0x16,
 122         0x2D,
 123         0x12,
 124         0xA,
 125         0x6,
 126         0x6,
 127         0x6,
 128         0x2,
 129         0x4,
 130         0xB,
 131         0x11,
 132         0x11,
 133         0x2D,
 134         0xC,
 135         0xC,
 136         0x7,
 137         0x10,
 138         0x10,
 139         0x3F,
 140         0x1A,
 141         0x16,
 142         0,
 143         0x7,
 144         0x4,
 145         0x6,
 146         1,
 147         0x2,
 148         0x1,
 149         0,
 150         0,
 151         0,
 152         0,
 153         0,
 154         0,
 155         0x30,
 156         0,
 157         0x1CF,
 158         0,
 159         0x166,
 160         1,
 161         1,
 162         1,
 163         1,
 164         12,
 165         12,
 166         12,
 167         0x15,
 168         0x1F,
 169         132,
 170         6,
 171         6,
 172         0,
 173         { 0, 0, 0, 0, 0, 0, 0, 0 },
 174         { 0, 0, 0, 0 },
 175         true
 176 };
 177
 178 static const struct ni_cac_weights cac_weights_cayman_le =
 179 {
 180         0x7,
 181         0xE,
 182         0x1,
 183         0xA,
 184         0x1,
 185         0x3F,
 186         0x2,
 187         0x18,
 188         0x10,
 189         0x1A,
 190         0x1,
 191         0x3F,
 192         0x1,
 193         0xE,
 194         0x6,
 195         0x6,
 196         0x6,
 197         0x2,
 198         0x4,
 199         0x9,
 200         0x1A,
 201         0x1A,
 202         0x2C,
 203         0xA,
 204         0x11,
 205         0x8,
 206         0x19,
 207         0x19,
 208         0x1,
 209         0x1,
 210         0x1A,
 211         0,
 212         0x8,
 213         0x5,
 214         0x8,
 215         0x1,
 216         0x3,
 217         0x1,
 218         0,
 219         0,
 220         0,
 221         0,
 222         0,
 223         0,
 224         0x38,
 225         0x38,
 226         0x239,
 227         0x3,
 228         0x18A,
 229         1,
 230         1,
 231         1,
 232         1,
 233         12,
 234         12,
 235         12,
 236         0x15,
 237         0x22,
 238         132,
 239         6,
 240         6,
 241         0,
 242         { 0, 0, 0, 0, 0, 0, 0, 0 },
 243         { 0, 0, 0, 0 },
 244         true
 245 };
 246
 247 #define NISLANDS_MGCG_SEQUENCE  300
 248
 249 static const u32 cayman_cgcg_cgls_default[] =
 250 {
 251         0x000008f8, 0x00000010, 0xffffffff,
 252         0x000008fc, 0x00000000, 0xffffffff,
 253         0x000008f8, 0x00000011, 0xffffffff,
 254         0x000008fc, 0x00000000, 0xffffffff,
 255         0x000008f8, 0x00000012, 0xffffffff,
 256         0x000008fc, 0x00000000, 0xffffffff,
 257         0x000008f8, 0x00000013, 0xffffffff,
 258         0x000008fc, 0x00000000, 0xffffffff,
 259         0x000008f8, 0x00000014, 0xffffffff,
 260         0x000008fc, 0x00000000, 0xffffffff,
 261         0x000008f8, 0x00000015, 0xffffffff,
 262         0x000008fc, 0x00000000, 0xffffffff,
 263         0x000008f8, 0x00000016, 0xffffffff,
 264         0x000008fc, 0x00000000, 0xffffffff,
 265         0x000008f8, 0x00000017, 0xffffffff,
 266         0x000008fc, 0x00000000, 0xffffffff,
 267         0x000008f8, 0x00000018, 0xffffffff,
 268         0x000008fc, 0x00000000, 0xffffffff,
 269         0x000008f8, 0x00000019, 0xffffffff,
 270         0x000008fc, 0x00000000, 0xffffffff,
 271         0x000008f8, 0x0000001a, 0xffffffff,
 272         0x000008fc, 0x00000000, 0xffffffff,
 273         0x000008f8, 0x0000001b, 0xffffffff,
 274         0x000008fc, 0x00000000, 0xffffffff,
 275         0x000008f8, 0x00000020, 0xffffffff,
 276         0x000008fc, 0x00000000, 0xffffffff,
 277         0x000008f8, 0x00000021, 0xffffffff,
 278         0x000008fc, 0x00000000, 0xffffffff,
 279         0x000008f8, 0x00000022, 0xffffffff,
 280         0x000008fc, 0x00000000, 0xffffffff,
 281         0x000008f8, 0x00000023, 0xffffffff,
 282         0x000008fc, 0x00000000, 0xffffffff,
 283         0x000008f8, 0x00000024, 0xffffffff,
 284         0x000008fc, 0x00000000, 0xffffffff,
 285         0x000008f8, 0x00000025, 0xffffffff,
 286         0x000008fc, 0x00000000, 0xffffffff,
 287         0x000008f8, 0x00000026, 0xffffffff,
 288         0x000008fc, 0x00000000, 0xffffffff,
 289         0x000008f8, 0x00000027, 0xffffffff,
 290         0x000008fc, 0x00000000, 0xffffffff,
 291         0x000008f8, 0x00000028, 0xffffffff,
 292         0x000008fc, 0x00000000, 0xffffffff,
 293         0x000008f8, 0x00000029, 0xffffffff,
 294         0x000008fc, 0x00000000, 0xffffffff,
 295         0x000008f8, 0x0000002a, 0xffffffff,
 296         0x000008fc, 0x00000000, 0xffffffff,
 297         0x000008f8, 0x0000002b, 0xffffffff,
 298         0x000008fc, 0x00000000, 0xffffffff
 299 };
 300 #define CAYMAN_CGCG_CGLS_DEFAULT_LENGTH sizeof(cayman_cgcg_cgls_default) / (3 * sizeof(u32))
 301
 302 static const u32 cayman_cgcg_cgls_disable[] =
 303 {
 304         0x000008f8, 0x00000010, 0xffffffff,
 305         0x000008fc, 0xffffffff, 0xffffffff,
 306         0x000008f8, 0x00000011, 0xffffffff,
 307         0x000008fc, 0xffffffff, 0xffffffff,
 308         0x000008f8, 0x00000012, 0xffffffff,
 309         0x000008fc, 0xffffffff, 0xffffffff,
 310         0x000008f8, 0x00000013, 0xffffffff,
 311         0x000008fc, 0xffffffff, 0xffffffff,
 312         0x000008f8, 0x00000014, 0xffffffff,
 313         0x000008fc, 0xffffffff, 0xffffffff,
 314         0x000008f8, 0x00000015, 0xffffffff,
 315         0x000008fc, 0xffffffff, 0xffffffff,
 316         0x000008f8, 0x00000016, 0xffffffff,
 317         0x000008fc, 0xffffffff, 0xffffffff,
 318         0x000008f8, 0x00000017, 0xffffffff,
 319         0x000008fc, 0xffffffff, 0xffffffff,
 320         0x000008f8, 0x00000018, 0xffffffff,
 321         0x000008fc, 0xffffffff, 0xffffffff,
 322         0x000008f8, 0x00000019, 0xffffffff,
 323         0x000008fc, 0xffffffff, 0xffffffff,
 324         0x000008f8, 0x0000001a, 0xffffffff,
 325         0x000008fc, 0xffffffff, 0xffffffff,
 326         0x000008f8, 0x0000001b, 0xffffffff,
 327         0x000008fc, 0xffffffff, 0xffffffff,
 328         0x000008f8, 0x00000020, 0xffffffff,
 329         0x000008fc, 0x00000000, 0xffffffff,
 330         0x000008f8, 0x00000021, 0xffffffff,
 331         0x000008fc, 0x00000000, 0xffffffff,
 332         0x000008f8, 0x00000022, 0xffffffff,
 333         0x000008fc, 0x00000000, 0xffffffff,
 334         0x000008f8, 0x00000023, 0xffffffff,
 335         0x000008fc, 0x00000000, 0xffffffff,
 336         0x000008f8, 0x00000024, 0xffffffff,
 337         0x000008fc, 0x00000000, 0xffffffff,
 338         0x000008f8, 0x00000025, 0xffffffff,
 339         0x000008fc, 0x00000000, 0xffffffff,
 340         0x000008f8, 0x00000026, 0xffffffff,
 341         0x000008fc, 0x00000000, 0xffffffff,
 342         0x000008f8, 0x00000027, 0xffffffff,
 343         0x000008fc, 0x00000000, 0xffffffff,
 344         0x000008f8, 0x00000028, 0xffffffff,
 345         0x000008fc, 0x00000000, 0xffffffff,
 346         0x000008f8, 0x00000029, 0xffffffff,
 347         0x000008fc, 0x00000000, 0xffffffff,
 348         0x000008f8, 0x0000002a, 0xffffffff,
 349         0x000008fc, 0x00000000, 0xffffffff,
 350         0x000008f8, 0x0000002b, 0xffffffff,
 351         0x000008fc, 0x00000000, 0xffffffff,
 352         0x00000644, 0x000f7902, 0x001f4180,
 353         0x00000644, 0x000f3802, 0x001f4180
 354 };
 355 #define CAYMAN_CGCG_CGLS_DISABLE_LENGTH sizeof(cayman_cgcg_cgls_disable) / (3 * sizeof(u32))
 356
 357 static const u32 cayman_cgcg_cgls_enable[] =
 358 {
 359         0x00000644, 0x000f7882, 0x001f4080,
 360         0x000008f8, 0x00000010, 0xffffffff,
 361         0x000008fc, 0x00000000, 0xffffffff,
 362         0x000008f8, 0x00000011, 0xffffffff,
 363         0x000008fc, 0x00000000, 0xffffffff,
 364         0x000008f8, 0x00000012, 0xffffffff,
 365         0x000008fc, 0x00000000, 0xffffffff,
 366         0x000008f8, 0x00000013, 0xffffffff,
 367         0x000008fc, 0x00000000, 0xffffffff,
 368         0x000008f8, 0x00000014, 0xffffffff,
 369         0x000008fc, 0x00000000, 0xffffffff,
 370         0x000008f8, 0x00000015, 0xffffffff,
 371         0x000008fc, 0x00000000, 0xffffffff,
 372         0x000008f8, 0x00000016, 0xffffffff,
 373         0x000008fc, 0x00000000, 0xffffffff,
 374         0x000008f8, 0x00000017, 0xffffffff,
 375         0x000008fc, 0x00000000, 0xffffffff,
 376         0x000008f8, 0x00000018, 0xffffffff,
 377         0x000008fc, 0x00000000, 0xffffffff,
 378         0x000008f8, 0x00000019, 0xffffffff,
 379         0x000008fc, 0x00000000, 0xffffffff,
 380         0x000008f8, 0x0000001a, 0xffffffff,
 381         0x000008fc, 0x00000000, 0xffffffff,
 382         0x000008f8, 0x0000001b, 0xffffffff,
 383         0x000008fc, 0x00000000, 0xffffffff,
 384         0x000008f8, 0x00000020, 0xffffffff,
 385         0x000008fc, 0xffffffff, 0xffffffff,
 386         0x000008f8, 0x00000021, 0xffffffff,
 387         0x000008fc, 0xffffffff, 0xffffffff,
 388         0x000008f8, 0x00000022, 0xffffffff,
 389         0x000008fc, 0xffffffff, 0xffffffff,
 390         0x000008f8, 0x00000023, 0xffffffff,
 391         0x000008fc, 0xffffffff, 0xffffffff,
 392         0x000008f8, 0x00000024, 0xffffffff,
 393         0x000008fc, 0xffffffff, 0xffffffff,
 394         0x000008f8, 0x00000025, 0xffffffff,
 395         0x000008fc, 0xffffffff, 0xffffffff,
 396         0x000008f8, 0x00000026, 0xffffffff,
 397         0x000008fc, 0xffffffff, 0xffffffff,
 398         0x000008f8, 0x00000027, 0xffffffff,
 399         0x000008fc, 0xffffffff, 0xffffffff,
 400         0x000008f8, 0x00000028, 0xffffffff,
 401         0x000008fc, 0xffffffff, 0xffffffff,
 402         0x000008f8, 0x00000029, 0xffffffff,
 403         0x000008fc, 0xffffffff, 0xffffffff,
 404         0x000008f8, 0x0000002a, 0xffffffff,
 405         0x000008fc, 0xffffffff, 0xffffffff,
 406         0x000008f8, 0x0000002b, 0xffffffff,
 407         0x000008fc, 0xffffffff, 0xffffffff
 408 };
 409 #define CAYMAN_CGCG_CGLS_ENABLE_LENGTH  sizeof(cayman_cgcg_cgls_enable) / (3 * sizeof(u32))
 410
 411 static const u32 cayman_mgcg_default[] =
 412 {
 413         0x0000802c, 0xc0000000, 0xffffffff,
 414         0x00003fc4, 0xc0000000, 0xffffffff,
 415         0x00005448, 0x00000100, 0xffffffff,
 416         0x000055e4, 0x00000100, 0xffffffff,
 417         0x0000160c, 0x00000100, 0xffffffff,
 418         0x00008984, 0x06000100, 0xffffffff,
 419         0x0000c164, 0x00000100, 0xffffffff,
 420         0x00008a18, 0x00000100, 0xffffffff,
 421         0x0000897c, 0x06000100, 0xffffffff,
 422         0x00008b28, 0x00000100, 0xffffffff,
 423         0x00009144, 0x00800200, 0xffffffff,
 424         0x00009a60, 0x00000100, 0xffffffff,
 425         0x00009868, 0x00000100, 0xffffffff,
 426         0x00008d58, 0x00000100, 0xffffffff,
 427         0x00009510, 0x00000100, 0xffffffff,
 428         0x0000949c, 0x00000100, 0xffffffff,
 429         0x00009654, 0x00000100, 0xffffffff,
 430         0x00009030, 0x00000100, 0xffffffff,
 431         0x00009034, 0x00000100, 0xffffffff,
 432         0x00009038, 0x00000100, 0xffffffff,
 433         0x0000903c, 0x00000100, 0xffffffff,
 434         0x00009040, 0x00000100, 0xffffffff,
 435         0x0000a200, 0x00000100, 0xffffffff,
 436         0x0000a204, 0x00000100, 0xffffffff,
 437         0x0000a208, 0x00000100, 0xffffffff,
 438         0x0000a20c, 0x00000100, 0xffffffff,
 439         0x00009744, 0x00000100, 0xffffffff,
 440         0x00003f80, 0x00000100, 0xffffffff,
 441         0x0000a210, 0x00000100, 0xffffffff,
 442         0x0000a214, 0x00000100, 0xffffffff,
 443         0x000004d8, 0x00000100, 0xffffffff,
 444         0x00009664, 0x00000100, 0xffffffff,
 445         0x00009698, 0x00000100, 0xffffffff,
 446         0x000004d4, 0x00000200, 0xffffffff,
 447         0x000004d0, 0x00000000, 0xffffffff,
 448         0x000030cc, 0x00000104, 0xffffffff,
 449         0x0000d0c0, 0x00000100, 0xffffffff,
 450         0x0000d8c0, 0x00000100, 0xffffffff,
 451         0x0000802c, 0x40000000, 0xffffffff,
 452         0x00003fc4, 0x40000000, 0xffffffff,
 453         0x0000915c, 0x00010000, 0xffffffff,
 454         0x00009160, 0x00030002, 0xffffffff,
 455         0x00009164, 0x00050004, 0xffffffff,
 456         0x00009168, 0x00070006, 0xffffffff,
 457         0x00009178, 0x00070000, 0xffffffff,
 458         0x0000917c, 0x00030002, 0xffffffff,
 459         0x00009180, 0x00050004, 0xffffffff,
 460         0x0000918c, 0x00010006, 0xffffffff,
 461         0x00009190, 0x00090008, 0xffffffff,
 462         0x00009194, 0x00070000, 0xffffffff,
 463         0x00009198, 0x00030002, 0xffffffff,
 464         0x0000919c, 0x00050004, 0xffffffff,
 465         0x000091a8, 0x00010006, 0xffffffff,
 466         0x000091ac, 0x00090008, 0xffffffff,
 467         0x000091b0, 0x00070000, 0xffffffff,
 468         0x000091b4, 0x00030002, 0xffffffff,
 469         0x000091b8, 0x00050004, 0xffffffff,
 470         0x000091c4, 0x00010006, 0xffffffff,
 471         0x000091c8, 0x00090008, 0xffffffff,
 472         0x000091cc, 0x00070000, 0xffffffff,
 473         0x000091d0, 0x00030002, 0xffffffff,
 474         0x000091d4, 0x00050004, 0xffffffff,
 475         0x000091e0, 0x00010006, 0xffffffff,
 476         0x000091e4, 0x00090008, 0xffffffff,
 477         0x000091e8, 0x00000000, 0xffffffff,
 478         0x000091ec, 0x00070000, 0xffffffff,
 479         0x000091f0, 0x00030002, 0xffffffff,
 480         0x000091f4, 0x00050004, 0xffffffff,
 481         0x00009200, 0x00010006, 0xffffffff,
 482         0x00009204, 0x00090008, 0xffffffff,
 483         0x00009208, 0x00070000, 0xffffffff,
 484         0x0000920c, 0x00030002, 0xffffffff,
 485         0x00009210, 0x00050004, 0xffffffff,
 486         0x0000921c, 0x00010006, 0xffffffff,
 487         0x00009220, 0x00090008, 0xffffffff,
 488         0x00009224, 0x00070000, 0xffffffff,
 489         0x00009228, 0x00030002, 0xffffffff,
 490         0x0000922c, 0x00050004, 0xffffffff,
 491         0x00009238, 0x00010006, 0xffffffff,
 492         0x0000923c, 0x00090008, 0xffffffff,
 493         0x00009240, 0x00070000, 0xffffffff,
 494         0x00009244, 0x00030002, 0xffffffff,
 495         0x00009248, 0x00050004, 0xffffffff,
 496         0x00009254, 0x00010006, 0xffffffff,
 497         0x00009258, 0x00090008, 0xffffffff,
 498         0x0000925c, 0x00070000, 0xffffffff,
 499         0x00009260, 0x00030002, 0xffffffff,
 500         0x00009264, 0x00050004, 0xffffffff,
 501         0x00009270, 0x00010006, 0xffffffff,
 502         0x00009274, 0x00090008, 0xffffffff,
 503         0x00009278, 0x00070000, 0xffffffff,
 504         0x0000927c, 0x00030002, 0xffffffff,
 505         0x00009280, 0x00050004, 0xffffffff,
 506         0x0000928c, 0x00010006, 0xffffffff,
 507         0x00009290, 0x00090008, 0xffffffff,
 508         0x000092a8, 0x00070000, 0xffffffff,
 509         0x000092ac, 0x00030002, 0xffffffff,
 510         0x000092b0, 0x00050004, 0xffffffff,
 511         0x000092bc, 0x00010006, 0xffffffff,
 512         0x000092c0, 0x00090008, 0xffffffff,
 513         0x000092c4, 0x00070000, 0xffffffff,
 514         0x000092c8, 0x00030002, 0xffffffff,
 515         0x000092cc, 0x00050004, 0xffffffff,
 516         0x000092d8, 0x00010006, 0xffffffff,
 517         0x000092dc, 0x00090008, 0xffffffff,
 518         0x00009294, 0x00000000, 0xffffffff,
 519         0x0000802c, 0x40010000, 0xffffffff,
 520         0x00003fc4, 0x40010000, 0xffffffff,
 521         0x0000915c, 0x00010000, 0xffffffff,
 522         0x00009160, 0x00030002, 0xffffffff,
 523         0x00009164, 0x00050004, 0xffffffff,
 524         0x00009168, 0x00070006, 0xffffffff,
 525         0x00009178, 0x00070000, 0xffffffff,
 526         0x0000917c, 0x00030002, 0xffffffff,
 527         0x00009180, 0x00050004, 0xffffffff,
 528         0x0000918c, 0x00010006, 0xffffffff,
 529         0x00009190, 0x00090008, 0xffffffff,
 530         0x00009194, 0x00070000, 0xffffffff,
 531         0x00009198, 0x00030002, 0xffffffff,
 532         0x0000919c, 0x00050004, 0xffffffff,
 533         0x000091a8, 0x00010006, 0xffffffff,
 534         0x000091ac, 0x00090008, 0xffffffff,
 535         0x000091b0, 0x00070000, 0xffffffff,
 536         0x000091b4, 0x00030002, 0xffffffff,
 537         0x000091b8, 0x00050004, 0xffffffff,
 538         0x000091c4, 0x00010006, 0xffffffff,
 539         0x000091c8, 0x00090008, 0xffffffff,
 540         0x000091cc, 0x00070000, 0xffffffff,
 541         0x000091d0, 0x00030002, 0xffffffff,
 542         0x000091d4, 0x00050004, 0xffffffff,
 543         0x000091e0, 0x00010006, 0xffffffff,
 544         0x000091e4, 0x00090008, 0xffffffff,
 545         0x000091e8, 0x00000000, 0xffffffff,
 546         0x000091ec, 0x00070000, 0xffffffff,
 547         0x000091f0, 0x00030002, 0xffffffff,
 548         0x000091f4, 0x00050004, 0xffffffff,
 549         0x00009200, 0x00010006, 0xffffffff,
 550         0x00009204, 0x00090008, 0xffffffff,
 551         0x00009208, 0x00070000, 0xffffffff,
 552         0x0000920c, 0x00030002, 0xffffffff,
 553         0x00009210, 0x00050004, 0xffffffff,
 554         0x0000921c, 0x00010006, 0xffffffff,
 555         0x00009220, 0x00090008, 0xffffffff,
 556         0x00009224, 0x00070000, 0xffffffff,
 557         0x00009228, 0x00030002, 0xffffffff,
 558         0x0000922c, 0x00050004, 0xffffffff,
 559         0x00009238, 0x00010006, 0xffffffff,
 560         0x0000923c, 0x00090008, 0xffffffff,
 561         0x00009240, 0x00070000, 0xffffffff,
 562         0x00009244, 0x00030002, 0xffffffff,
 563         0x00009248, 0x00050004, 0xffffffff,
 564         0x00009254, 0x00010006, 0xffffffff,
 565         0x00009258, 0x00090008, 0xffffffff,
 566         0x0000925c, 0x00070000, 0xffffffff,
 567         0x00009260, 0x00030002, 0xffffffff,
 568         0x00009264, 0x00050004, 0xffffffff,
 569         0x00009270, 0x00010006, 0xffffffff,
 570         0x00009274, 0x00090008, 0xffffffff,
 571         0x00009278, 0x00070000, 0xffffffff,
 572         0x0000927c, 0x00030002, 0xffffffff,
 573         0x00009280, 0x00050004, 0xffffffff,
 574         0x0000928c, 0x00010006, 0xffffffff,
 575         0x00009290, 0x00090008, 0xffffffff,
 576         0x000092a8, 0x00070000, 0xffffffff,
 577         0x000092ac, 0x00030002, 0xffffffff,
 578         0x000092b0, 0x00050004, 0xffffffff,
 579         0x000092bc, 0x00010006, 0xffffffff,
 580         0x000092c0, 0x00090008, 0xffffffff,
 581         0x000092c4, 0x00070000, 0xffffffff,
 582         0x000092c8, 0x00030002, 0xffffffff,
 583         0x000092cc, 0x00050004, 0xffffffff,
 584         0x000092d8, 0x00010006, 0xffffffff,
 585         0x000092dc, 0x00090008, 0xffffffff,
 586         0x00009294, 0x00000000, 0xffffffff,
 587         0x0000802c, 0xc0000000, 0xffffffff,
 588         0x00003fc4, 0xc0000000, 0xffffffff,
 589         0x000008f8, 0x00000010, 0xffffffff,
 590         0x000008fc, 0x00000000, 0xffffffff,
 591         0x000008f8, 0x00000011, 0xffffffff,
 592         0x000008fc, 0x00000000, 0xffffffff,
 593         0x000008f8, 0x00000012, 0xffffffff,
 594         0x000008fc, 0x00000000, 0xffffffff,
 595         0x000008f8, 0x00000013, 0xffffffff,
 596         0x000008fc, 0x00000000, 0xffffffff,
 597         0x000008f8, 0x00000014, 0xffffffff,
 598         0x000008fc, 0x00000000, 0xffffffff,
 599         0x000008f8, 0x00000015, 0xffffffff,
 600         0x000008fc, 0x00000000, 0xffffffff,
 601         0x000008f8, 0x00000016, 0xffffffff,
 602         0x000008fc, 0x00000000, 0xffffffff,
 603         0x000008f8, 0x00000017, 0xffffffff,
 604         0x000008fc, 0x00000000, 0xffffffff,
 605         0x000008f8, 0x00000018, 0xffffffff,
 606         0x000008fc, 0x00000000, 0xffffffff,
 607         0x000008f8, 0x00000019, 0xffffffff,
 608         0x000008fc, 0x00000000, 0xffffffff,
 609         0x000008f8, 0x0000001a, 0xffffffff,
 610         0x000008fc, 0x00000000, 0xffffffff,
 611         0x000008f8, 0x0000001b, 0xffffffff,
 612         0x000008fc, 0x00000000, 0xffffffff
 613 };
 614 #define CAYMAN_MGCG_DEFAULT_LENGTH sizeof(cayman_mgcg_default) / (3 * sizeof(u32))
 615
 616 static const u32 cayman_mgcg_disable[] =
 617 {
 618         0x0000802c, 0xc0000000, 0xffffffff,
 619         0x000008f8, 0x00000000, 0xffffffff,
 620         0x000008fc, 0xffffffff, 0xffffffff,
 621         0x000008f8, 0x00000001, 0xffffffff,
 622         0x000008fc, 0xffffffff, 0xffffffff,
 623         0x000008f8, 0x00000002, 0xffffffff,
 624         0x000008fc, 0xffffffff, 0xffffffff,
 625         0x000008f8, 0x00000003, 0xffffffff,
 626         0x000008fc, 0xffffffff, 0xffffffff,
 627         0x00009150, 0x00600000, 0xffffffff
 628 };
 629 #define CAYMAN_MGCG_DISABLE_LENGTH   sizeof(cayman_mgcg_disable) / (3 * sizeof(u32))
 630
 631 static const u32 cayman_mgcg_enable[] =
 632 {
 633         0x0000802c, 0xc0000000, 0xffffffff,
 634         0x000008f8, 0x00000000, 0xffffffff,
 635         0x000008fc, 0x00000000, 0xffffffff,
 636         0x000008f8, 0x00000001, 0xffffffff,
 637         0x000008fc, 0x00000000, 0xffffffff,
 638         0x000008f8, 0x00000002, 0xffffffff,
 639         0x000008fc, 0x00600000, 0xffffffff,
 640         0x000008f8, 0x00000003, 0xffffffff,
 641         0x000008fc, 0x00000000, 0xffffffff,
 642         0x00009150, 0x96944200, 0xffffffff
 643 };
 644
 645 #define CAYMAN_MGCG_ENABLE_LENGTH   sizeof(cayman_mgcg_enable) / (3 * sizeof(u32))
 646
 647 #define NISLANDS_SYSLS_SEQUENCE  100
 648
 649 static const u32 cayman_sysls_default[] =
 650 {
 651         /* Register,   Value,     Mask bits */
 652         0x000055e8, 0x00000000, 0xffffffff,
 653         0x0000d0bc, 0x00000000, 0xffffffff,
 654         0x0000d8bc, 0x00000000, 0xffffffff,
 655         0x000015c0, 0x000c1401, 0xffffffff,
 656         0x0000264c, 0x000c0400, 0xffffffff,
 657         0x00002648, 0x000c0400, 0xffffffff,
 658         0x00002650, 0x000c0400, 0xffffffff,
 659         0x000020b8, 0x000c0400, 0xffffffff,
 660         0x000020bc, 0x000c0400, 0xffffffff,
 661         0x000020c0, 0x000c0c80, 0xffffffff,
 662         0x0000f4a0, 0x000000c0, 0xffffffff,
 663         0x0000f4a4, 0x00680fff, 0xffffffff,
 664         0x00002f50, 0x00000404, 0xffffffff,
 665         0x000004c8, 0x00000001, 0xffffffff,
 666         0x000064ec, 0x00000000, 0xffffffff,
 667         0x00000c7c, 0x00000000, 0xffffffff,
 668         0x00008dfc, 0x00000000, 0xffffffff
 669 };
 670 #define CAYMAN_SYSLS_DEFAULT_LENGTH sizeof(cayman_sysls_default) / (3 * sizeof(u32))
 671
 672 static const u32 cayman_sysls_disable[] =
 673 {
 674         /* Register,   Value,     Mask bits */
 675         0x0000d0c0, 0x00000000, 0xffffffff,
 676         0x0000d8c0, 0x00000000, 0xffffffff,
 677         0x000055e8, 0x00000000, 0xffffffff,
 678         0x0000d0bc, 0x00000000, 0xffffffff,
 679         0x0000d8bc, 0x00000000, 0xffffffff,
 680         0x000015c0, 0x00041401, 0xffffffff,
 681         0x0000264c, 0x00040400, 0xffffffff,
 682         0x00002648, 0x00040400, 0xffffffff,
 683         0x00002650, 0x00040400, 0xffffffff,
 684         0x000020b8, 0x00040400, 0xffffffff,
 685         0x000020bc, 0x00040400, 0xffffffff,
 686         0x000020c0, 0x00040c80, 0xffffffff,
 687         0x0000f4a0, 0x000000c0, 0xffffffff,
 688         0x0000f4a4, 0x00680000, 0xffffffff,
 689         0x00002f50, 0x00000404, 0xffffffff,
 690         0x000004c8, 0x00000001, 0xffffffff,
 691         0x000064ec, 0x00007ffd, 0xffffffff,
 692         0x00000c7c, 0x0000ff00, 0xffffffff,
 693         0x00008dfc, 0x0000007f, 0xffffffff
 694 };
 695 #define CAYMAN_SYSLS_DISABLE_LENGTH sizeof(cayman_sysls_disable) / (3 * sizeof(u32))
 696
 697 static const u32 cayman_sysls_enable[] =
 698 {
 699         /* Register,   Value,     Mask bits */
 700         0x000055e8, 0x00000001, 0xffffffff,
 701         0x0000d0bc, 0x00000100, 0xffffffff,
 702         0x0000d8bc, 0x00000100, 0xffffffff,
 703         0x000015c0, 0x000c1401, 0xffffffff,
 704         0x0000264c, 0x000c0400, 0xffffffff,
 705         0x00002648, 0x000c0400, 0xffffffff,
 706         0x00002650, 0x000c0400, 0xffffffff,
 707         0x000020b8, 0x000c0400, 0xffffffff,
 708         0x000020bc, 0x000c0400, 0xffffffff,
 709         0x000020c0, 0x000c0c80, 0xffffffff,
 710         0x0000f4a0, 0x000000c0, 0xffffffff,
 711         0x0000f4a4, 0x00680fff, 0xffffffff,
 712         0x00002f50, 0x00000903, 0xffffffff,
 713         0x000004c8, 0x00000000, 0xffffffff,
 714         0x000064ec, 0x00000000, 0xffffffff,
 715         0x00000c7c, 0x00000000, 0xffffffff,
 716         0x00008dfc, 0x00000000, 0xffffffff
 717 };
 718 #define CAYMAN_SYSLS_ENABLE_LENGTH sizeof(cayman_sysls_enable) / (3 * sizeof(u32))
 719
 720 struct rv7xx_power_info *rv770_get_pi(struct radeon_device *rdev);
 721 struct evergreen_power_info *evergreen_get_pi(struct radeon_device *rdev);
 722
 723 struct ni_power_info *ni_get_pi(struct radeon_device *rdev)
 724 {
 725         struct ni_power_info *pi = rdev->pm.dpm.priv;
 726
 727         return pi;
 728 }
 729
 730 struct ni_ps *ni_get_ps(struct radeon_ps *rps)
 731 {
 732         struct ni_ps *ps = rps->ps_priv;
 733
 734         return ps;
 735 }
 736
 737 static void ni_calculate_leakage_for_v_and_t_formula(const struct ni_leakage_coeffients *coeff,
 738                                                      u16 v, s32 t,
 739                                                      u32 ileakage,
 740                                                      u32 *leakage)
 741 {
 742         s64 kt, kv, leakage_w, i_leakage, vddc, temperature;
 743
 744         i_leakage = div64_s64(drm_int2fixp(ileakage), 1000);
 745         vddc = div64_s64(drm_int2fixp(v), 1000);
 746         temperature = div64_s64(drm_int2fixp(t), 1000);
 747
 748         kt = drm_fixp_mul(div64_s64(drm_int2fixp(coeff->at), 1000),
 749                           drm_fixp_exp(drm_fixp_mul(div64_s64(drm_int2fixp(coeff->bt), 1000), temperature)));
 750         kv = drm_fixp_mul(div64_s64(drm_int2fixp(coeff->av), 1000),
 751                           drm_fixp_exp(drm_fixp_mul(div64_s64(drm_int2fixp(coeff->bv), 1000), vddc)));
 752
 753         leakage_w = drm_fixp_mul(drm_fixp_mul(drm_fixp_mul(i_leakage, kt), kv), vddc);
 754
 755         *leakage = drm_fixp2int(leakage_w * 1000);
 756 }
 757
 758 static void ni_calculate_leakage_for_v_and_t(struct radeon_device *rdev,
 759                                              const struct ni_leakage_coeffients *coeff,
 760                                              u16 v,
 761                                              s32 t,
 762                                              u32 i_leakage,
 763                                              u32 *leakage)
 764 {
 765         ni_calculate_leakage_for_v_and_t_formula(coeff, v, t, i_leakage, leakage);
 766 }
 767
 768 bool ni_dpm_vblank_too_short(struct radeon_device *rdev)
 769 {
 770         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
 771         u32 vblank_time = r600_dpm_get_vblank_time(rdev);
 772         u32 switch_limit = pi->mem_gddr5 ? 450 : 300;
 773
 774         if (vblank_time < switch_limit)
 775                 return true;
 776         else
 777                 return false;
 778
 779 }
 780
 781 static void ni_apply_state_adjust_rules(struct radeon_device *rdev,
 782                                         struct radeon_ps *rps)
 783 {
 784         struct ni_ps *ps = ni_get_ps(rps);
 785         struct radeon_clock_and_voltage_limits *max_limits;
 786         bool disable_mclk_switching;
 787         u32 mclk, sclk;
 788         u16 vddc, vddci;
 789         int i;
 790
 791         if ((rdev->pm.dpm.new_active_crtc_count > 1) ||
 792             ni_dpm_vblank_too_short(rdev))
 793                 disable_mclk_switching = true;
 794         else
 795                 disable_mclk_switching = false;
 796
 797         if (rdev->pm.dpm.ac_power)
 798                 max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
 799         else
 800                 max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc;
 801
 802         if (rdev->pm.dpm.ac_power == false) {
 803                 for (i = 0; i < ps->performance_level_count; i++) {
 804                         if (ps->performance_levels[i].mclk > max_limits->mclk)
 805                                 ps->performance_levels[i].mclk = max_limits->mclk;
 806                         if (ps->performance_levels[i].sclk > max_limits->sclk)
 807                                 ps->performance_levels[i].sclk = max_limits->sclk;
 808                         if (ps->performance_levels[i].vddc > max_limits->vddc)
 809                                 ps->performance_levels[i].vddc = max_limits->vddc;
 810                         if (ps->performance_levels[i].vddci > max_limits->vddci)
 811                                 ps->performance_levels[i].vddci = max_limits->vddci;
 812                 }
 813         }
 814
 815         /* XXX validate the min clocks required for display */
 816
 817         if (disable_mclk_switching) {
 818                 mclk  = ps->performance_levels[ps->performance_level_count - 1].mclk;
 819                 sclk = ps->performance_levels[0].sclk;
 820                 vddc = ps->performance_levels[0].vddc;
 821                 vddci = ps->performance_levels[ps->performance_level_count - 1].vddci;
 822         } else {
 823                 sclk = ps->performance_levels[0].sclk;
 824                 mclk = ps->performance_levels[0].mclk;
 825                 vddc = ps->performance_levels[0].vddc;
 826                 vddci = ps->performance_levels[0].vddci;
 827         }
 828
 829         /* adjusted low state */
 830         ps->performance_levels[0].sclk = sclk;
 831         ps->performance_levels[0].mclk = mclk;
 832         ps->performance_levels[0].vddc = vddc;
 833         ps->performance_levels[0].vddci = vddci;
 834
 835         btc_skip_blacklist_clocks(rdev, max_limits->sclk, max_limits->mclk,
 836                                   &ps->performance_levels[0].sclk,
 837                                   &ps->performance_levels[0].mclk);
 838
 839         for (i = 1; i < ps->performance_level_count; i++) {
 840                 if (ps->performance_levels[i].sclk < ps->performance_levels[i - 1].sclk)
 841                         ps->performance_levels[i].sclk = ps->performance_levels[i - 1].sclk;
 842                 if (ps->performance_levels[i].vddc < ps->performance_levels[i - 1].vddc)
 843                         ps->performance_levels[i].vddc = ps->performance_levels[i - 1].vddc;
 844         }
 845
 846         if (disable_mclk_switching) {
 847                 mclk = ps->performance_levels[0].mclk;
 848                 for (i = 1; i < ps->performance_level_count; i++) {
 849                         if (mclk < ps->performance_levels[i].mclk)
 850                                 mclk = ps->performance_levels[i].mclk;
 851                 }
 852                 for (i = 0; i < ps->performance_level_count; i++) {
 853                         ps->performance_levels[i].mclk = mclk;
 854                         ps->performance_levels[i].vddci = vddci;
 855                 }
 856         } else {
 857                 for (i = 1; i < ps->performance_level_count; i++) {
 858                         if (ps->performance_levels[i].mclk < ps->performance_levels[i - 1].mclk)
 859                                 ps->performance_levels[i].mclk = ps->performance_levels[i - 1].mclk;
 860                         if (ps->performance_levels[i].vddci < ps->performance_levels[i - 1].vddci)
 861                                 ps->performance_levels[i].vddci = ps->performance_levels[i - 1].vddci;
 862                 }
 863         }
 864
 865         for (i = 1; i < ps->performance_level_count; i++)
 866                 btc_skip_blacklist_clocks(rdev, max_limits->sclk, max_limits->mclk,
 867                                           &ps->performance_levels[i].sclk,
 868                                           &ps->performance_levels[i].mclk);
 869
 870         for (i = 0; i < ps->performance_level_count; i++)
 871                 btc_adjust_clock_combinations(rdev, max_limits,
 872                                               &ps->performance_levels[i]);
 873
 874         for (i = 0; i < ps->performance_level_count; i++) {
 875                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk,
 876                                                    ps->performance_levels[i].sclk,
 877                                                    max_limits->vddc,  &ps->performance_levels[i].vddc);
 878                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk,
 879                                                    ps->performance_levels[i].mclk,
 880                                                    max_limits->vddci, &ps->performance_levels[i].vddci);
 881                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk,
 882                                                    ps->performance_levels[i].mclk,
 883                                                    max_limits->vddc,  &ps->performance_levels[i].vddc);
 884                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk,
 885                                                    rdev->clock.current_dispclk,
 886                                                    max_limits->vddc,  &ps->performance_levels[i].vddc);
 887         }
 888
 889         for (i = 0; i < ps->performance_level_count; i++) {
 890                 btc_apply_voltage_delta_rules(rdev,
 891                                               max_limits->vddc, max_limits->vddci,
 892                                               &ps->performance_levels[i].vddc,
 893                                               &ps->performance_levels[i].vddci);
 894         }
 895
 896         ps->dc_compatible = true;
 897         for (i = 0; i < ps->performance_level_count; i++) {
 898                 if (ps->performance_levels[i].vddc > rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.vddc)
 899                         ps->dc_compatible = false;
 900
 901                 if (ps->performance_levels[i].vddc < rdev->pm.dpm.dyn_state.min_vddc_for_pcie_gen2)
 902                         ps->performance_levels[i].flags &= ~ATOM_PPLIB_R600_FLAGS_PCIEGEN2;
 903         }
 904 }
 905
 906 static void ni_cg_clockgating_default(struct radeon_device *rdev)
 907 {
 908         u32 count;
 909         const u32 *ps = NULL;
 910
 911         ps = (const u32 *)&cayman_cgcg_cgls_default;
 912         count = CAYMAN_CGCG_CGLS_DEFAULT_LENGTH;
 913
 914         btc_program_mgcg_hw_sequence(rdev, ps, count);
 915 }
 916
 917 static void ni_gfx_clockgating_enable(struct radeon_device *rdev,
 918                                       bool enable)
 919 {
 920         u32 count;
 921         const u32 *ps = NULL;
 922
 923         if (enable) {
 924                 ps = (const u32 *)&cayman_cgcg_cgls_enable;
 925                 count = CAYMAN_CGCG_CGLS_ENABLE_LENGTH;
 926         } else {
 927                 ps = (const u32 *)&cayman_cgcg_cgls_disable;
 928                 count = CAYMAN_CGCG_CGLS_DISABLE_LENGTH;
 929         }
 930
 931         btc_program_mgcg_hw_sequence(rdev, ps, count);
 932 }
 933
 934 static void ni_mg_clockgating_default(struct radeon_device *rdev)
 935 {
 936         u32 count;
 937         const u32 *ps = NULL;
 938
 939         ps = (const u32 *)&cayman_mgcg_default;
 940         count = CAYMAN_MGCG_DEFAULT_LENGTH;
 941
 942         btc_program_mgcg_hw_sequence(rdev, ps, count);
 943 }
 944
 945 static void ni_mg_clockgating_enable(struct radeon_device *rdev,
 946                                      bool enable)
 947 {
 948         u32 count;
 949         const u32 *ps = NULL;
 950
 951         if (enable) {
 952                 ps = (const u32 *)&cayman_mgcg_enable;
 953                 count = CAYMAN_MGCG_ENABLE_LENGTH;
 954         } else {
 955                 ps = (const u32 *)&cayman_mgcg_disable;
 956                 count = CAYMAN_MGCG_DISABLE_LENGTH;
 957         }
 958
 959         btc_program_mgcg_hw_sequence(rdev, ps, count);
 960 }
 961
 962 static void ni_ls_clockgating_default(struct radeon_device *rdev)
 963 {
 964         u32 count;
 965         const u32 *ps = NULL;
 966
 967         ps = (const u32 *)&cayman_sysls_default;
 968         count = CAYMAN_SYSLS_DEFAULT_LENGTH;
 969
 970         btc_program_mgcg_hw_sequence(rdev, ps, count);
 971 }
 972
 973 static void ni_ls_clockgating_enable(struct radeon_device *rdev,
 974                                      bool enable)
 975 {
 976         u32 count;
 977         const u32 *ps = NULL;
 978
 979         if (enable) {
 980                 ps = (const u32 *)&cayman_sysls_enable;
 981                 count = CAYMAN_SYSLS_ENABLE_LENGTH;
 982         } else {
 983                 ps = (const u32 *)&cayman_sysls_disable;
 984                 count = CAYMAN_SYSLS_DISABLE_LENGTH;
 985         }
 986
 987         btc_program_mgcg_hw_sequence(rdev, ps, count);
 988
 989 }
 990
 991 static int ni_patch_single_dependency_table_based_on_leakage(struct radeon_device *rdev,
 992                                                              struct radeon_clock_voltage_dependency_table *table)
 993 {
 994         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
 995         u32 i;
 996
 997         if (table) {
 998                 for (i = 0; i < table->count; i++) {
 999                         if (0xff01 == table->entries[i].v) {
1000                                 if (pi->max_vddc == 0)
1001                                         return -EINVAL;
1002                                 table->entries[i].v = pi->max_vddc;
1003                         }
1004                 }
1005         }
1006         return 0;
1007 }
1008
1009 static int ni_patch_dependency_tables_based_on_leakage(struct radeon_device *rdev)
1010 {
1011         int ret = 0;
1012
1013         ret = ni_patch_single_dependency_table_based_on_leakage(rdev,
1014                                                                 &rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk);
1015
1016         ret = ni_patch_single_dependency_table_based_on_leakage(rdev,
1017                                                                 &rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk);
1018         return ret;
1019 }
1020
1021 static void ni_stop_dpm(struct radeon_device *rdev)
1022 {
1023         WREG32_P(GENERAL_PWRMGT, 0, ~GLOBAL_PWRMGT_EN);
1024 }
1025
1026 #if 0
1027 static int ni_notify_hw_of_power_source(struct radeon_device *rdev,
1028                                         bool ac_power)
1029 {
1030         if (ac_power)
1031                 return (rv770_send_msg_to_smc(rdev, PPSMC_MSG_RunningOnAC) == PPSMC_Result_OK) ?
1032                         0 : -EINVAL;
1033
1034         return 0;
1035 }
1036 #endif
1037
1038 static PPSMC_Result ni_send_msg_to_smc_with_parameter(struct radeon_device *rdev,
1039                                                       PPSMC_Msg msg, u32 parameter)
1040 {
1041         WREG32(SMC_SCRATCH0, parameter);
1042         return rv770_send_msg_to_smc(rdev, msg);
1043 }
1044
1045 static int ni_restrict_performance_levels_before_switch(struct radeon_device *rdev)
1046 {
1047         if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_NoForcedLevel) != PPSMC_Result_OK)
1048                 return -EINVAL;
1049
1050         return (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 1) == PPSMC_Result_OK) ?
1051                 0 : -EINVAL;
1052 }
1053
1054 int ni_dpm_force_performance_level(struct radeon_device *rdev,
1055                                    enum radeon_dpm_forced_level level)
1056 {
1057         if (level == RADEON_DPM_FORCED_LEVEL_HIGH) {
1058                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 0) != PPSMC_Result_OK)
1059                         return -EINVAL;
1060
1061                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 1) != PPSMC_Result_OK)
1062                         return -EINVAL;
1063         } else if (level == RADEON_DPM_FORCED_LEVEL_LOW) {
1064                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 0) != PPSMC_Result_OK)
1065                         return -EINVAL;
1066
1067                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 1) != PPSMC_Result_OK)
1068                         return -EINVAL;
1069         } else if (level == RADEON_DPM_FORCED_LEVEL_AUTO) {
1070                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 0) != PPSMC_Result_OK)
1071                         return -EINVAL;
1072
1073                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 0) != PPSMC_Result_OK)
1074                         return -EINVAL;
1075         }
1076
1077         rdev->pm.dpm.forced_level = level;
1078
1079         return 0;
1080 }
1081
1082 static void ni_stop_smc(struct radeon_device *rdev)
1083 {
1084         u32 tmp;
1085         int i;
1086
1087         for (i = 0; i < rdev->usec_timeout; i++) {
1088                 tmp = RREG32(LB_SYNC_RESET_SEL) & LB_SYNC_RESET_SEL_MASK;
1089                 if (tmp != 1)
1090                         break;
1091                 udelay(1);
1092         }
1093
1094         udelay(100);
1095
1096         r7xx_stop_smc(rdev);
1097 }
1098
1099 static int ni_process_firmware_header(struct radeon_device *rdev)
1100 {
1101         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1102         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1103         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1104         u32 tmp;
1105         int ret;
1106
1107         ret = rv770_read_smc_sram_dword(rdev,
1108                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1109                                         NISLANDS_SMC_FIRMWARE_HEADER_stateTable,
1110                                         &tmp, pi->sram_end);
1111
1112         if (ret)
1113                 return ret;
1114
1115         pi->state_table_start = (u16)tmp;
1116
1117         ret = rv770_read_smc_sram_dword(rdev,
1118                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1119                                         NISLANDS_SMC_FIRMWARE_HEADER_softRegisters,
1120                                         &tmp, pi->sram_end);
1121
1122         if (ret)
1123                 return ret;
1124
1125         pi->soft_regs_start = (u16)tmp;
1126
1127         ret = rv770_read_smc_sram_dword(rdev,
1128                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1129                                         NISLANDS_SMC_FIRMWARE_HEADER_mcRegisterTable,
1130                                         &tmp, pi->sram_end);
1131
1132         if (ret)
1133                 return ret;
1134
1135         eg_pi->mc_reg_table_start = (u16)tmp;
1136
1137         ret = rv770_read_smc_sram_dword(rdev,
1138                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1139                                         NISLANDS_SMC_FIRMWARE_HEADER_fanTable,
1140                                         &tmp, pi->sram_end);
1141
1142         if (ret)
1143                 return ret;
1144
1145         ni_pi->fan_table_start = (u16)tmp;
1146
1147         ret = rv770_read_smc_sram_dword(rdev,
1148                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1149                                         NISLANDS_SMC_FIRMWARE_HEADER_mcArbDramAutoRefreshTable,
1150                                         &tmp, pi->sram_end);
1151
1152         if (ret)
1153                 return ret;
1154
1155         ni_pi->arb_table_start = (u16)tmp;
1156
1157         ret = rv770_read_smc_sram_dword(rdev,
1158                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1159                                         NISLANDS_SMC_FIRMWARE_HEADER_cacTable,
1160                                         &tmp, pi->sram_end);
1161
1162         if (ret)
1163                 return ret;
1164
1165         ni_pi->cac_table_start = (u16)tmp;
1166
1167         ret = rv770_read_smc_sram_dword(rdev,
1168                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1169                                         NISLANDS_SMC_FIRMWARE_HEADER_spllTable,
1170                                         &tmp, pi->sram_end);
1171
1172         if (ret)
1173                 return ret;
1174
1175         ni_pi->spll_table_start = (u16)tmp;
1176
1177
1178         return ret;
1179 }
1180
1181 static void ni_read_clock_registers(struct radeon_device *rdev)
1182 {
1183         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1184
1185         ni_pi->clock_registers.cg_spll_func_cntl = RREG32(CG_SPLL_FUNC_CNTL);
1186         ni_pi->clock_registers.cg_spll_func_cntl_2 = RREG32(CG_SPLL_FUNC_CNTL_2);
1187         ni_pi->clock_registers.cg_spll_func_cntl_3 = RREG32(CG_SPLL_FUNC_CNTL_3);
1188         ni_pi->clock_registers.cg_spll_func_cntl_4 = RREG32(CG_SPLL_FUNC_CNTL_4);
1189         ni_pi->clock_registers.cg_spll_spread_spectrum = RREG32(CG_SPLL_SPREAD_SPECTRUM);
1190         ni_pi->clock_registers.cg_spll_spread_spectrum_2 = RREG32(CG_SPLL_SPREAD_SPECTRUM_2);
1191         ni_pi->clock_registers.mpll_ad_func_cntl = RREG32(MPLL_AD_FUNC_CNTL);
1192         ni_pi->clock_registers.mpll_ad_func_cntl_2 = RREG32(MPLL_AD_FUNC_CNTL_2);
1193         ni_pi->clock_registers.mpll_dq_func_cntl = RREG32(MPLL_DQ_FUNC_CNTL);
1194         ni_pi->clock_registers.mpll_dq_func_cntl_2 = RREG32(MPLL_DQ_FUNC_CNTL_2);
1195         ni_pi->clock_registers.mclk_pwrmgt_cntl = RREG32(MCLK_PWRMGT_CNTL);
1196         ni_pi->clock_registers.dll_cntl = RREG32(DLL_CNTL);
1197         ni_pi->clock_registers.mpll_ss1 = RREG32(MPLL_SS1);
1198         ni_pi->clock_registers.mpll_ss2 = RREG32(MPLL_SS2);
1199 }
1200
1201 #if 0
1202 static int ni_enter_ulp_state(struct radeon_device *rdev)
1203 {
1204         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1205
1206         if (pi->gfx_clock_gating) {
1207                 WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN);
1208                 WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON);
1209                 WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON);
1210                 RREG32(GB_ADDR_CONFIG);
1211         }
1212
1213         WREG32_P(SMC_MSG, HOST_SMC_MSG(PPSMC_MSG_SwitchToMinimumPower),
1214                  ~HOST_SMC_MSG_MASK);
1215
1216         udelay(25000);
1217
1218         return 0;
1219 }
1220 #endif
1221
1222 static void ni_program_response_times(struct radeon_device *rdev)
1223 {
1224         u32 voltage_response_time, backbias_response_time, acpi_delay_time, vbi_time_out;
1225         u32 vddc_dly, bb_dly, acpi_dly, vbi_dly, mclk_switch_limit;
1226         u32 reference_clock;
1227
1228         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mvdd_chg_time, 1);
1229
1230         voltage_response_time = (u32)rdev->pm.dpm.voltage_response_time;
1231         backbias_response_time = (u32)rdev->pm.dpm.backbias_response_time;
1232
1233         if (voltage_response_time == 0)
1234                 voltage_response_time = 1000;
1235
1236         if (backbias_response_time == 0)
1237                 backbias_response_time = 1000;
1238
1239         acpi_delay_time = 15000;
1240         vbi_time_out = 100000;
1241
1242         reference_clock = radeon_get_xclk(rdev);
1243
1244         vddc_dly = (voltage_response_time  * reference_clock) / 1600;
1245         bb_dly   = (backbias_response_time * reference_clock) / 1600;
1246         acpi_dly = (acpi_delay_time * reference_clock) / 1600;
1247         vbi_dly  = (vbi_time_out * reference_clock) / 1600;
1248
1249         mclk_switch_limit = (460 * reference_clock) / 100;
1250
1251         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_vreg,  vddc_dly);
1252         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_bbias, bb_dly);
1253         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_acpi,  acpi_dly);
1254         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mclk_chg_timeout, vbi_dly);
1255         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mc_block_delay, 0xAA);
1256         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mclk_switch_lim, mclk_switch_limit);
1257 }
1258
1259 static void ni_populate_smc_voltage_table(struct radeon_device *rdev,
1260                                           struct atom_voltage_table *voltage_table,
1261                                           NISLANDS_SMC_STATETABLE *table)
1262 {
1263         unsigned int i;
1264
1265         for (i = 0; i < voltage_table->count; i++) {
1266                 table->highSMIO[i] = 0;
1267                 table->lowSMIO[i] |= cpu_to_be32(voltage_table->entries[i].smio_low);
1268         }
1269 }
1270
1271 static void ni_populate_smc_voltage_tables(struct radeon_device *rdev,
1272                                            NISLANDS_SMC_STATETABLE *table)
1273 {
1274         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1275         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1276         unsigned char i;
1277
1278         if (eg_pi->vddc_voltage_table.count) {
1279                 ni_populate_smc_voltage_table(rdev, &eg_pi->vddc_voltage_table, table);
1280                 table->voltageMaskTable.highMask[NISLANDS_SMC_VOLTAGEMASK_VDDC] = 0;
1281                 table->voltageMaskTable.lowMask[NISLANDS_SMC_VOLTAGEMASK_VDDC] =
1282                         cpu_to_be32(eg_pi->vddc_voltage_table.mask_low);
1283
1284                 for (i = 0; i < eg_pi->vddc_voltage_table.count; i++) {
1285                         if (pi->max_vddc_in_table <= eg_pi->vddc_voltage_table.entries[i].value) {
1286                                 table->maxVDDCIndexInPPTable = i;
1287                                 break;
1288                         }
1289                 }
1290         }
1291
1292         if (eg_pi->vddci_voltage_table.count) {
1293                 ni_populate_smc_voltage_table(rdev, &eg_pi->vddci_voltage_table, table);
1294
1295                 table->voltageMaskTable.highMask[NISLANDS_SMC_VOLTAGEMASK_VDDCI] = 0;
1296                 table->voltageMaskTable.lowMask[NISLANDS_SMC_VOLTAGEMASK_VDDCI] =
1297                         cpu_to_be32(eg_pi->vddc_voltage_table.mask_low);
1298         }
1299 }
1300
1301 static int ni_populate_voltage_value(struct radeon_device *rdev,
1302                                      struct atom_voltage_table *table,
1303                                      u16 value,
1304                                      NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1305 {
1306         unsigned int i;
1307
1308         for (i = 0; i < table->count; i++) {
1309                 if (value <= table->entries[i].value) {
1310                         voltage->index = (u8)i;
1311                         voltage->value = cpu_to_be16(table->entries[i].value);
1312                         break;
1313                 }
1314         }
1315
1316         if (i >= table->count)
1317                 return -EINVAL;
1318
1319         return 0;
1320 }
1321
1322 static void ni_populate_mvdd_value(struct radeon_device *rdev,
1323                                    u32 mclk,
1324                                    NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1325 {
1326         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1327         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1328
1329         if (!pi->mvdd_control) {
1330                 voltage->index = eg_pi->mvdd_high_index;
1331                 voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1332                 return;
1333         }
1334
1335         if (mclk <= pi->mvdd_split_frequency) {
1336                 voltage->index = eg_pi->mvdd_low_index;
1337                 voltage->value = cpu_to_be16(MVDD_LOW_VALUE);
1338         } else {
1339                 voltage->index = eg_pi->mvdd_high_index;
1340                 voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1341         }
1342 }
1343
1344 static int ni_get_std_voltage_value(struct radeon_device *rdev,
1345                                     NISLANDS_SMC_VOLTAGE_VALUE *voltage,
1346                                     u16 *std_voltage)
1347 {
1348         if (rdev->pm.dpm.dyn_state.cac_leakage_table.entries &&
1349             ((u32)voltage->index < rdev->pm.dpm.dyn_state.cac_leakage_table.count))
1350                 *std_voltage = rdev->pm.dpm.dyn_state.cac_leakage_table.entries[voltage->index].vddc;
1351         else
1352                 *std_voltage = be16_to_cpu(voltage->value);
1353
1354         return 0;
1355 }
1356
1357 static void ni_populate_std_voltage_value(struct radeon_device *rdev,
1358                                           u16 value, u8 index,
1359                                           NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1360 {
1361         voltage->index = index;
1362         voltage->value = cpu_to_be16(value);
1363 }
1364
1365 static u32 ni_get_smc_power_scaling_factor(struct radeon_device *rdev)
1366 {
1367         u32 xclk_period;
1368         u32 xclk = radeon_get_xclk(rdev);
1369         u32 tmp = RREG32(CG_CAC_CTRL) & TID_CNT_MASK;
1370
1371         xclk_period = (1000000000UL / xclk);
1372         xclk_period /= 10000UL;
1373
1374         return tmp * xclk_period;
1375 }
1376
1377 static u32 ni_scale_power_for_smc(u32 power_in_watts, u32 scaling_factor)
1378 {
1379         return (power_in_watts * scaling_factor) << 2;
1380 }
1381
1382 static u32 ni_calculate_power_boost_limit(struct radeon_device *rdev,
1383                                           struct radeon_ps *radeon_state,
1384                                           u32 near_tdp_limit)
1385 {
1386         struct ni_ps *state = ni_get_ps(radeon_state);
1387         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1388         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1389         u32 power_boost_limit = 0;
1390         int ret;
1391
1392         if (ni_pi->enable_power_containment &&
1393             ni_pi->use_power_boost_limit) {
1394                 NISLANDS_SMC_VOLTAGE_VALUE vddc;
1395                 u16 std_vddc_med;
1396                 u16 std_vddc_high;
1397                 u64 tmp, n, d;
1398
1399                 if (state->performance_level_count < 3)
1400                         return 0;
1401
1402                 ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
1403                                                 state->performance_levels[state->performance_level_count - 2].vddc,
1404                                                 &vddc);
1405                 if (ret)
1406                         return 0;
1407
1408                 ret = ni_get_std_voltage_value(rdev, &vddc, &std_vddc_med);
1409                 if (ret)
1410                         return 0;
1411
1412                 ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
1413                                                 state->performance_levels[state->performance_level_count - 1].vddc,
1414                                                 &vddc);
1415                 if (ret)
1416                         return 0;
1417
1418                 ret = ni_get_std_voltage_value(rdev, &vddc, &std_vddc_high);
1419                 if (ret)
1420                         return 0;
1421
1422                 n = ((u64)near_tdp_limit * ((u64)std_vddc_med * (u64)std_vddc_med) * 90);
1423                 d = ((u64)std_vddc_high * (u64)std_vddc_high * 100);
1424                 tmp = div64_u64(n, d);
1425
1426                 if (tmp >> 32)
1427                         return 0;
1428                 power_boost_limit = (u32)tmp;
1429         }
1430
1431         return power_boost_limit;
1432 }
1433
1434 static int ni_calculate_adjusted_tdp_limits(struct radeon_device *rdev,
1435                                             bool adjust_polarity,
1436                                             u32 tdp_adjustment,
1437                                             u32 *tdp_limit,
1438                                             u32 *near_tdp_limit)
1439 {
1440         if (tdp_adjustment > (u32)rdev->pm.dpm.tdp_od_limit)
1441                 return -EINVAL;
1442
1443         if (adjust_polarity) {
1444                 *tdp_limit = ((100 + tdp_adjustment) * rdev->pm.dpm.tdp_limit) / 100;
1445                 *near_tdp_limit = rdev->pm.dpm.near_tdp_limit + (*tdp_limit - rdev->pm.dpm.tdp_limit);
1446         } else {
1447                 *tdp_limit = ((100 - tdp_adjustment) * rdev->pm.dpm.tdp_limit) / 100;
1448                 *near_tdp_limit = rdev->pm.dpm.near_tdp_limit - (rdev->pm.dpm.tdp_limit - *tdp_limit);
1449         }
1450
1451         return 0;
1452 }
1453
1454 static int ni_populate_smc_tdp_limits(struct radeon_device *rdev,
1455                                       struct radeon_ps *radeon_state)
1456 {
1457         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1458         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1459
1460         if (ni_pi->enable_power_containment) {
1461                 NISLANDS_SMC_STATETABLE *smc_table = &ni_pi->smc_statetable;
1462                 u32 scaling_factor = ni_get_smc_power_scaling_factor(rdev);
1463                 u32 tdp_limit;
1464                 u32 near_tdp_limit;
1465                 u32 power_boost_limit;
1466                 int ret;
1467
1468                 if (scaling_factor == 0)
1469                         return -EINVAL;
1470
1471                 memset(smc_table, 0, sizeof(NISLANDS_SMC_STATETABLE));
1472
1473                 ret = ni_calculate_adjusted_tdp_limits(rdev,
1474                                                        false, /* ??? */
1475                                                        rdev->pm.dpm.tdp_adjustment,
1476                                                        &tdp_limit,
1477                                                        &near_tdp_limit);
1478                 if (ret)
1479                         return ret;
1480
1481                 power_boost_limit = ni_calculate_power_boost_limit(rdev, radeon_state,
1482                                                                    near_tdp_limit);
1483
1484                 smc_table->dpm2Params.TDPLimit =
1485                         cpu_to_be32(ni_scale_power_for_smc(tdp_limit, scaling_factor));
1486                 smc_table->dpm2Params.NearTDPLimit =
1487                         cpu_to_be32(ni_scale_power_for_smc(near_tdp_limit, scaling_factor));
1488                 smc_table->dpm2Params.SafePowerLimit =
1489                         cpu_to_be32(ni_scale_power_for_smc((near_tdp_limit * NISLANDS_DPM2_TDP_SAFE_LIMIT_PERCENT) / 100,
1490                                                            scaling_factor));
1491                 smc_table->dpm2Params.PowerBoostLimit =
1492                         cpu_to_be32(ni_scale_power_for_smc(power_boost_limit, scaling_factor));
1493
1494                 ret = rv770_copy_bytes_to_smc(rdev,
1495                                               (u16)(pi->state_table_start + offsetof(NISLANDS_SMC_STATETABLE, dpm2Params) +
1496                                                     offsetof(PP_NIslands_DPM2Parameters, TDPLimit)),
1497                                               (u8 *)(&smc_table->dpm2Params.TDPLimit),
1498                                               sizeof(u32) * 4, pi->sram_end);
1499                 if (ret)
1500                         return ret;
1501         }
1502
1503         return 0;
1504 }
1505
1506 int ni_copy_and_switch_arb_sets(struct radeon_device *rdev,
1507                                 u32 arb_freq_src, u32 arb_freq_dest)
1508 {
1509         u32 mc_arb_dram_timing;
1510         u32 mc_arb_dram_timing2;
1511         u32 burst_time;
1512         u32 mc_cg_config;
1513
1514         switch (arb_freq_src) {
1515         case MC_CG_ARB_FREQ_F0:
1516                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING);
1517                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2);
1518                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE0_MASK) >> STATE0_SHIFT;
1519                 break;
1520         case MC_CG_ARB_FREQ_F1:
1521                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_1);
1522                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_1);
1523                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE1_MASK) >> STATE1_SHIFT;
1524                 break;
1525         case MC_CG_ARB_FREQ_F2:
1526                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_2);
1527                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_2);
1528                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE2_MASK) >> STATE2_SHIFT;
1529                 break;
1530         case MC_CG_ARB_FREQ_F3:
1531                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_3);
1532                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_3);
1533                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE3_MASK) >> STATE3_SHIFT;
1534                 break;
1535         default:
1536                 return -EINVAL;
1537         }
1538
1539         switch (arb_freq_dest) {
1540         case MC_CG_ARB_FREQ_F0:
1541                 WREG32(MC_ARB_DRAM_TIMING, mc_arb_dram_timing);
1542                 WREG32(MC_ARB_DRAM_TIMING2, mc_arb_dram_timing2);
1543                 WREG32_P(MC_ARB_BURST_TIME, STATE0(burst_time), ~STATE0_MASK);
1544                 break;
1545         case MC_CG_ARB_FREQ_F1:
1546                 WREG32(MC_ARB_DRAM_TIMING_1, mc_arb_dram_timing);
1547                 WREG32(MC_ARB_DRAM_TIMING2_1, mc_arb_dram_timing2);
1548                 WREG32_P(MC_ARB_BURST_TIME, STATE1(burst_time), ~STATE1_MASK);
1549                 break;
1550         case MC_CG_ARB_FREQ_F2:
1551                 WREG32(MC_ARB_DRAM_TIMING_2, mc_arb_dram_timing);
1552                 WREG32(MC_ARB_DRAM_TIMING2_2, mc_arb_dram_timing2);
1553                 WREG32_P(MC_ARB_BURST_TIME, STATE2(burst_time), ~STATE2_MASK);
1554                 break;
1555         case MC_CG_ARB_FREQ_F3:
1556                 WREG32(MC_ARB_DRAM_TIMING_3, mc_arb_dram_timing);
1557                 WREG32(MC_ARB_DRAM_TIMING2_3, mc_arb_dram_timing2);
1558                 WREG32_P(MC_ARB_BURST_TIME, STATE3(burst_time), ~STATE3_MASK);
1559                 break;
1560         default:
1561                 return -EINVAL;
1562         }
1563
1564         mc_cg_config = RREG32(MC_CG_CONFIG) | 0x0000000F;
1565         WREG32(MC_CG_CONFIG, mc_cg_config);
1566         WREG32_P(MC_ARB_CG, CG_ARB_REQ(arb_freq_dest), ~CG_ARB_REQ_MASK);
1567
1568         return 0;
1569 }
1570
1571 static int ni_init_arb_table_index(struct radeon_device *rdev)
1572 {
1573         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1574         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1575         u32 tmp;
1576         int ret;
1577
1578         ret = rv770_read_smc_sram_dword(rdev, ni_pi->arb_table_start,
1579                                         &tmp, pi->sram_end);
1580         if (ret)
1581                 return ret;
1582
1583         tmp &= 0x00FFFFFF;
1584         tmp |= ((u32)MC_CG_ARB_FREQ_F1) << 24;
1585
1586         return rv770_write_smc_sram_dword(rdev, ni_pi->arb_table_start,
1587                                           tmp, pi->sram_end);
1588 }
1589
1590 static int ni_initial_switch_from_arb_f0_to_f1(struct radeon_device *rdev)
1591 {
1592         return ni_copy_and_switch_arb_sets(rdev, MC_CG_ARB_FREQ_F0, MC_CG_ARB_FREQ_F1);
1593 }
1594
1595 static int ni_force_switch_to_arb_f0(struct radeon_device *rdev)
1596 {
1597         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1598         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1599         u32 tmp;
1600         int ret;
1601
1602         ret = rv770_read_smc_sram_dword(rdev, ni_pi->arb_table_start,
1603                                         &tmp, pi->sram_end);
1604         if (ret)
1605                 return ret;
1606
1607         tmp = (tmp >> 24) & 0xff;
1608
1609         if (tmp == MC_CG_ARB_FREQ_F0)
1610                 return 0;
1611
1612         return ni_copy_and_switch_arb_sets(rdev, tmp, MC_CG_ARB_FREQ_F0);
1613 }
1614
1615 static int ni_populate_memory_timing_parameters(struct radeon_device *rdev,
1616                                                 struct rv7xx_pl *pl,
1617                                                 SMC_NIslands_MCArbDramTimingRegisterSet *arb_regs)
1618 {
1619         u32 dram_timing;
1620         u32 dram_timing2;
1621
1622         arb_regs->mc_arb_rfsh_rate =
1623                 (u8)rv770_calculate_memory_refresh_rate(rdev, pl->sclk);
1624
1625
1626         radeon_atom_set_engine_dram_timings(rdev,
1627                                             pl->sclk,
1628                                             pl->mclk);
1629
1630         dram_timing = RREG32(MC_ARB_DRAM_TIMING);
1631         dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2);
1632
1633         arb_regs->mc_arb_dram_timing  = cpu_to_be32(dram_timing);
1634         arb_regs->mc_arb_dram_timing2 = cpu_to_be32(dram_timing2);
1635
1636         return 0;
1637 }
1638
1639 static int ni_do_program_memory_timing_parameters(struct radeon_device *rdev,
1640                                                   struct radeon_ps *radeon_state,
1641                                                   unsigned int first_arb_set)
1642 {
1643         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1644         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1645         struct ni_ps *state = ni_get_ps(radeon_state);
1646         SMC_NIslands_MCArbDramTimingRegisterSet arb_regs = { 0 };
1647         int i, ret = 0;
1648
1649         for (i = 0; i < state->performance_level_count; i++) {
1650                 ret = ni_populate_memory_timing_parameters(rdev, &state->performance_levels[i], &arb_regs);
1651                 if (ret)
1652                         break;
1653
1654                 ret = rv770_copy_bytes_to_smc(rdev,
1655                                               (u16)(ni_pi->arb_table_start +
1656                                                     offsetof(SMC_NIslands_MCArbDramTimingRegisters, data) +
1657                                                     sizeof(SMC_NIslands_MCArbDramTimingRegisterSet) * (first_arb_set + i)),
1658                                               (u8 *)&arb_regs,
1659                                               (u16)sizeof(SMC_NIslands_MCArbDramTimingRegisterSet),
1660                                               pi->sram_end);
1661                 if (ret)
1662                         break;
1663         }
1664         return ret;
1665 }
1666
1667 static int ni_program_memory_timing_parameters(struct radeon_device *rdev,
1668                                                struct radeon_ps *radeon_new_state)
1669 {
1670         return ni_do_program_memory_timing_parameters(rdev, radeon_new_state,
1671                                                       NISLANDS_DRIVER_STATE_ARB_INDEX);
1672 }
1673
1674 static void ni_populate_initial_mvdd_value(struct radeon_device *rdev,
1675                                            struct NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1676 {
1677         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1678
1679         voltage->index = eg_pi->mvdd_high_index;
1680         voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1681 }
1682
1683 static int ni_populate_smc_initial_state(struct radeon_device *rdev,
1684                                          struct radeon_ps *radeon_initial_state,
1685                                          NISLANDS_SMC_STATETABLE *table)
1686 {
1687         struct ni_ps *initial_state = ni_get_ps(radeon_initial_state);
1688         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1689         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1690         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1691         u32 reg;
1692         int ret;
1693
1694         table->initialState.levels[0].mclk.vMPLL_AD_FUNC_CNTL =
1695                 cpu_to_be32(ni_pi->clock_registers.mpll_ad_func_cntl);
1696         table->initialState.levels[0].mclk.vMPLL_AD_FUNC_CNTL_2 =
1697                 cpu_to_be32(ni_pi->clock_registers.mpll_ad_func_cntl_2);
1698         table->initialState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL =
1699                 cpu_to_be32(ni_pi->clock_registers.mpll_dq_func_cntl);
1700         table->initialState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL_2 =
1701                 cpu_to_be32(ni_pi->clock_registers.mpll_dq_func_cntl_2);
1702         table->initialState.levels[0].mclk.vMCLK_PWRMGT_CNTL =
1703                 cpu_to_be32(ni_pi->clock_registers.mclk_pwrmgt_cntl);
1704         table->initialState.levels[0].mclk.vDLL_CNTL =
1705                 cpu_to_be32(ni_pi->clock_registers.dll_cntl);
1706         table->initialState.levels[0].mclk.vMPLL_SS =
1707                 cpu_to_be32(ni_pi->clock_registers.mpll_ss1);
1708         table->initialState.levels[0].mclk.vMPLL_SS2 =
1709                 cpu_to_be32(ni_pi->clock_registers.mpll_ss2);
1710         table->initialState.levels[0].mclk.mclk_value =
1711                 cpu_to_be32(initial_state->performance_levels[0].mclk);
1712
1713         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL =
1714                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl);
1715         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 =
1716                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_2);
1717         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 =
1718                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_3);
1719         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_4 =
1720                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_4);
1721         table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM =
1722                 cpu_to_be32(ni_pi->clock_registers.cg_spll_spread_spectrum);
1723         table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM_2 =
1724                 cpu_to_be32(ni_pi->clock_registers.cg_spll_spread_spectrum_2);
1725         table->initialState.levels[0].sclk.sclk_value =
1726                 cpu_to_be32(initial_state->performance_levels[0].sclk);
1727         table->initialState.levels[0].arbRefreshState =
1728                 NISLANDS_INITIAL_STATE_ARB_INDEX;
1729
1730         table->initialState.levels[0].ACIndex = 0;
1731
1732         ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
1733                                         initial_state->performance_levels[0].vddc,
1734                                         &table->initialState.levels[0].vddc);
1735         if (!ret) {
1736                 u16 std_vddc;
1737
1738                 ret = ni_get_std_voltage_value(rdev,
1739                                                &table->initialState.levels[0].vddc,
1740                                                &std_vddc);
1741                 if (!ret)
1742                         ni_populate_std_voltage_value(rdev, std_vddc,
1743                                                       table->initialState.levels[0].vddc.index,
1744                                                       &table->initialState.levels[0].std_vddc);
1745         }
1746
1747         if (eg_pi->vddci_control)
1748                 ni_populate_voltage_value(rdev,
1749                                           &eg_pi->vddci_voltage_table,
1750                                           initial_state->performance_levels[0].vddci,
1751                                           &table->initialState.levels[0].vddci);
1752
1753         ni_populate_initial_mvdd_value(rdev, &table->initialState.levels[0].mvdd);
1754
1755         reg = CG_R(0xffff) | CG_L(0);
1756         table->initialState.levels[0].aT = cpu_to_be32(reg);
1757
1758         table->initialState.levels[0].bSP = cpu_to_be32(pi->dsp);
1759
1760         if (pi->boot_in_gen2)
1761                 table->initialState.levels[0].gen2PCIE = 1;
1762         else
1763                 table->initialState.levels[0].gen2PCIE = 0;
1764
1765         if (pi->mem_gddr5) {
1766                 table->initialState.levels[0].strobeMode =
1767                         cypress_get_strobe_mode_settings(rdev,
1768                                                          initial_state->performance_levels[0].mclk);
1769
1770                 if (initial_state->performance_levels[0].mclk > pi->mclk_edc_enable_threshold)
1771                         table->initialState.levels[0].mcFlags = NISLANDS_SMC_MC_EDC_RD_FLAG | NISLANDS_SMC_MC_EDC_WR_FLAG;
1772                 else
1773                         table->initialState.levels[0].mcFlags =  0;
1774         }
1775
1776         table->initialState.levelCount = 1;
1777
1778         table->initialState.flags |= PPSMC_SWSTATE_FLAG_DC;
1779
1780         table->initialState.levels[0].dpm2.MaxPS = 0;
1781         table->initialState.levels[0].dpm2.NearTDPDec = 0;
1782         table->initialState.levels[0].dpm2.AboveSafeInc = 0;
1783         table->initialState.levels[0].dpm2.BelowSafeInc = 0;
1784
1785         reg = MIN_POWER_MASK | MAX_POWER_MASK;
1786         table->initialState.levels[0].SQPowerThrottle = cpu_to_be32(reg);
1787
1788         reg = MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
1789         table->initialState.levels[0].SQPowerThrottle_2 = cpu_to_be32(reg);
1790
1791         return 0;
1792 }
1793
1794 static int ni_populate_smc_acpi_state(struct radeon_device *rdev,
1795                                       NISLANDS_SMC_STATETABLE *table)
1796 {
1797         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1798         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1799         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1800         u32 mpll_ad_func_cntl   = ni_pi->clock_registers.mpll_ad_func_cntl;
1801         u32 mpll_ad_func_cntl_2 = ni_pi->clock_registers.mpll_ad_func_cntl_2;
1802         u32 mpll_dq_func_cntl   = ni_pi->clock_registers.mpll_dq_func_cntl;
1803         u32 mpll_dq_func_cntl_2 = ni_pi->clock_registers.mpll_dq_func_cntl_2;
1804         u32 spll_func_cntl      = ni_pi->clock_registers.cg_spll_func_cntl;
1805         u32 spll_func_cntl_2    = ni_pi->clock_registers.cg_spll_func_cntl_2;
1806         u32 spll_func_cntl_3    = ni_pi->clock_registers.cg_spll_func_cntl_3;
1807         u32 spll_func_cntl_4    = ni_pi->clock_registers.cg_spll_func_cntl_4;
1808         u32 mclk_pwrmgt_cntl    = ni_pi->clock_registers.mclk_pwrmgt_cntl;
1809         u32 dll_cntl            = ni_pi->clock_registers.dll_cntl;
1810         u32 reg;
1811         int ret;
1812
1813         table->ACPIState = table->initialState;
1814
1815         table->ACPIState.flags &= ~PPSMC_SWSTATE_FLAG_DC;
1816
1817         if (pi->acpi_vddc) {
1818                 ret = ni_populate_voltage_value(rdev,
1819                                                 &eg_pi->vddc_voltage_table,
1820                                                 pi->acpi_vddc, &table->ACPIState.levels[0].vddc);
1821                 if (!ret) {
1822                         u16 std_vddc;
1823
1824                         ret = ni_get_std_voltage_value(rdev,
1825                                                        &table->ACPIState.levels[0].vddc, &std_vddc);
1826                         if (!ret)
1827                                 ni_populate_std_voltage_value(rdev, std_vddc,
1828                                                               table->ACPIState.levels[0].vddc.index,
1829                                                               &table->ACPIState.levels[0].std_vddc);
1830                 }
1831
1832                 if (pi->pcie_gen2) {
1833                         if (pi->acpi_pcie_gen2)
1834                                 table->ACPIState.levels[0].gen2PCIE = 1;
1835                         else
1836                                 table->ACPIState.levels[0].gen2PCIE = 0;
1837                 } else {
1838                         table->ACPIState.levels[0].gen2PCIE = 0;
1839                 }
1840         } else {
1841                 ret = ni_populate_voltage_value(rdev,
1842                                                 &eg_pi->vddc_voltage_table,
1843                                                 pi->min_vddc_in_table,
1844                                                 &table->ACPIState.levels[0].vddc);
1845                 if (!ret) {
1846                         u16 std_vddc;
1847
1848                         ret = ni_get_std_voltage_value(rdev,
1849                                                        &table->ACPIState.levels[0].vddc,
1850                                                        &std_vddc);
1851                         if (!ret)
1852                                 ni_populate_std_voltage_value(rdev, std_vddc,
1853                                                               table->ACPIState.levels[0].vddc.index,
1854                                                               &table->ACPIState.levels[0].std_vddc);
1855                 }
1856                 table->ACPIState.levels[0].gen2PCIE = 0;
1857         }
1858
1859         if (eg_pi->acpi_vddci) {
1860                 if (eg_pi->vddci_control)
1861                         ni_populate_voltage_value(rdev,
1862                                                   &eg_pi->vddci_voltage_table,
1863                                                   eg_pi->acpi_vddci,
1864                                                   &table->ACPIState.levels[0].vddci);
1865         }
1866
1867
1868         mpll_ad_func_cntl &= ~PDNB;
1869
1870         mpll_ad_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN;
1871
1872         if (pi->mem_gddr5)
1873                 mpll_dq_func_cntl &= ~PDNB;
1874         mpll_dq_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN | BYPASS;
1875
1876
1877         mclk_pwrmgt_cntl |= (MRDCKA0_RESET |
1878                              MRDCKA1_RESET |
1879                              MRDCKB0_RESET |
1880                              MRDCKB1_RESET |
1881                              MRDCKC0_RESET |
1882                              MRDCKC1_RESET |
1883                              MRDCKD0_RESET |
1884                              MRDCKD1_RESET);
1885
1886         mclk_pwrmgt_cntl &= ~(MRDCKA0_PDNB |
1887                               MRDCKA1_PDNB |
1888                               MRDCKB0_PDNB |
1889                               MRDCKB1_PDNB |
1890                               MRDCKC0_PDNB |
1891                               MRDCKC1_PDNB |
1892                               MRDCKD0_PDNB |
1893                               MRDCKD1_PDNB);
1894
1895         dll_cntl |= (MRDCKA0_BYPASS |
1896                      MRDCKA1_BYPASS |
1897                      MRDCKB0_BYPASS |
1898                      MRDCKB1_BYPASS |
1899                      MRDCKC0_BYPASS |
1900                      MRDCKC1_BYPASS |
1901                      MRDCKD0_BYPASS |
1902                      MRDCKD1_BYPASS);
1903
1904         spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
1905         spll_func_cntl_2 |= SCLK_MUX_SEL(4);
1906
1907         table->ACPIState.levels[0].mclk.vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
1908         table->ACPIState.levels[0].mclk.vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
1909         table->ACPIState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
1910         table->ACPIState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
1911         table->ACPIState.levels[0].mclk.vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
1912         table->ACPIState.levels[0].mclk.vDLL_CNTL = cpu_to_be32(dll_cntl);
1913
1914         table->ACPIState.levels[0].mclk.mclk_value = 0;
1915
1916         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL = cpu_to_be32(spll_func_cntl);
1917         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(spll_func_cntl_2);
1918         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(spll_func_cntl_3);
1919         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_4 = cpu_to_be32(spll_func_cntl_4);
1920
1921         table->ACPIState.levels[0].sclk.sclk_value = 0;
1922
1923         ni_populate_mvdd_value(rdev, 0, &table->ACPIState.levels[0].mvdd);
1924
1925         if (eg_pi->dynamic_ac_timing)
1926                 table->ACPIState.levels[0].ACIndex = 1;
1927
1928         table->ACPIState.levels[0].dpm2.MaxPS = 0;
1929         table->ACPIState.levels[0].dpm2.NearTDPDec = 0;
1930         table->ACPIState.levels[0].dpm2.AboveSafeInc = 0;
1931         table->ACPIState.levels[0].dpm2.BelowSafeInc = 0;
1932
1933         reg = MIN_POWER_MASK | MAX_POWER_MASK;
1934         table->ACPIState.levels[0].SQPowerThrottle = cpu_to_be32(reg);
1935
1936         reg = MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
1937         table->ACPIState.levels[0].SQPowerThrottle_2 = cpu_to_be32(reg);
1938
1939         return 0;
1940 }
1941
1942 static int ni_init_smc_table(struct radeon_device *rdev)
1943 {
1944         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1945         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1946         int ret;
1947         struct radeon_ps *radeon_boot_state = rdev->pm.dpm.boot_ps;
1948         NISLANDS_SMC_STATETABLE *table = &ni_pi->smc_statetable;
1949
1950         memset(table, 0, sizeof(NISLANDS_SMC_STATETABLE));
1951
1952         ni_populate_smc_voltage_tables(rdev, table);
1953
1954         switch (rdev->pm.int_thermal_type) {
1955         case THERMAL_TYPE_NI:
1956         case THERMAL_TYPE_EMC2103_WITH_INTERNAL:
1957                 table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_INTERNAL;
1958                 break;
1959         case THERMAL_TYPE_NONE:
1960                 table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_NONE;
1961                 break;
1962         default:
1963                 table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_EXTERNAL;
1964                 break;
1965         }
1966
1967         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC)
1968                 table->systemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC;
1969
1970         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_REGULATOR_HOT)
1971                 table->systemFlags |= PPSMC_SYSTEMFLAG_REGULATOR_HOT;
1972
1973         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC)
1974                 table->systemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC;
1975
1976         if (pi->mem_gddr5)
1977                 table->systemFlags |= PPSMC_SYSTEMFLAG_GDDR5;
1978
1979         ret = ni_populate_smc_initial_state(rdev, radeon_boot_state, table);
1980         if (ret)
1981                 return ret;
1982
1983         ret = ni_populate_smc_acpi_state(rdev, table);
1984         if (ret)
1985                 return ret;
1986
1987         table->driverState = table->initialState;
1988
1989         table->ULVState = table->initialState;
1990
1991         ret = ni_do_program_memory_timing_parameters(rdev, radeon_boot_state,
1992                                                      NISLANDS_INITIAL_STATE_ARB_INDEX);
1993         if (ret)
1994                 return ret;
1995
1996         return rv770_copy_bytes_to_smc(rdev, pi->state_table_start, (u8 *)table,
1997                                        sizeof(NISLANDS_SMC_STATETABLE), pi->sram_end);
1998 }
1999
2000 static int ni_calculate_sclk_params(struct radeon_device *rdev,
2001                                     u32 engine_clock,
2002                                     NISLANDS_SMC_SCLK_VALUE *sclk)
2003 {
2004         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2005         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2006         struct atom_clock_dividers dividers;
2007         u32 spll_func_cntl = ni_pi->clock_registers.cg_spll_func_cntl;
2008         u32 spll_func_cntl_2 = ni_pi->clock_registers.cg_spll_func_cntl_2;
2009         u32 spll_func_cntl_3 = ni_pi->clock_registers.cg_spll_func_cntl_3;
2010         u32 spll_func_cntl_4 = ni_pi->clock_registers.cg_spll_func_cntl_4;
2011         u32 cg_spll_spread_spectrum = ni_pi->clock_registers.cg_spll_spread_spectrum;
2012         u32 cg_spll_spread_spectrum_2 = ni_pi->clock_registers.cg_spll_spread_spectrum_2;
2013         u64 tmp;
2014         u32 reference_clock = rdev->clock.spll.reference_freq;
2015         u32 reference_divider;
2016         u32 fbdiv;
2017         int ret;
2018
2019         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
2020                                              engine_clock, false, &dividers);
2021         if (ret)
2022                 return ret;
2023
2024         reference_divider = 1 + dividers.ref_div;
2025
2026
2027         tmp = (u64) engine_clock * reference_divider * dividers.post_div * 16834;
2028         do_div(tmp, reference_clock);
2029         fbdiv = (u32) tmp;
2030
2031         spll_func_cntl &= ~(SPLL_PDIV_A_MASK | SPLL_REF_DIV_MASK);
2032         spll_func_cntl |= SPLL_REF_DIV(dividers.ref_div);
2033         spll_func_cntl |= SPLL_PDIV_A(dividers.post_div);
2034
2035         spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
2036         spll_func_cntl_2 |= SCLK_MUX_SEL(2);
2037
2038         spll_func_cntl_3 &= ~SPLL_FB_DIV_MASK;
2039         spll_func_cntl_3 |= SPLL_FB_DIV(fbdiv);
2040         spll_func_cntl_3 |= SPLL_DITHEN;
2041
2042         if (pi->sclk_ss) {
2043                 struct radeon_atom_ss ss;
2044                 u32 vco_freq = engine_clock * dividers.post_div;
2045
2046                 if (radeon_atombios_get_asic_ss_info(rdev, &ss,
2047                                                      ASIC_INTERNAL_ENGINE_SS, vco_freq)) {
2048                         u32 clk_s = reference_clock * 5 / (reference_divider * ss.rate);
2049                         u32 clk_v = 4 * ss.percentage * fbdiv / (clk_s * 10000);
2050
2051                         cg_spll_spread_spectrum &= ~CLK_S_MASK;
2052                         cg_spll_spread_spectrum |= CLK_S(clk_s);
2053                         cg_spll_spread_spectrum |= SSEN;
2054
2055                         cg_spll_spread_spectrum_2 &= ~CLK_V_MASK;
2056                         cg_spll_spread_spectrum_2 |= CLK_V(clk_v);
2057                 }
2058         }
2059
2060         sclk->sclk_value = engine_clock;
2061         sclk->vCG_SPLL_FUNC_CNTL = spll_func_cntl;
2062         sclk->vCG_SPLL_FUNC_CNTL_2 = spll_func_cntl_2;
2063         sclk->vCG_SPLL_FUNC_CNTL_3 = spll_func_cntl_3;
2064         sclk->vCG_SPLL_FUNC_CNTL_4 = spll_func_cntl_4;
2065         sclk->vCG_SPLL_SPREAD_SPECTRUM = cg_spll_spread_spectrum;
2066         sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cg_spll_spread_spectrum_2;
2067
2068         return 0;
2069 }
2070
2071 static int ni_populate_sclk_value(struct radeon_device *rdev,
2072                                   u32 engine_clock,
2073                                   NISLANDS_SMC_SCLK_VALUE *sclk)
2074 {
2075         NISLANDS_SMC_SCLK_VALUE sclk_tmp;
2076         int ret;
2077
2078         ret = ni_calculate_sclk_params(rdev, engine_clock, &sclk_tmp);
2079         if (!ret) {
2080                 sclk->sclk_value = cpu_to_be32(sclk_tmp.sclk_value);
2081                 sclk->vCG_SPLL_FUNC_CNTL = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL);
2082                 sclk->vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_2);
2083                 sclk->vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_3);
2084                 sclk->vCG_SPLL_FUNC_CNTL_4 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_4);
2085                 sclk->vCG_SPLL_SPREAD_SPECTRUM = cpu_to_be32(sclk_tmp.vCG_SPLL_SPREAD_SPECTRUM);
2086                 sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cpu_to_be32(sclk_tmp.vCG_SPLL_SPREAD_SPECTRUM_2);
2087         }
2088
2089         return ret;
2090 }
2091
2092 static int ni_init_smc_spll_table(struct radeon_device *rdev)
2093 {
2094         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2095         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2096         SMC_NISLANDS_SPLL_DIV_TABLE *spll_table;
2097         NISLANDS_SMC_SCLK_VALUE sclk_params;
2098         u32 fb_div;
2099         u32 p_div;
2100         u32 clk_s;
2101         u32 clk_v;
2102         u32 sclk = 0;
2103         int i, ret;
2104         u32 tmp;
2105
2106         if (ni_pi->spll_table_start == 0)
2107                 return -EINVAL;
2108
2109         spll_table = kzalloc(sizeof(SMC_NISLANDS_SPLL_DIV_TABLE), GFP_KERNEL);
2110         if (spll_table == NULL)
2111                 return -ENOMEM;
2112
2113         for (i = 0; i < 256; i++) {
2114                 ret = ni_calculate_sclk_params(rdev, sclk, &sclk_params);
2115                 if (ret)
2116                         break;
2117
2118                 p_div = (sclk_params.vCG_SPLL_FUNC_CNTL & SPLL_PDIV_A_MASK) >> SPLL_PDIV_A_SHIFT;
2119                 fb_div = (sclk_params.vCG_SPLL_FUNC_CNTL_3 & SPLL_FB_DIV_MASK) >> SPLL_FB_DIV_SHIFT;
2120                 clk_s = (sclk_params.vCG_SPLL_SPREAD_SPECTRUM & CLK_S_MASK) >> CLK_S_SHIFT;
2121                 clk_v = (sclk_params.vCG_SPLL_SPREAD_SPECTRUM_2 & CLK_V_MASK) >> CLK_V_SHIFT;
2122
2123                 fb_div &= ~0x00001FFF;
2124                 fb_div >>= 1;
2125                 clk_v >>= 6;
2126
2127                 if (p_div & ~(SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT))
2128                         ret = -EINVAL;
2129
2130                 if (clk_s & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT))
2131                         ret = -EINVAL;
2132
2133                 if (clk_s & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT))
2134                         ret = -EINVAL;
2135
2136                 if (clk_v & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT))
2137                         ret = -EINVAL;
2138
2139                 if (ret)
2140                         break;
2141
2142                 tmp = ((fb_div << SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_MASK) |
2143                         ((p_div << SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_MASK);
2144                 spll_table->freq[i] = cpu_to_be32(tmp);
2145
2146                 tmp = ((clk_v << SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_MASK) |
2147                         ((clk_s << SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK);
2148                 spll_table->ss[i] = cpu_to_be32(tmp);
2149
2150                 sclk += 512;
2151         }
2152
2153         if (!ret)
2154                 ret = rv770_copy_bytes_to_smc(rdev, ni_pi->spll_table_start, (u8 *)spll_table,
2155                                               sizeof(SMC_NISLANDS_SPLL_DIV_TABLE), pi->sram_end);
2156
2157         kfree(spll_table);
2158
2159         return ret;
2160 }
2161
2162 static int ni_populate_mclk_value(struct radeon_device *rdev,
2163                                   u32 engine_clock,
2164                                   u32 memory_clock,
2165                                   NISLANDS_SMC_MCLK_VALUE *mclk,
2166                                   bool strobe_mode,
2167                                   bool dll_state_on)
2168 {
2169         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2170         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2171         u32 mpll_ad_func_cntl = ni_pi->clock_registers.mpll_ad_func_cntl;
2172         u32 mpll_ad_func_cntl_2 = ni_pi->clock_registers.mpll_ad_func_cntl_2;
2173         u32 mpll_dq_func_cntl = ni_pi->clock_registers.mpll_dq_func_cntl;
2174         u32 mpll_dq_func_cntl_2 = ni_pi->clock_registers.mpll_dq_func_cntl_2;
2175         u32 mclk_pwrmgt_cntl = ni_pi->clock_registers.mclk_pwrmgt_cntl;
2176         u32 dll_cntl = ni_pi->clock_registers.dll_cntl;
2177         u32 mpll_ss1 = ni_pi->clock_registers.mpll_ss1;
2178         u32 mpll_ss2 = ni_pi->clock_registers.mpll_ss2;
2179         struct atom_clock_dividers dividers;
2180         u32 ibias;
2181         u32 dll_speed;
2182         int ret;
2183         u32 mc_seq_misc7;
2184
2185         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_MEMORY_PLL_PARAM,
2186                                              memory_clock, strobe_mode, &dividers);
2187         if (ret)
2188                 return ret;
2189
2190         if (!strobe_mode) {
2191                 mc_seq_misc7 = RREG32(MC_SEQ_MISC7);
2192
2193                 if (mc_seq_misc7 & 0x8000000)
2194                         dividers.post_div = 1;
2195         }
2196
2197         ibias = cypress_map_clkf_to_ibias(rdev, dividers.whole_fb_div);
2198
2199         mpll_ad_func_cntl &= ~(CLKR_MASK |
2200                                YCLK_POST_DIV_MASK |
2201                                CLKF_MASK |
2202                                CLKFRAC_MASK |
2203                                IBIAS_MASK);
2204         mpll_ad_func_cntl |= CLKR(dividers.ref_div);
2205         mpll_ad_func_cntl |= YCLK_POST_DIV(dividers.post_div);
2206         mpll_ad_func_cntl |= CLKF(dividers.whole_fb_div);
2207         mpll_ad_func_cntl |= CLKFRAC(dividers.frac_fb_div);
2208         mpll_ad_func_cntl |= IBIAS(ibias);
2209
2210         if (dividers.vco_mode)
2211                 mpll_ad_func_cntl_2 |= VCO_MODE;
2212         else
2213                 mpll_ad_func_cntl_2 &= ~VCO_MODE;
2214
2215         if (pi->mem_gddr5) {
2216                 mpll_dq_func_cntl &= ~(CLKR_MASK |
2217                                        YCLK_POST_DIV_MASK |
2218                                        CLKF_MASK |
2219                                        CLKFRAC_MASK |
2220                                        IBIAS_MASK);
2221                 mpll_dq_func_cntl |= CLKR(dividers.ref_div);
2222                 mpll_dq_func_cntl |= YCLK_POST_DIV(dividers.post_div);
2223                 mpll_dq_func_cntl |= CLKF(dividers.whole_fb_div);
2224                 mpll_dq_func_cntl |= CLKFRAC(dividers.frac_fb_div);
2225                 mpll_dq_func_cntl |= IBIAS(ibias);
2226
2227                 if (strobe_mode)
2228                         mpll_dq_func_cntl &= ~PDNB;
2229                 else
2230                         mpll_dq_func_cntl |= PDNB;
2231
2232                 if (dividers.vco_mode)
2233                         mpll_dq_func_cntl_2 |= VCO_MODE;
2234                 else
2235                         mpll_dq_func_cntl_2 &= ~VCO_MODE;
2236         }
2237
2238         if (pi->mclk_ss) {
2239                 struct radeon_atom_ss ss;
2240                 u32 vco_freq = memory_clock * dividers.post_div;
2241
2242                 if (radeon_atombios_get_asic_ss_info(rdev, &ss,
2243                                                      ASIC_INTERNAL_MEMORY_SS, vco_freq)) {
2244                         u32 reference_clock = rdev->clock.mpll.reference_freq;
2245                         u32 decoded_ref = rv740_get_decoded_reference_divider(dividers.ref_div);
2246                         u32 clk_s = reference_clock * 5 / (decoded_ref * ss.rate);
2247                         u32 clk_v = ss.percentage *
2248                                 (0x4000 * dividers.whole_fb_div + 0x800 * dividers.frac_fb_div) / (clk_s * 625);
2249
2250                         mpll_ss1 &= ~CLKV_MASK;
2251                         mpll_ss1 |= CLKV(clk_v);
2252
2253                         mpll_ss2 &= ~CLKS_MASK;
2254                         mpll_ss2 |= CLKS(clk_s);
2255                 }
2256         }
2257
2258         dll_speed = rv740_get_dll_speed(pi->mem_gddr5,
2259                                         memory_clock);
2260
2261         mclk_pwrmgt_cntl &= ~DLL_SPEED_MASK;
2262         mclk_pwrmgt_cntl |= DLL_SPEED(dll_speed);
2263         if (dll_state_on)
2264                 mclk_pwrmgt_cntl |= (MRDCKA0_PDNB |
2265                                      MRDCKA1_PDNB |
2266                                      MRDCKB0_PDNB |
2267                                      MRDCKB1_PDNB |
2268                                      MRDCKC0_PDNB |
2269                                      MRDCKC1_PDNB |
2270                                      MRDCKD0_PDNB |
2271                                      MRDCKD1_PDNB);
2272         else
2273                 mclk_pwrmgt_cntl &= ~(MRDCKA0_PDNB |
2274                                       MRDCKA1_PDNB |
2275                                       MRDCKB0_PDNB |
2276                                       MRDCKB1_PDNB |
2277                                       MRDCKC0_PDNB |
2278                                       MRDCKC1_PDNB |
2279                                       MRDCKD0_PDNB |
2280                                       MRDCKD1_PDNB);
2281
2282
2283         mclk->mclk_value = cpu_to_be32(memory_clock);
2284         mclk->vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
2285         mclk->vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
2286         mclk->vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
2287         mclk->vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
2288         mclk->vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
2289         mclk->vDLL_CNTL = cpu_to_be32(dll_cntl);
2290         mclk->vMPLL_SS = cpu_to_be32(mpll_ss1);
2291         mclk->vMPLL_SS2 = cpu_to_be32(mpll_ss2);
2292
2293         return 0;
2294 }
2295
2296 static void ni_populate_smc_sp(struct radeon_device *rdev,
2297                                struct radeon_ps *radeon_state,
2298                                NISLANDS_SMC_SWSTATE *smc_state)
2299 {
2300         struct ni_ps *ps = ni_get_ps(radeon_state);
2301         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2302         int i;
2303
2304         for (i = 0; i < ps->performance_level_count - 1; i++)
2305                 smc_state->levels[i].bSP = cpu_to_be32(pi->dsp);
2306
2307         smc_state->levels[ps->performance_level_count - 1].bSP =
2308                 cpu_to_be32(pi->psp);
2309 }
2310
2311 static int ni_convert_power_level_to_smc(struct radeon_device *rdev,
2312                                          struct rv7xx_pl *pl,
2313                                          NISLANDS_SMC_HW_PERFORMANCE_LEVEL *level)
2314 {
2315         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2316         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2317         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2318         int ret;
2319         bool dll_state_on;
2320         u16 std_vddc;
2321         u32 tmp = RREG32(DC_STUTTER_CNTL);
2322
2323         level->gen2PCIE = pi->pcie_gen2 ?
2324                 ((pl->flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? 1 : 0) : 0;
2325
2326         ret = ni_populate_sclk_value(rdev, pl->sclk, &level->sclk);
2327         if (ret)
2328                 return ret;
2329
2330         level->mcFlags =  0;
2331         if (pi->mclk_stutter_mode_threshold &&
2332             (pl->mclk <= pi->mclk_stutter_mode_threshold) &&
2333             !eg_pi->uvd_enabled &&
2334             (tmp & DC_STUTTER_ENABLE_A) &&
2335             (tmp & DC_STUTTER_ENABLE_B))
2336                 level->mcFlags |= NISLANDS_SMC_MC_STUTTER_EN;
2337
2338         if (pi->mem_gddr5) {
2339                 if (pl->mclk > pi->mclk_edc_enable_threshold)
2340                         level->mcFlags |= NISLANDS_SMC_MC_EDC_RD_FLAG;
2341                 if (pl->mclk > eg_pi->mclk_edc_wr_enable_threshold)
2342                         level->mcFlags |= NISLANDS_SMC_MC_EDC_WR_FLAG;
2343
2344                 level->strobeMode = cypress_get_strobe_mode_settings(rdev, pl->mclk);
2345
2346                 if (level->strobeMode & NISLANDS_SMC_STROBE_ENABLE) {
2347                         if (cypress_get_mclk_frequency_ratio(rdev, pl->mclk, true) >=
2348                             ((RREG32(MC_SEQ_MISC7) >> 16) & 0xf))
2349                                 dll_state_on = ((RREG32(MC_SEQ_MISC5) >> 1) & 0x1) ? true : false;
2350                         else
2351                                 dll_state_on = ((RREG32(MC_SEQ_MISC6) >> 1) & 0x1) ? true : false;
2352                 } else {
2353                         dll_state_on = false;
2354                         if (pl->mclk > ni_pi->mclk_rtt_mode_threshold)
2355                                 level->mcFlags |= NISLANDS_SMC_MC_RTT_ENABLE;
2356                 }
2357
2358                 ret = ni_populate_mclk_value(rdev, pl->sclk, pl->mclk,
2359                                              &level->mclk,
2360                                              (level->strobeMode & NISLANDS_SMC_STROBE_ENABLE) != 0,
2361                                              dll_state_on);
2362         } else
2363                 ret = ni_populate_mclk_value(rdev, pl->sclk, pl->mclk, &level->mclk, 1, 1);
2364
2365         if (ret)
2366                 return ret;
2367
2368         ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
2369                                         pl->vddc, &level->vddc);
2370         if (ret)
2371                 return ret;
2372
2373         ret = ni_get_std_voltage_value(rdev, &level->vddc, &std_vddc);
2374         if (ret)
2375                 return ret;
2376
2377         ni_populate_std_voltage_value(rdev, std_vddc,
2378                                       level->vddc.index, &level->std_vddc);
2379
2380         if (eg_pi->vddci_control) {
2381                 ret = ni_populate_voltage_value(rdev, &eg_pi->vddci_voltage_table,
2382                                                 pl->vddci, &level->vddci);
2383                 if (ret)
2384                         return ret;
2385         }
2386
2387         ni_populate_mvdd_value(rdev, pl->mclk, &level->mvdd);
2388
2389         return ret;
2390 }
2391
2392 static int ni_populate_smc_t(struct radeon_device *rdev,
2393                              struct radeon_ps *radeon_state,
2394                              NISLANDS_SMC_SWSTATE *smc_state)
2395 {
2396         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2397         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2398         struct ni_ps *state = ni_get_ps(radeon_state);
2399         u32 a_t;
2400         u32 t_l, t_h;
2401         u32 high_bsp;
2402         int i, ret;
2403
2404         if (state->performance_level_count >= 9)
2405                 return -EINVAL;
2406
2407         if (state->performance_level_count < 2) {
2408                 a_t = CG_R(0xffff) | CG_L(0);
2409                 smc_state->levels[0].aT = cpu_to_be32(a_t);
2410                 return 0;
2411         }
2412
2413         smc_state->levels[0].aT = cpu_to_be32(0);
2414
2415         for (i = 0; i <= state->performance_level_count - 2; i++) {
2416                 if (eg_pi->uvd_enabled)
2417                         ret = r600_calculate_at(
2418                                 1000 * (i * (eg_pi->smu_uvd_hs ? 2 : 8) + 2),
2419                                 100 * R600_AH_DFLT,
2420                                 state->performance_levels[i + 1].sclk,
2421                                 state->performance_levels[i].sclk,
2422                                 &t_l,
2423                                 &t_h);
2424                 else
2425                         ret = r600_calculate_at(
2426                                 1000 * (i + 1),
2427                                 100 * R600_AH_DFLT,
2428                                 state->performance_levels[i + 1].sclk,
2429                                 state->performance_levels[i].sclk,
2430                                 &t_l,
2431                                 &t_h);
2432
2433                 if (ret) {
2434                         t_h = (i + 1) * 1000 - 50 * R600_AH_DFLT;
2435                         t_l = (i + 1) * 1000 + 50 * R600_AH_DFLT;
2436                 }
2437
2438                 a_t = be32_to_cpu(smc_state->levels[i].aT) & ~CG_R_MASK;
2439                 a_t |= CG_R(t_l * pi->bsp / 20000);
2440                 smc_state->levels[i].aT = cpu_to_be32(a_t);
2441
2442                 high_bsp = (i == state->performance_level_count - 2) ?
2443                         pi->pbsp : pi->bsp;
2444
2445                 a_t = CG_R(0xffff) | CG_L(t_h * high_bsp / 20000);
2446                 smc_state->levels[i + 1].aT = cpu_to_be32(a_t);
2447         }
2448
2449         return 0;
2450 }
2451
2452 static int ni_populate_power_containment_values(struct radeon_device *rdev,
2453                                                 struct radeon_ps *radeon_state,
2454                                                 NISLANDS_SMC_SWSTATE *smc_state)
2455 {
2456         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2457         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2458         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2459         struct ni_ps *state = ni_get_ps(radeon_state);
2460         u32 prev_sclk;
2461         u32 max_sclk;
2462         u32 min_sclk;
2463         int i, ret;
2464         u32 tdp_limit;
2465         u32 near_tdp_limit;
2466         u32 power_boost_limit;
2467         u8 max_ps_percent;
2468
2469         if (ni_pi->enable_power_containment == false)
2470                 return 0;
2471
2472         if (state->performance_level_count == 0)
2473                 return -EINVAL;
2474
2475         if (smc_state->levelCount != state->performance_level_count)
2476                 return -EINVAL;
2477
2478         ret = ni_calculate_adjusted_tdp_limits(rdev,
2479                                                false, /* ??? */
2480                                                rdev->pm.dpm.tdp_adjustment,
2481                                                &tdp_limit,
2482                                                &near_tdp_limit);
2483         if (ret)
2484                 return ret;
2485
2486         power_boost_limit = ni_calculate_power_boost_limit(rdev, radeon_state, near_tdp_limit);
2487
2488         ret = rv770_write_smc_sram_dword(rdev,
2489                                          pi->state_table_start +
2490                                          offsetof(NISLANDS_SMC_STATETABLE, dpm2Params) +
2491                                          offsetof(PP_NIslands_DPM2Parameters, PowerBoostLimit),
2492                                          ni_scale_power_for_smc(power_boost_limit, ni_get_smc_power_scaling_factor(rdev)),
2493                                          pi->sram_end);
2494         if (ret)
2495                 power_boost_limit = 0;
2496
2497         smc_state->levels[0].dpm2.MaxPS = 0;
2498         smc_state->levels[0].dpm2.NearTDPDec = 0;
2499         smc_state->levels[0].dpm2.AboveSafeInc = 0;
2500         smc_state->levels[0].dpm2.BelowSafeInc = 0;
2501         smc_state->levels[0].stateFlags |= power_boost_limit ? PPSMC_STATEFLAG_POWERBOOST : 0;
2502
2503         for (i = 1; i < state->performance_level_count; i++) {
2504                 prev_sclk = state->performance_levels[i-1].sclk;
2505                 max_sclk  = state->performance_levels[i].sclk;
2506                 max_ps_percent = (i != (state->performance_level_count - 1)) ?
2507                         NISLANDS_DPM2_MAXPS_PERCENT_M : NISLANDS_DPM2_MAXPS_PERCENT_H;
2508
2509                 if (max_sclk < prev_sclk)
2510                         return -EINVAL;
2511
2512                 if ((max_ps_percent == 0) || (prev_sclk == max_sclk) || eg_pi->uvd_enabled)
2513                         min_sclk = max_sclk;
2514                 else if (1 == i)
2515                         min_sclk = prev_sclk;
2516                 else
2517                         min_sclk = (prev_sclk * (u32)max_ps_percent) / 100;
2518
2519                 if (min_sclk < state->performance_levels[0].sclk)
2520                         min_sclk = state->performance_levels[0].sclk;
2521
2522                 if (min_sclk == 0)
2523                         return -EINVAL;
2524
2525                 smc_state->levels[i].dpm2.MaxPS =
2526                         (u8)((NISLANDS_DPM2_MAX_PULSE_SKIP * (max_sclk - min_sclk)) / max_sclk);
2527                 smc_state->levels[i].dpm2.NearTDPDec = NISLANDS_DPM2_NEAR_TDP_DEC;
2528                 smc_state->levels[i].dpm2.AboveSafeInc = NISLANDS_DPM2_ABOVE_SAFE_INC;
2529                 smc_state->levels[i].dpm2.BelowSafeInc = NISLANDS_DPM2_BELOW_SAFE_INC;
2530                 smc_state->levels[i].stateFlags |=
2531                         ((i != (state->performance_level_count - 1)) && power_boost_limit) ?
2532                         PPSMC_STATEFLAG_POWERBOOST : 0;
2533         }
2534
2535         return 0;
2536 }
2537
2538 static int ni_populate_sq_ramping_values(struct radeon_device *rdev,
2539                                          struct radeon_ps *radeon_state,
2540                                          NISLANDS_SMC_SWSTATE *smc_state)
2541 {
2542         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2543         struct ni_ps *state = ni_get_ps(radeon_state);
2544         u32 sq_power_throttle;
2545         u32 sq_power_throttle2;
2546         bool enable_sq_ramping = ni_pi->enable_sq_ramping;
2547         int i;
2548
2549         if (state->performance_level_count == 0)
2550                 return -EINVAL;
2551
2552         if (smc_state->levelCount != state->performance_level_count)
2553                 return -EINVAL;
2554
2555         if (rdev->pm.dpm.sq_ramping_threshold == 0)
2556                 return -EINVAL;
2557
2558         if (NISLANDS_DPM2_SQ_RAMP_MAX_POWER > (MAX_POWER_MASK >> MAX_POWER_SHIFT))
2559                 enable_sq_ramping = false;
2560
2561         if (NISLANDS_DPM2_SQ_RAMP_MIN_POWER > (MIN_POWER_MASK >> MIN_POWER_SHIFT))
2562                 enable_sq_ramping = false;
2563
2564         if (NISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA > (MAX_POWER_DELTA_MASK >> MAX_POWER_DELTA_SHIFT))
2565                 enable_sq_ramping = false;
2566
2567         if (NISLANDS_DPM2_SQ_RAMP_STI_SIZE > (STI_SIZE_MASK >> STI_SIZE_SHIFT))
2568                 enable_sq_ramping = false;
2569
2570         if (NISLANDS_DPM2_SQ_RAMP_LTI_RATIO <= (LTI_RATIO_MASK >> LTI_RATIO_SHIFT))
2571                 enable_sq_ramping = false;
2572
2573         for (i = 0; i < state->performance_level_count; i++) {
2574                 sq_power_throttle  = 0;
2575                 sq_power_throttle2 = 0;
2576
2577                 if ((state->performance_levels[i].sclk >= rdev->pm.dpm.sq_ramping_threshold) &&
2578                     enable_sq_ramping) {
2579                         sq_power_throttle |= MAX_POWER(NISLANDS_DPM2_SQ_RAMP_MAX_POWER);
2580                         sq_power_throttle |= MIN_POWER(NISLANDS_DPM2_SQ_RAMP_MIN_POWER);
2581                         sq_power_throttle2 |= MAX_POWER_DELTA(NISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA);
2582                         sq_power_throttle2 |= STI_SIZE(NISLANDS_DPM2_SQ_RAMP_STI_SIZE);
2583                         sq_power_throttle2 |= LTI_RATIO(NISLANDS_DPM2_SQ_RAMP_LTI_RATIO);
2584                 } else {
2585                         sq_power_throttle |= MAX_POWER_MASK | MIN_POWER_MASK;
2586                         sq_power_throttle2 |= MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
2587                 }
2588
2589                 smc_state->levels[i].SQPowerThrottle   = cpu_to_be32(sq_power_throttle);
2590                 smc_state->levels[i].SQPowerThrottle_2 = cpu_to_be32(sq_power_throttle2);
2591         }
2592
2593         return 0;
2594 }
2595
2596 static int ni_enable_power_containment(struct radeon_device *rdev,
2597                                        struct radeon_ps *radeon_new_state,
2598                                        bool enable)
2599 {
2600         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2601         PPSMC_Result smc_result;
2602         int ret = 0;
2603
2604         if (ni_pi->enable_power_containment) {
2605                 if (enable) {
2606                         if (!r600_is_uvd_state(radeon_new_state->class, radeon_new_state->class2)) {
2607                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_TDPClampingActive);
2608                                 if (smc_result != PPSMC_Result_OK) {
2609                                         ret = -EINVAL;
2610                                         ni_pi->pc_enabled = false;
2611                                 } else {
2612                                         ni_pi->pc_enabled = true;
2613                                 }
2614                         }
2615                 } else {
2616                         smc_result = rv770_send_msg_to_smc(rdev, PPSMC_TDPClampingInactive);
2617                         if (smc_result != PPSMC_Result_OK)
2618                                 ret = -EINVAL;
2619                         ni_pi->pc_enabled = false;
2620                 }
2621         }
2622
2623         return ret;
2624 }
2625
2626 static int ni_convert_power_state_to_smc(struct radeon_device *rdev,
2627                                          struct radeon_ps *radeon_state,
2628                                          NISLANDS_SMC_SWSTATE *smc_state)
2629 {
2630         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2631         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2632         struct ni_ps *state = ni_get_ps(radeon_state);
2633         int i, ret;
2634         u32 threshold = state->performance_levels[state->performance_level_count - 1].sclk * 100 / 100;
2635
2636         if (!(radeon_state->caps & ATOM_PPLIB_DISALLOW_ON_DC))
2637                 smc_state->flags |= PPSMC_SWSTATE_FLAG_DC;
2638
2639         smc_state->levelCount = 0;
2640
2641         if (state->performance_level_count > NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE)
2642                 return -EINVAL;
2643
2644         for (i = 0; i < state->performance_level_count; i++) {
2645                 ret = ni_convert_power_level_to_smc(rdev, &state->performance_levels[i],
2646                                                     &smc_state->levels[i]);
2647                 smc_state->levels[i].arbRefreshState =
2648                         (u8)(NISLANDS_DRIVER_STATE_ARB_INDEX + i);
2649
2650                 if (ret)
2651                         return ret;
2652
2653                 if (ni_pi->enable_power_containment)
2654                         smc_state->levels[i].displayWatermark =
2655                                 (state->performance_levels[i].sclk < threshold) ?
2656                                 PPSMC_DISPLAY_WATERMARK_LOW : PPSMC_DISPLAY_WATERMARK_HIGH;
2657                 else
2658                         smc_state->levels[i].displayWatermark = (i < 2) ?
2659                                 PPSMC_DISPLAY_WATERMARK_LOW : PPSMC_DISPLAY_WATERMARK_HIGH;
2660
2661                 if (eg_pi->dynamic_ac_timing)
2662                         smc_state->levels[i].ACIndex = NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT + i;
2663                 else
2664                         smc_state->levels[i].ACIndex = 0;
2665
2666                 smc_state->levelCount++;
2667         }
2668
2669         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_watermark_threshold,
2670                                       cpu_to_be32(threshold / 512));
2671
2672         ni_populate_smc_sp(rdev, radeon_state, smc_state);
2673
2674         ret = ni_populate_power_containment_values(rdev, radeon_state, smc_state);
2675         if (ret)
2676                 ni_pi->enable_power_containment = false;
2677
2678         ret = ni_populate_sq_ramping_values(rdev, radeon_state, smc_state);
2679         if (ret)
2680                 ni_pi->enable_sq_ramping = false;
2681
2682         return ni_populate_smc_t(rdev, radeon_state, smc_state);
2683 }
2684
2685 static int ni_upload_sw_state(struct radeon_device *rdev,
2686                               struct radeon_ps *radeon_new_state)
2687 {
2688         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2689         u16 address = pi->state_table_start +
2690                 offsetof(NISLANDS_SMC_STATETABLE, driverState);
2691         u16 state_size = sizeof(NISLANDS_SMC_SWSTATE) +
2692                 ((NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1) * sizeof(NISLANDS_SMC_HW_PERFORMANCE_LEVEL));
2693         int ret;
2694         NISLANDS_SMC_SWSTATE *smc_state = kzalloc(state_size, GFP_KERNEL);
2695
2696         if (smc_state == NULL)
2697                 return -ENOMEM;
2698
2699         ret = ni_convert_power_state_to_smc(rdev, radeon_new_state, smc_state);
2700         if (ret)
2701                 goto done;
2702
2703         ret = rv770_copy_bytes_to_smc(rdev, address, (u8 *)smc_state, state_size, pi->sram_end);
2704
2705 done:
2706         kfree(smc_state);
2707
2708         return ret;
2709 }
2710
2711 static int ni_set_mc_special_registers(struct radeon_device *rdev,
2712                                        struct ni_mc_reg_table *table)
2713 {
2714         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2715         u8 i, j, k;
2716         u32 temp_reg;
2717
2718         for (i = 0, j = table->last; i < table->last; i++) {
2719                 switch (table->mc_reg_address[i].s1) {
2720                 case MC_SEQ_MISC1 >> 2:
2721                         if (j >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2722                                 return -EINVAL;
2723                         temp_reg = RREG32(MC_PMG_CMD_EMRS);
2724                         table->mc_reg_address[j].s1 = MC_PMG_CMD_EMRS >> 2;
2725                         table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_EMRS_LP >> 2;
2726                         for (k = 0; k < table->num_entries; k++)
2727                                 table->mc_reg_table_entry[k].mc_data[j] =
2728                                         ((temp_reg & 0xffff0000)) |
2729                                         ((table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16);
2730                         j++;
2731                         if (j >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2732                                 return -EINVAL;
2733
2734                         temp_reg = RREG32(MC_PMG_CMD_MRS);
2735                         table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS >> 2;
2736                         table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS_LP >> 2;
2737                         for(k = 0; k < table->num_entries; k++) {
2738                                 table->mc_reg_table_entry[k].mc_data[j] =
2739                                         (temp_reg & 0xffff0000) |
2740                                         (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
2741                                 if (!pi->mem_gddr5)
2742                                         table->mc_reg_table_entry[k].mc_data[j] |= 0x100;
2743                         }
2744                         j++;
2745                         if (j > SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2746                                 return -EINVAL;
2747                         break;
2748                 case MC_SEQ_RESERVE_M >> 2:
2749                         temp_reg = RREG32(MC_PMG_CMD_MRS1);
2750                         table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS1 >> 2;
2751                         table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS1_LP >> 2;
2752                         for (k = 0; k < table->num_entries; k++)
2753                                 table->mc_reg_table_entry[k].mc_data[j] =
2754                                         (temp_reg & 0xffff0000) |
2755                                         (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
2756                         j++;
2757                         if (j > SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2758                                 return -EINVAL;
2759                         break;
2760                 default:
2761                         break;
2762                 }
2763         }
2764
2765         table->last = j;
2766
2767         return 0;
2768 }
2769
2770 static bool ni_check_s0_mc_reg_index(u16 in_reg, u16 *out_reg)
2771 {
2772         bool result = true;
2773
2774         switch (in_reg) {
2775         case  MC_SEQ_RAS_TIMING >> 2:
2776                 *out_reg = MC_SEQ_RAS_TIMING_LP >> 2;
2777                 break;
2778         case MC_SEQ_CAS_TIMING >> 2:
2779                 *out_reg = MC_SEQ_CAS_TIMING_LP >> 2;
2780                 break;
2781         case MC_SEQ_MISC_TIMING >> 2:
2782                 *out_reg = MC_SEQ_MISC_TIMING_LP >> 2;
2783                 break;
2784         case MC_SEQ_MISC_TIMING2 >> 2:
2785                 *out_reg = MC_SEQ_MISC_TIMING2_LP >> 2;
2786                 break;
2787         case MC_SEQ_RD_CTL_D0 >> 2:
2788                 *out_reg = MC_SEQ_RD_CTL_D0_LP >> 2;
2789                 break;
2790         case MC_SEQ_RD_CTL_D1 >> 2:
2791                 *out_reg = MC_SEQ_RD_CTL_D1_LP >> 2;
2792                 break;
2793         case MC_SEQ_WR_CTL_D0 >> 2:
2794                 *out_reg = MC_SEQ_WR_CTL_D0_LP >> 2;
2795                 break;
2796         case MC_SEQ_WR_CTL_D1 >> 2:
2797                 *out_reg = MC_SEQ_WR_CTL_D1_LP >> 2;
2798                 break;
2799         case MC_PMG_CMD_EMRS >> 2:
2800                 *out_reg = MC_SEQ_PMG_CMD_EMRS_LP >> 2;
2801                 break;
2802         case MC_PMG_CMD_MRS >> 2:
2803                 *out_reg = MC_SEQ_PMG_CMD_MRS_LP >> 2;
2804                 break;
2805         case MC_PMG_CMD_MRS1 >> 2:
2806                 *out_reg = MC_SEQ_PMG_CMD_MRS1_LP >> 2;
2807                 break;
2808         case MC_SEQ_PMG_TIMING >> 2:
2809                 *out_reg = MC_SEQ_PMG_TIMING_LP >> 2;
2810                 break;
2811         case MC_PMG_CMD_MRS2 >> 2:
2812                 *out_reg = MC_SEQ_PMG_CMD_MRS2_LP >> 2;
2813                 break;
2814         default:
2815                 result = false;
2816                 break;
2817         }
2818
2819         return result;
2820 }
2821
2822 static void ni_set_valid_flag(struct ni_mc_reg_table *table)
2823 {
2824         u8 i, j;
2825
2826         for (i = 0; i < table->last; i++) {
2827                 for (j = 1; j < table->num_entries; j++) {
2828                         if (table->mc_reg_table_entry[j-1].mc_data[i] != table->mc_reg_table_entry[j].mc_data[i]) {
2829                                 table->valid_flag |= 1 << i;
2830                                 break;
2831                         }
2832                 }
2833         }
2834 }
2835
2836 static void ni_set_s0_mc_reg_index(struct ni_mc_reg_table *table)
2837 {
2838         u32 i;
2839         u16 address;
2840
2841         for (i = 0; i < table->last; i++)
2842                 table->mc_reg_address[i].s0 =
2843                         ni_check_s0_mc_reg_index(table->mc_reg_address[i].s1, &address) ?
2844                         address : table->mc_reg_address[i].s1;
2845 }
2846
2847 static int ni_copy_vbios_mc_reg_table(struct atom_mc_reg_table *table,
2848                                       struct ni_mc_reg_table *ni_table)
2849 {
2850         u8 i, j;
2851
2852         if (table->last > SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2853                 return -EINVAL;
2854         if (table->num_entries > MAX_AC_TIMING_ENTRIES)
2855                 return -EINVAL;
2856
2857         for (i = 0; i < table->last; i++)
2858                 ni_table->mc_reg_address[i].s1 = table->mc_reg_address[i].s1;
2859         ni_table->last = table->last;
2860
2861         for (i = 0; i < table->num_entries; i++) {
2862                 ni_table->mc_reg_table_entry[i].mclk_max =
2863                         table->mc_reg_table_entry[i].mclk_max;
2864                 for (j = 0; j < table->last; j++)
2865                         ni_table->mc_reg_table_entry[i].mc_data[j] =
2866                                 table->mc_reg_table_entry[i].mc_data[j];
2867         }
2868         ni_table->num_entries = table->num_entries;
2869
2870         return 0;
2871 }
2872
2873 static int ni_initialize_mc_reg_table(struct radeon_device *rdev)
2874 {
2875         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2876         int ret;
2877         struct atom_mc_reg_table *table;
2878         struct ni_mc_reg_table *ni_table = &ni_pi->mc_reg_table;
2879         u8 module_index = rv770_get_memory_module_index(rdev);
2880
2881         table = kzalloc(sizeof(struct atom_mc_reg_table), GFP_KERNEL);
2882         if (!table)
2883                 return -ENOMEM;
2884
2885         WREG32(MC_SEQ_RAS_TIMING_LP, RREG32(MC_SEQ_RAS_TIMING));
2886         WREG32(MC_SEQ_CAS_TIMING_LP, RREG32(MC_SEQ_CAS_TIMING));
2887         WREG32(MC_SEQ_MISC_TIMING_LP, RREG32(MC_SEQ_MISC_TIMING));
2888         WREG32(MC_SEQ_MISC_TIMING2_LP, RREG32(MC_SEQ_MISC_TIMING2));
2889         WREG32(MC_SEQ_PMG_CMD_EMRS_LP, RREG32(MC_PMG_CMD_EMRS));
2890         WREG32(MC_SEQ_PMG_CMD_MRS_LP, RREG32(MC_PMG_CMD_MRS));
2891         WREG32(MC_SEQ_PMG_CMD_MRS1_LP, RREG32(MC_PMG_CMD_MRS1));
2892         WREG32(MC_SEQ_WR_CTL_D0_LP, RREG32(MC_SEQ_WR_CTL_D0));
2893         WREG32(MC_SEQ_WR_CTL_D1_LP, RREG32(MC_SEQ_WR_CTL_D1));
2894         WREG32(MC_SEQ_RD_CTL_D0_LP, RREG32(MC_SEQ_RD_CTL_D0));
2895         WREG32(MC_SEQ_RD_CTL_D1_LP, RREG32(MC_SEQ_RD_CTL_D1));
2896         WREG32(MC_SEQ_PMG_TIMING_LP, RREG32(MC_SEQ_PMG_TIMING));
2897         WREG32(MC_SEQ_PMG_CMD_MRS2_LP, RREG32(MC_PMG_CMD_MRS2));
2898
2899         ret = radeon_atom_init_mc_reg_table(rdev, module_index, table);
2900
2901         if (ret)
2902                 goto init_mc_done;
2903
2904         ret = ni_copy_vbios_mc_reg_table(table, ni_table);
2905
2906         if (ret)
2907                 goto init_mc_done;
2908
2909         ni_set_s0_mc_reg_index(ni_table);
2910
2911         ret = ni_set_mc_special_registers(rdev, ni_table);
2912
2913         if (ret)
2914                 goto init_mc_done;
2915
2916         ni_set_valid_flag(ni_table);
2917
2918 init_mc_done:
2919         kfree(table);
2920
2921         return ret;
2922 }
2923
2924 static void ni_populate_mc_reg_addresses(struct radeon_device *rdev,
2925                                          SMC_NIslands_MCRegisters *mc_reg_table)
2926 {
2927         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2928         u32 i, j;
2929
2930         for (i = 0, j = 0; j < ni_pi->mc_reg_table.last; j++) {
2931                 if (ni_pi->mc_reg_table.valid_flag & (1 << j)) {
2932                         if (i >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2933                                 break;
2934                         mc_reg_table->address[i].s0 =
2935                                 cpu_to_be16(ni_pi->mc_reg_table.mc_reg_address[j].s0);
2936                         mc_reg_table->address[i].s1 =
2937                                 cpu_to_be16(ni_pi->mc_reg_table.mc_reg_address[j].s1);
2938                         i++;
2939                 }
2940         }
2941         mc_reg_table->last = (u8)i;
2942 }
2943
2944
2945 static void ni_convert_mc_registers(struct ni_mc_reg_entry *entry,
2946                                     SMC_NIslands_MCRegisterSet *data,
2947                                     u32 num_entries, u32 valid_flag)
2948 {
2949         u32 i, j;
2950
2951         for (i = 0, j = 0; j < num_entries; j++) {
2952                 if (valid_flag & (1 << j)) {
2953                         data->value[i] = cpu_to_be32(entry->mc_data[j]);
2954                         i++;
2955                 }
2956         }
2957 }
2958
2959 static void ni_convert_mc_reg_table_entry_to_smc(struct radeon_device *rdev,
2960                                                  struct rv7xx_pl *pl,
2961                                                  SMC_NIslands_MCRegisterSet *mc_reg_table_data)
2962 {
2963         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2964         u32 i = 0;
2965
2966         for (i = 0; i < ni_pi->mc_reg_table.num_entries; i++) {
2967                 if (pl->mclk <= ni_pi->mc_reg_table.mc_reg_table_entry[i].mclk_max)
2968                         break;
2969         }
2970
2971         if ((i == ni_pi->mc_reg_table.num_entries) && (i > 0))
2972                 --i;
2973
2974         ni_convert_mc_registers(&ni_pi->mc_reg_table.mc_reg_table_entry[i],
2975                                 mc_reg_table_data,
2976                                 ni_pi->mc_reg_table.last,
2977                                 ni_pi->mc_reg_table.valid_flag);
2978 }
2979
2980 static void ni_convert_mc_reg_table_to_smc(struct radeon_device *rdev,
2981                                            struct radeon_ps *radeon_state,
2982                                            SMC_NIslands_MCRegisters *mc_reg_table)
2983 {
2984         struct ni_ps *state = ni_get_ps(radeon_state);
2985         int i;
2986
2987         for (i = 0; i < state->performance_level_count; i++) {
2988                 ni_convert_mc_reg_table_entry_to_smc(rdev,
2989                                                      &state->performance_levels[i],
2990                                                      &mc_reg_table->data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT + i]);
2991         }
2992 }
2993
2994 static int ni_populate_mc_reg_table(struct radeon_device *rdev,
2995                                     struct radeon_ps *radeon_boot_state)
2996 {
2997         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2998         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2999         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3000         struct ni_ps *boot_state = ni_get_ps(radeon_boot_state);
3001         SMC_NIslands_MCRegisters *mc_reg_table = &ni_pi->smc_mc_reg_table;
3002
3003         memset(mc_reg_table, 0, sizeof(SMC_NIslands_MCRegisters));
3004
3005         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_seq_index, 1);
3006
3007         ni_populate_mc_reg_addresses(rdev, mc_reg_table);
3008
3009         ni_convert_mc_reg_table_entry_to_smc(rdev, &boot_state->performance_levels[0],
3010                                              &mc_reg_table->data[0]);
3011
3012         ni_convert_mc_registers(&ni_pi->mc_reg_table.mc_reg_table_entry[0],
3013                                 &mc_reg_table->data[1],
3014                                 ni_pi->mc_reg_table.last,
3015                                 ni_pi->mc_reg_table.valid_flag);
3016
3017         ni_convert_mc_reg_table_to_smc(rdev, radeon_boot_state, mc_reg_table);
3018
3019         return rv770_copy_bytes_to_smc(rdev, eg_pi->mc_reg_table_start,
3020                                        (u8 *)mc_reg_table,
3021                                        sizeof(SMC_NIslands_MCRegisters),
3022                                        pi->sram_end);
3023 }
3024
3025 static int ni_upload_mc_reg_table(struct radeon_device *rdev,
3026                                   struct radeon_ps *radeon_new_state)
3027 {
3028         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3029         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3030         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3031         struct ni_ps *ni_new_state = ni_get_ps(radeon_new_state);
3032         SMC_NIslands_MCRegisters *mc_reg_table = &ni_pi->smc_mc_reg_table;
3033         u16 address;
3034
3035         memset(mc_reg_table, 0, sizeof(SMC_NIslands_MCRegisters));
3036
3037         ni_convert_mc_reg_table_to_smc(rdev, radeon_new_state, mc_reg_table);
3038
3039         address = eg_pi->mc_reg_table_start +
3040                 (u16)offsetof(SMC_NIslands_MCRegisters, data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT]);
3041
3042         return rv770_copy_bytes_to_smc(rdev, address,
3043                                        (u8 *)&mc_reg_table->data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT],
3044                                        sizeof(SMC_NIslands_MCRegisterSet) * ni_new_state->performance_level_count,
3045                                        pi->sram_end);
3046 }
3047
3048 static int ni_init_driver_calculated_leakage_table(struct radeon_device *rdev,
3049                                                    PP_NIslands_CACTABLES *cac_tables)
3050 {
3051         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3052         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3053         u32 leakage = 0;
3054         unsigned int i, j, table_size;
3055         s32 t;
3056         u32 smc_leakage, max_leakage = 0;
3057         u32 scaling_factor;
3058
3059         table_size = eg_pi->vddc_voltage_table.count;
3060
3061         if (SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES < table_size)
3062                 table_size = SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES;
3063
3064         scaling_factor = ni_get_smc_power_scaling_factor(rdev);
3065
3066         for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++) {
3067                 for (j = 0; j < table_size; j++) {
3068                         t = (1000 * ((i + 1) * 8));
3069
3070                         if (t < ni_pi->cac_data.leakage_minimum_temperature)
3071                                 t = ni_pi->cac_data.leakage_minimum_temperature;
3072
3073                         ni_calculate_leakage_for_v_and_t(rdev,
3074                                                          &ni_pi->cac_data.leakage_coefficients,
3075                                                          eg_pi->vddc_voltage_table.entries[j].value,
3076                                                          t,
3077                                                          ni_pi->cac_data.i_leakage,
3078                                                          &leakage);
3079
3080                         smc_leakage = ni_scale_power_for_smc(leakage, scaling_factor) / 1000;
3081                         if (smc_leakage > max_leakage)
3082                                 max_leakage = smc_leakage;
3083
3084                         cac_tables->cac_lkge_lut[i][j] = cpu_to_be32(smc_leakage);
3085                 }
3086         }
3087
3088         for (j = table_size; j < SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; j++) {
3089                 for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
3090                         cac_tables->cac_lkge_lut[i][j] = cpu_to_be32(max_leakage);
3091         }
3092         return 0;
3093 }
3094
3095 static int ni_init_simplified_leakage_table(struct radeon_device *rdev,
3096                                             PP_NIslands_CACTABLES *cac_tables)
3097 {
3098         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3099         struct radeon_cac_leakage_table *leakage_table =
3100                 &rdev->pm.dpm.dyn_state.cac_leakage_table;
3101         u32 i, j, table_size;
3102         u32 smc_leakage, max_leakage = 0;
3103         u32 scaling_factor;
3104
3105         if (!leakage_table)
3106                 return -EINVAL;
3107
3108         table_size = leakage_table->count;
3109
3110         if (eg_pi->vddc_voltage_table.count != table_size)
3111                 table_size = (eg_pi->vddc_voltage_table.count < leakage_table->count) ?
3112                         eg_pi->vddc_voltage_table.count : leakage_table->count;
3113
3114         if (SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES < table_size)
3115                 table_size = SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES;
3116
3117         if (table_size == 0)
3118                 return -EINVAL;
3119
3120         scaling_factor = ni_get_smc_power_scaling_factor(rdev);
3121
3122         for (j = 0; j < table_size; j++) {
3123                 smc_leakage = leakage_table->entries[j].leakage;
3124
3125                 if (smc_leakage > max_leakage)
3126                         max_leakage = smc_leakage;
3127
3128                 for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
3129                         cac_tables->cac_lkge_lut[i][j] =
3130                                 cpu_to_be32(ni_scale_power_for_smc(smc_leakage, scaling_factor));
3131         }
3132
3133         for (j = table_size; j < SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; j++) {
3134                 for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
3135                         cac_tables->cac_lkge_lut[i][j] =
3136                                 cpu_to_be32(ni_scale_power_for_smc(max_leakage, scaling_factor));
3137         }
3138         return 0;
3139 }
3140
3141 static int ni_initialize_smc_cac_tables(struct radeon_device *rdev)
3142 {
3143         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3144         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3145         PP_NIslands_CACTABLES *cac_tables = NULL;
3146         int i, ret;
3147         u32 reg;
3148
3149         if (ni_pi->enable_cac == false)
3150                 return 0;
3151
3152         cac_tables = kzalloc(sizeof(PP_NIslands_CACTABLES), GFP_KERNEL);
3153         if (!cac_tables)
3154                 return -ENOMEM;
3155
3156         reg = RREG32(CG_CAC_CTRL) & ~(TID_CNT_MASK | TID_UNIT_MASK);
3157         reg |= (TID_CNT(ni_pi->cac_weights->tid_cnt) |
3158                 TID_UNIT(ni_pi->cac_weights->tid_unit));
3159         WREG32(CG_CAC_CTRL, reg);
3160
3161         for (i = 0; i < NISLANDS_DCCAC_MAX_LEVELS; i++)
3162                 ni_pi->dc_cac_table[i] = ni_pi->cac_weights->dc_cac[i];
3163
3164         for (i = 0; i < SMC_NISLANDS_BIF_LUT_NUM_OF_ENTRIES; i++)
3165                 cac_tables->cac_bif_lut[i] = ni_pi->cac_weights->pcie_cac[i];
3166
3167         ni_pi->cac_data.i_leakage = rdev->pm.dpm.cac_leakage;
3168         ni_pi->cac_data.pwr_const = 0;
3169         ni_pi->cac_data.dc_cac_value = ni_pi->dc_cac_table[NISLANDS_DCCAC_LEVEL_0];
3170         ni_pi->cac_data.bif_cac_value = 0;
3171         ni_pi->cac_data.mc_wr_weight = ni_pi->cac_weights->mc_write_weight;
3172         ni_pi->cac_data.mc_rd_weight = ni_pi->cac_weights->mc_read_weight;
3173         ni_pi->cac_data.allow_ovrflw = 0;
3174         ni_pi->cac_data.l2num_win_tdp = ni_pi->lta_window_size;
3175         ni_pi->cac_data.num_win_tdp = 0;
3176         ni_pi->cac_data.lts_truncate_n = ni_pi->lts_truncate;
3177
3178         if (ni_pi->driver_calculate_cac_leakage)
3179                 ret = ni_init_driver_calculated_leakage_table(rdev, cac_tables);
3180         else
3181                 ret = ni_init_simplified_leakage_table(rdev, cac_tables);
3182
3183         if (ret)
3184                 goto done_free;
3185
3186         cac_tables->pwr_const      = cpu_to_be32(ni_pi->cac_data.pwr_const);
3187         cac_tables->dc_cacValue    = cpu_to_be32(ni_pi->cac_data.dc_cac_value);
3188         cac_tables->bif_cacValue   = cpu_to_be32(ni_pi->cac_data.bif_cac_value);
3189         cac_tables->AllowOvrflw    = ni_pi->cac_data.allow_ovrflw;
3190         cac_tables->MCWrWeight     = ni_pi->cac_data.mc_wr_weight;
3191         cac_tables->MCRdWeight     = ni_pi->cac_data.mc_rd_weight;
3192         cac_tables->numWin_TDP     = ni_pi->cac_data.num_win_tdp;
3193         cac_tables->l2numWin_TDP   = ni_pi->cac_data.l2num_win_tdp;
3194         cac_tables->lts_truncate_n = ni_pi->cac_data.lts_truncate_n;
3195
3196         ret = rv770_copy_bytes_to_smc(rdev, ni_pi->cac_table_start, (u8 *)cac_tables,
3197                                       sizeof(PP_NIslands_CACTABLES), pi->sram_end);
3198
3199 done_free:
3200         if (ret) {
3201                 ni_pi->enable_cac = false;
3202                 ni_pi->enable_power_containment = false;
3203         }
3204
3205         kfree(cac_tables);
3206
3207         return 0;
3208 }
3209
3210 static int ni_initialize_hardware_cac_manager(struct radeon_device *rdev)
3211 {
3212         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3213         u32 reg;
3214
3215         if (!ni_pi->enable_cac ||
3216             !ni_pi->cac_configuration_required)
3217                 return 0;
3218
3219         if (ni_pi->cac_weights == NULL)
3220                 return -EINVAL;
3221
3222         reg = RREG32_CG(CG_CAC_REGION_1_WEIGHT_0) & ~(WEIGHT_TCP_SIG0_MASK |
3223                                                       WEIGHT_TCP_SIG1_MASK |
3224                                                       WEIGHT_TA_SIG_MASK);
3225         reg |= (WEIGHT_TCP_SIG0(ni_pi->cac_weights->weight_tcp_sig0) |
3226                 WEIGHT_TCP_SIG1(ni_pi->cac_weights->weight_tcp_sig1) |
3227                 WEIGHT_TA_SIG(ni_pi->cac_weights->weight_ta_sig));
3228         WREG32_CG(CG_CAC_REGION_1_WEIGHT_0, reg);
3229
3230         reg = RREG32_CG(CG_CAC_REGION_1_WEIGHT_1) & ~(WEIGHT_TCC_EN0_MASK |
3231                                                       WEIGHT_TCC_EN1_MASK |
3232                                                       WEIGHT_TCC_EN2_MASK);
3233         reg |= (WEIGHT_TCC_EN0(ni_pi->cac_weights->weight_tcc_en0) |
3234                 WEIGHT_TCC_EN1(ni_pi->cac_weights->weight_tcc_en1) |
3235                 WEIGHT_TCC_EN2(ni_pi->cac_weights->weight_tcc_en2));
3236         WREG32_CG(CG_CAC_REGION_1_WEIGHT_1, reg);
3237
3238         reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_0) & ~(WEIGHT_CB_EN0_MASK |
3239                                                       WEIGHT_CB_EN1_MASK |
3240                                                       WEIGHT_CB_EN2_MASK |
3241                                                       WEIGHT_CB_EN3_MASK);
3242         reg |= (WEIGHT_CB_EN0(ni_pi->cac_weights->weight_cb_en0) |
3243                 WEIGHT_CB_EN1(ni_pi->cac_weights->weight_cb_en1) |
3244                 WEIGHT_CB_EN2(ni_pi->cac_weights->weight_cb_en2) |
3245                 WEIGHT_CB_EN3(ni_pi->cac_weights->weight_cb_en3));
3246         WREG32_CG(CG_CAC_REGION_2_WEIGHT_0, reg);
3247
3248         reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_1) & ~(WEIGHT_DB_SIG0_MASK |
3249                                                       WEIGHT_DB_SIG1_MASK |
3250                                                       WEIGHT_DB_SIG2_MASK |
3251                                                       WEIGHT_DB_SIG3_MASK);
3252         reg |= (WEIGHT_DB_SIG0(ni_pi->cac_weights->weight_db_sig0) |
3253                 WEIGHT_DB_SIG1(ni_pi->cac_weights->weight_db_sig1) |
3254                 WEIGHT_DB_SIG2(ni_pi->cac_weights->weight_db_sig2) |
3255                 WEIGHT_DB_SIG3(ni_pi->cac_weights->weight_db_sig3));
3256         WREG32_CG(CG_CAC_REGION_2_WEIGHT_1, reg);
3257
3258         reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_2) & ~(WEIGHT_SXM_SIG0_MASK |
3259                                                       WEIGHT_SXM_SIG1_MASK |
3260                                                       WEIGHT_SXM_SIG2_MASK |
3261                                                       WEIGHT_SXS_SIG0_MASK |
3262                                                       WEIGHT_SXS_SIG1_MASK);
3263         reg |= (WEIGHT_SXM_SIG0(ni_pi->cac_weights->weight_sxm_sig0) |
3264                 WEIGHT_SXM_SIG1(ni_pi->cac_weights->weight_sxm_sig1) |
3265                 WEIGHT_SXM_SIG2(ni_pi->cac_weights->weight_sxm_sig2) |
3266                 WEIGHT_SXS_SIG0(ni_pi->cac_weights->weight_sxs_sig0) |
3267                 WEIGHT_SXS_SIG1(ni_pi->cac_weights->weight_sxs_sig1));
3268         WREG32_CG(CG_CAC_REGION_2_WEIGHT_2, reg);
3269
3270         reg = RREG32_CG(CG_CAC_REGION_3_WEIGHT_0) & ~(WEIGHT_XBR_0_MASK |
3271                                                       WEIGHT_XBR_1_MASK |
3272                                                       WEIGHT_XBR_2_MASK |
3273                                                       WEIGHT_SPI_SIG0_MASK);
3274         reg |= (WEIGHT_XBR_0(ni_pi->cac_weights->weight_xbr_0) |
3275                 WEIGHT_XBR_1(ni_pi->cac_weights->weight_xbr_1) |
3276                 WEIGHT_XBR_2(ni_pi->cac_weights->weight_xbr_2) |
3277                 WEIGHT_SPI_SIG0(ni_pi->cac_weights->weight_spi_sig0));
3278         WREG32_CG(CG_CAC_REGION_3_WEIGHT_0, reg);
3279
3280         reg = RREG32_CG(CG_CAC_REGION_3_WEIGHT_1) & ~(WEIGHT_SPI_SIG1_MASK |
3281                                                       WEIGHT_SPI_SIG2_MASK |
3282                                                       WEIGHT_SPI_SIG3_MASK |
3283                                                       WEIGHT_SPI_SIG4_MASK |
3284                                                       WEIGHT_SPI_SIG5_MASK);
3285         reg |= (WEIGHT_SPI_SIG1(ni_pi->cac_weights->weight_spi_sig1) |
3286                 WEIGHT_SPI_SIG2(ni_pi->cac_weights->weight_spi_sig2) |
3287                 WEIGHT_SPI_SIG3(ni_pi->cac_weights->weight_spi_sig3) |
3288                 WEIGHT_SPI_SIG4(ni_pi->cac_weights->weight_spi_sig4) |
3289                 WEIGHT_SPI_SIG5(ni_pi->cac_weights->weight_spi_sig5));
3290         WREG32_CG(CG_CAC_REGION_3_WEIGHT_1, reg);
3291
3292         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_0) & ~(WEIGHT_LDS_SIG0_MASK |
3293                                                       WEIGHT_LDS_SIG1_MASK |
3294                                                       WEIGHT_SC_MASK);
3295         reg |= (WEIGHT_LDS_SIG0(ni_pi->cac_weights->weight_lds_sig0) |
3296                 WEIGHT_LDS_SIG1(ni_pi->cac_weights->weight_lds_sig1) |
3297                 WEIGHT_SC(ni_pi->cac_weights->weight_sc));
3298         WREG32_CG(CG_CAC_REGION_4_WEIGHT_0, reg);
3299
3300         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_1) & ~(WEIGHT_BIF_MASK |
3301                                                       WEIGHT_CP_MASK |
3302                                                       WEIGHT_PA_SIG0_MASK |
3303                                                       WEIGHT_PA_SIG1_MASK |
3304                                                       WEIGHT_VGT_SIG0_MASK);
3305         reg |= (WEIGHT_BIF(ni_pi->cac_weights->weight_bif) |
3306                 WEIGHT_CP(ni_pi->cac_weights->weight_cp) |
3307                 WEIGHT_PA_SIG0(ni_pi->cac_weights->weight_pa_sig0) |
3308                 WEIGHT_PA_SIG1(ni_pi->cac_weights->weight_pa_sig1) |
3309                 WEIGHT_VGT_SIG0(ni_pi->cac_weights->weight_vgt_sig0));
3310         WREG32_CG(CG_CAC_REGION_4_WEIGHT_1, reg);
3311
3312         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_2) & ~(WEIGHT_VGT_SIG1_MASK |
3313                                                       WEIGHT_VGT_SIG2_MASK |
3314                                                       WEIGHT_DC_SIG0_MASK |
3315                                                       WEIGHT_DC_SIG1_MASK |
3316                                                       WEIGHT_DC_SIG2_MASK);
3317         reg |= (WEIGHT_VGT_SIG1(ni_pi->cac_weights->weight_vgt_sig1) |
3318                 WEIGHT_VGT_SIG2(ni_pi->cac_weights->weight_vgt_sig2) |
3319                 WEIGHT_DC_SIG0(ni_pi->cac_weights->weight_dc_sig0) |
3320                 WEIGHT_DC_SIG1(ni_pi->cac_weights->weight_dc_sig1) |
3321                 WEIGHT_DC_SIG2(ni_pi->cac_weights->weight_dc_sig2));
3322         WREG32_CG(CG_CAC_REGION_4_WEIGHT_2, reg);
3323
3324         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_3) & ~(WEIGHT_DC_SIG3_MASK |
3325                                                       WEIGHT_UVD_SIG0_MASK |
3326                                                       WEIGHT_UVD_SIG1_MASK |
3327                                                       WEIGHT_SPARE0_MASK |
3328                                                       WEIGHT_SPARE1_MASK);
3329         reg |= (WEIGHT_DC_SIG3(ni_pi->cac_weights->weight_dc_sig3) |
3330                 WEIGHT_UVD_SIG0(ni_pi->cac_weights->weight_uvd_sig0) |
3331                 WEIGHT_UVD_SIG1(ni_pi->cac_weights->weight_uvd_sig1) |
3332                 WEIGHT_SPARE0(ni_pi->cac_weights->weight_spare0) |
3333                 WEIGHT_SPARE1(ni_pi->cac_weights->weight_spare1));
3334         WREG32_CG(CG_CAC_REGION_4_WEIGHT_3, reg);
3335
3336         reg = RREG32_CG(CG_CAC_REGION_5_WEIGHT_0) & ~(WEIGHT_SQ_VSP_MASK |
3337                                                       WEIGHT_SQ_VSP0_MASK);
3338         reg |= (WEIGHT_SQ_VSP(ni_pi->cac_weights->weight_sq_vsp) |
3339                 WEIGHT_SQ_VSP0(ni_pi->cac_weights->weight_sq_vsp0));
3340         WREG32_CG(CG_CAC_REGION_5_WEIGHT_0, reg);
3341
3342         reg = RREG32_CG(CG_CAC_REGION_5_WEIGHT_1) & ~(WEIGHT_SQ_GPR_MASK);
3343         reg |= WEIGHT_SQ_GPR(ni_pi->cac_weights->weight_sq_gpr);
3344         WREG32_CG(CG_CAC_REGION_5_WEIGHT_1, reg);
3345
3346         reg = RREG32_CG(CG_CAC_REGION_4_OVERRIDE_4) & ~(OVR_MODE_SPARE_0_MASK |
3347                                                         OVR_VAL_SPARE_0_MASK |
3348                                                         OVR_MODE_SPARE_1_MASK |
3349                                                         OVR_VAL_SPARE_1_MASK);
3350         reg |= (OVR_MODE_SPARE_0(ni_pi->cac_weights->ovr_mode_spare_0) |
3351                 OVR_VAL_SPARE_0(ni_pi->cac_weights->ovr_val_spare_0) |
3352                 OVR_MODE_SPARE_1(ni_pi->cac_weights->ovr_mode_spare_1) |
3353                 OVR_VAL_SPARE_1(ni_pi->cac_weights->ovr_val_spare_1));
3354         WREG32_CG(CG_CAC_REGION_4_OVERRIDE_4, reg);
3355
3356         reg = RREG32(SQ_CAC_THRESHOLD) & ~(VSP_MASK |
3357                                            VSP0_MASK |
3358                                            GPR_MASK);
3359         reg |= (VSP(ni_pi->cac_weights->vsp) |
3360                 VSP0(ni_pi->cac_weights->vsp0) |
3361                 GPR(ni_pi->cac_weights->gpr));
3362         WREG32(SQ_CAC_THRESHOLD, reg);
3363
3364         reg = (MCDW_WR_ENABLE |
3365                MCDX_WR_ENABLE |
3366                MCDY_WR_ENABLE |
3367                MCDZ_WR_ENABLE |
3368                INDEX(0x09D4));
3369         WREG32(MC_CG_CONFIG, reg);
3370
3371         reg = (READ_WEIGHT(ni_pi->cac_weights->mc_read_weight) |
3372                WRITE_WEIGHT(ni_pi->cac_weights->mc_write_weight) |
3373                ALLOW_OVERFLOW);
3374         WREG32(MC_CG_DATAPORT, reg);
3375
3376         return 0;
3377 }
3378
3379 static int ni_enable_smc_cac(struct radeon_device *rdev,
3380                              struct radeon_ps *radeon_new_state,
3381                              bool enable)
3382 {
3383         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3384         int ret = 0;
3385         PPSMC_Result smc_result;
3386
3387         if (ni_pi->enable_cac) {
3388                 if (enable) {
3389                         if (!r600_is_uvd_state(radeon_new_state->class, radeon_new_state->class2)) {
3390                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_CollectCAC_PowerCorreln);
3391
3392                                 if (ni_pi->support_cac_long_term_average) {
3393                                         smc_result = rv770_send_msg_to_smc(rdev, PPSMC_CACLongTermAvgEnable);
3394                                         if (PPSMC_Result_OK != smc_result)
3395                                                 ni_pi->support_cac_long_term_average = false;
3396                                 }
3397
3398                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_EnableCac);
3399                                 if (PPSMC_Result_OK != smc_result)
3400                                         ret = -EINVAL;
3401
3402                                 ni_pi->cac_enabled = (PPSMC_Result_OK == smc_result) ? true : false;
3403                         }
3404                 } else if (ni_pi->cac_enabled) {
3405                         smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_DisableCac);
3406
3407                         ni_pi->cac_enabled = false;
3408
3409                         if (ni_pi->support_cac_long_term_average) {
3410                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_CACLongTermAvgDisable);
3411                                 if (PPSMC_Result_OK != smc_result)
3412                                         ni_pi->support_cac_long_term_average = false;
3413                         }
3414                 }
3415         }
3416
3417         return ret;
3418 }
3419
3420 static int ni_pcie_performance_request(struct radeon_device *rdev,
3421                                        u8 perf_req, bool advertise)
3422 {
3423         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3424
3425 #if defined(CONFIG_ACPI)
3426         if ((perf_req == PCIE_PERF_REQ_PECI_GEN1) ||
3427             (perf_req == PCIE_PERF_REQ_PECI_GEN2)) {
3428                 if (eg_pi->pcie_performance_request_registered == false)
3429                         radeon_acpi_pcie_notify_device_ready(rdev);
3430                 eg_pi->pcie_performance_request_registered = true;
3431                 return radeon_acpi_pcie_performance_request(rdev, perf_req, advertise);
3432         } else if ((perf_req == PCIE_PERF_REQ_REMOVE_REGISTRY) &&
3433                    eg_pi->pcie_performance_request_registered) {
3434                 eg_pi->pcie_performance_request_registered = false;
3435                 return radeon_acpi_pcie_performance_request(rdev, perf_req, advertise);
3436         }
3437 #endif
3438         return 0;
3439 }
3440
3441 static int ni_advertise_gen2_capability(struct radeon_device *rdev)
3442 {
3443         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3444         u32 tmp;
3445
3446         tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3447
3448         if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
3449             (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2))
3450                 pi->pcie_gen2 = true;
3451         else
3452                 pi->pcie_gen2 = false;
3453
3454         if (!pi->pcie_gen2)
3455                 ni_pcie_performance_request(rdev, PCIE_PERF_REQ_PECI_GEN2, true);
3456
3457         return 0;
3458 }
3459
3460 static void ni_enable_bif_dynamic_pcie_gen2(struct radeon_device *rdev,
3461                                             bool enable)
3462 {
3463         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3464         u32 tmp, bif;
3465
3466         tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3467
3468         if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
3469             (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
3470                 if (enable) {
3471                         if (!pi->boot_in_gen2) {
3472                                 bif = RREG32(CG_BIF_REQ_AND_RSP) & ~CG_CLIENT_REQ_MASK;
3473                                 bif |= CG_CLIENT_REQ(0xd);
3474                                 WREG32(CG_BIF_REQ_AND_RSP, bif);
3475                         }
3476                         tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
3477                         tmp |= LC_HW_VOLTAGE_IF_CONTROL(1);
3478                         tmp |= LC_GEN2_EN_STRAP;
3479
3480                         tmp |= LC_CLR_FAILED_SPD_CHANGE_CNT;
3481                         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
3482                         udelay(10);
3483                         tmp &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
3484                         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
3485                 } else {
3486                         if (!pi->boot_in_gen2) {
3487                                 bif = RREG32(CG_BIF_REQ_AND_RSP) & ~CG_CLIENT_REQ_MASK;
3488                                 bif |= CG_CLIENT_REQ(0xd);
3489                                 WREG32(CG_BIF_REQ_AND_RSP, bif);
3490
3491                                 tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
3492                                 tmp &= ~LC_GEN2_EN_STRAP;
3493                         }
3494                         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
3495                 }
3496         }
3497 }
3498
3499 static void ni_enable_dynamic_pcie_gen2(struct radeon_device *rdev,
3500                                         bool enable)
3501 {
3502         ni_enable_bif_dynamic_pcie_gen2(rdev, enable);
3503
3504         if (enable)
3505                 WREG32_P(GENERAL_PWRMGT, ENABLE_GEN2PCIE, ~ENABLE_GEN2PCIE);
3506         else
3507                 WREG32_P(GENERAL_PWRMGT, 0, ~ENABLE_GEN2PCIE);
3508 }
3509
3510 void ni_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev,
3511                                            struct radeon_ps *new_ps,
3512                                            struct radeon_ps *old_ps)
3513 {
3514         struct ni_ps *new_state = ni_get_ps(new_ps);
3515         struct ni_ps *current_state = ni_get_ps(old_ps);
3516
3517         if ((new_ps->vclk == old_ps->vclk) &&
3518             (new_ps->dclk == old_ps->dclk))
3519                 return;
3520
3521         if (new_state->performance_levels[new_state->performance_level_count - 1].sclk >=
3522             current_state->performance_levels[current_state->performance_level_count - 1].sclk)
3523                 return;
3524
3525         radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
3526 }
3527
3528 void ni_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev,
3529                                           struct radeon_ps *new_ps,
3530                                           struct radeon_ps *old_ps)
3531 {
3532         struct ni_ps *new_state = ni_get_ps(new_ps);
3533         struct ni_ps *current_state = ni_get_ps(old_ps);
3534
3535         if ((new_ps->vclk == old_ps->vclk) &&
3536             (new_ps->dclk == old_ps->dclk))
3537                 return;
3538
3539         if (new_state->performance_levels[new_state->performance_level_count - 1].sclk <
3540             current_state->performance_levels[current_state->performance_level_count - 1].sclk)
3541                 return;
3542
3543         radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
3544 }
3545
3546 void ni_dpm_setup_asic(struct radeon_device *rdev)
3547 {
3548         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3549
3550         ni_read_clock_registers(rdev);
3551         btc_read_arb_registers(rdev);
3552         rv770_get_memory_type(rdev);
3553         if (eg_pi->pcie_performance_request)
3554                 ni_advertise_gen2_capability(rdev);
3555         rv770_get_pcie_gen2_status(rdev);
3556         rv770_enable_acpi_pm(rdev);
3557 }
3558
3559 void ni_update_current_ps(struct radeon_device *rdev,
3560                           struct radeon_ps *rps)
3561 {
3562         struct ni_ps *new_ps = ni_get_ps(rps);
3563         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3564         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3565
3566         eg_pi->current_rps = *rps;
3567         ni_pi->current_ps = *new_ps;
3568         eg_pi->current_rps.ps_priv = &ni_pi->current_ps;
3569 }
3570
3571 void ni_update_requested_ps(struct radeon_device *rdev,
3572                             struct radeon_ps *rps)
3573 {
3574         struct ni_ps *new_ps = ni_get_ps(rps);
3575         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3576         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3577
3578         eg_pi->requested_rps = *rps;
3579         ni_pi->requested_ps = *new_ps;
3580         eg_pi->requested_rps.ps_priv = &ni_pi->requested_ps;
3581 }
3582
3583 int ni_dpm_enable(struct radeon_device *rdev)
3584 {
3585         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3586         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3587         struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
3588         int ret;
3589
3590         if (pi->gfx_clock_gating)
3591                 ni_cg_clockgating_default(rdev);
3592         if (btc_dpm_enabled(rdev))
3593                 return -EINVAL;
3594         if (pi->mg_clock_gating)
3595                 ni_mg_clockgating_default(rdev);
3596         if (eg_pi->ls_clock_gating)
3597                 ni_ls_clockgating_default(rdev);
3598         if (pi->voltage_control) {
3599                 rv770_enable_voltage_control(rdev, true);
3600                 ret = cypress_construct_voltage_tables(rdev);
3601                 if (ret) {
3602                         DRM_ERROR("cypress_construct_voltage_tables failed\n");
3603                         return ret;
3604                 }
3605         }
3606         if (eg_pi->dynamic_ac_timing) {
3607                 ret = ni_initialize_mc_reg_table(rdev);
3608                 if (ret)
3609                         eg_pi->dynamic_ac_timing = false;
3610         }
3611         if (pi->dynamic_ss)
3612                 cypress_enable_spread_spectrum(rdev, true);
3613         if (pi->thermal_protection)
3614                 rv770_enable_thermal_protection(rdev, true);
3615         rv770_setup_bsp(rdev);
3616         rv770_program_git(rdev);
3617         rv770_program_tp(rdev);
3618         rv770_program_tpp(rdev);
3619         rv770_program_sstp(rdev);
3620         cypress_enable_display_gap(rdev);
3621         rv770_program_vc(rdev);
3622         if (pi->dynamic_pcie_gen2)
3623                 ni_enable_dynamic_pcie_gen2(rdev, true);
3624         ret = rv770_upload_firmware(rdev);
3625         if (ret) {
3626                 DRM_ERROR("rv770_upload_firmware failed\n");
3627                 return ret;
3628         }
3629         ret = ni_process_firmware_header(rdev);
3630         if (ret) {
3631                 DRM_ERROR("ni_process_firmware_header failed\n");
3632                 return ret;
3633         }
3634         ret = ni_initial_switch_from_arb_f0_to_f1(rdev);
3635         if (ret) {
3636                 DRM_ERROR("ni_initial_switch_from_arb_f0_to_f1 failed\n");
3637                 return ret;
3638         }
3639         ret = ni_init_smc_table(rdev);
3640         if (ret) {
3641                 DRM_ERROR("ni_init_smc_table failed\n");
3642                 return ret;
3643         }
3644         ret = ni_init_smc_spll_table(rdev);
3645         if (ret) {
3646                 DRM_ERROR("ni_init_smc_spll_table failed\n");
3647                 return ret;
3648         }
3649         ret = ni_init_arb_table_index(rdev);
3650         if (ret) {
3651                 DRM_ERROR("ni_init_arb_table_index failed\n");
3652                 return ret;
3653         }
3654         if (eg_pi->dynamic_ac_timing) {
3655                 ret = ni_populate_mc_reg_table(rdev, boot_ps);
3656                 if (ret) {
3657                         DRM_ERROR("ni_populate_mc_reg_table failed\n");
3658                         return ret;
3659                 }
3660         }
3661         ret = ni_initialize_smc_cac_tables(rdev);
3662         if (ret) {
3663                 DRM_ERROR("ni_initialize_smc_cac_tables failed\n");
3664                 return ret;
3665         }
3666         ret = ni_initialize_hardware_cac_manager(rdev);
3667         if (ret) {
3668                 DRM_ERROR("ni_initialize_hardware_cac_manager failed\n");
3669                 return ret;
3670         }
3671         ret = ni_populate_smc_tdp_limits(rdev, boot_ps);
3672         if (ret) {
3673                 DRM_ERROR("ni_populate_smc_tdp_limits failed\n");
3674                 return ret;
3675         }
3676         ni_program_response_times(rdev);
3677         r7xx_start_smc(rdev);
3678         ret = cypress_notify_smc_display_change(rdev, false);
3679         if (ret) {
3680                 DRM_ERROR("cypress_notify_smc_display_change failed\n");
3681                 return ret;
3682         }
3683         cypress_enable_sclk_control(rdev, true);
3684         if (eg_pi->memory_transition)
3685                 cypress_enable_mclk_control(rdev, true);
3686         cypress_start_dpm(rdev);
3687         if (pi->gfx_clock_gating)
3688                 ni_gfx_clockgating_enable(rdev, true);
3689         if (pi->mg_clock_gating)
3690                 ni_mg_clockgating_enable(rdev, true);
3691         if (eg_pi->ls_clock_gating)
3692                 ni_ls_clockgating_enable(rdev, true);
3693
3694         if (rdev->irq.installed &&
3695             r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
3696                 PPSMC_Result result;
3697
3698                 ret = rv770_set_thermal_temperature_range(rdev, R600_TEMP_RANGE_MIN, 0xff * 1000);
3699                 if (ret)
3700                         return ret;
3701                 rdev->irq.dpm_thermal = true;
3702                 radeon_irq_set(rdev);
3703                 result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_EnableThermalInterrupt);
3704
3705                 if (result != PPSMC_Result_OK)
3706                         DRM_DEBUG_KMS("Could not enable thermal interrupts.\n");
3707         }
3708
3709         rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, true);
3710
3711         ni_update_current_ps(rdev, boot_ps);
3712
3713         return 0;
3714 }
3715
3716 void ni_dpm_disable(struct radeon_device *rdev)
3717 {
3718         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3719         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3720         struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
3721
3722         if (!btc_dpm_enabled(rdev))
3723                 return;
3724         rv770_clear_vc(rdev);
3725         if (pi->thermal_protection)
3726                 rv770_enable_thermal_protection(rdev, false);
3727         ni_enable_power_containment(rdev, boot_ps, false);
3728         ni_enable_smc_cac(rdev, boot_ps, false);
3729         cypress_enable_spread_spectrum(rdev, false);
3730         rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, false);
3731         if (pi->dynamic_pcie_gen2)
3732                 ni_enable_dynamic_pcie_gen2(rdev, false);
3733
3734         if (rdev->irq.installed &&
3735             r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
3736                 rdev->irq.dpm_thermal = false;
3737                 radeon_irq_set(rdev);
3738         }
3739
3740         if (pi->gfx_clock_gating)
3741                 ni_gfx_clockgating_enable(rdev, false);
3742         if (pi->mg_clock_gating)
3743                 ni_mg_clockgating_enable(rdev, false);
3744         if (eg_pi->ls_clock_gating)
3745                 ni_ls_clockgating_enable(rdev, false);
3746         ni_stop_dpm(rdev);
3747         btc_reset_to_default(rdev);
3748         ni_stop_smc(rdev);
3749         ni_force_switch_to_arb_f0(rdev);
3750
3751         ni_update_current_ps(rdev, boot_ps);
3752 }
3753
3754 static int ni_power_control_set_level(struct radeon_device *rdev)
3755 {
3756         struct radeon_ps *new_ps = rdev->pm.dpm.requested_ps;
3757         int ret;
3758
3759         ret = ni_restrict_performance_levels_before_switch(rdev);
3760         if (ret)
3761                 return ret;
3762         ret = rv770_halt_smc(rdev);
3763         if (ret)
3764                 return ret;
3765         ret = ni_populate_smc_tdp_limits(rdev, new_ps);
3766         if (ret)
3767                 return ret;
3768         ret = rv770_resume_smc(rdev);
3769         if (ret)
3770                 return ret;
3771         ret = rv770_set_sw_state(rdev);
3772         if (ret)
3773                 return ret;
3774
3775         return 0;
3776 }
3777
3778 int ni_dpm_pre_set_power_state(struct radeon_device *rdev)
3779 {
3780         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3781         struct radeon_ps requested_ps = *rdev->pm.dpm.requested_ps;
3782         struct radeon_ps *new_ps = &requested_ps;
3783
3784         ni_update_requested_ps(rdev, new_ps);
3785
3786         ni_apply_state_adjust_rules(rdev, &eg_pi->requested_rps);
3787
3788         return 0;
3789 }
3790
3791 int ni_dpm_set_power_state(struct radeon_device *rdev)
3792 {
3793         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3794         struct radeon_ps *new_ps = &eg_pi->requested_rps;
3795         struct radeon_ps *old_ps = &eg_pi->current_rps;
3796         int ret;
3797
3798         ret = ni_restrict_performance_levels_before_switch(rdev);
3799         if (ret) {
3800                 DRM_ERROR("ni_restrict_performance_levels_before_switch failed\n");
3801                 return ret;
3802         }
3803         ni_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps);
3804         ret = ni_enable_power_containment(rdev, new_ps, false);
3805         if (ret) {
3806                 DRM_ERROR("ni_enable_power_containment failed\n");
3807                 return ret;
3808         }
3809         ret = ni_enable_smc_cac(rdev, new_ps, false);
3810         if (ret) {
3811                 DRM_ERROR("ni_enable_smc_cac failed\n");
3812                 return ret;
3813         }
3814         ret = rv770_halt_smc(rdev);
3815         if (ret) {
3816                 DRM_ERROR("rv770_halt_smc failed\n");
3817                 return ret;
3818         }
3819         if (eg_pi->smu_uvd_hs)
3820                 btc_notify_uvd_to_smc(rdev, new_ps);
3821         ret = ni_upload_sw_state(rdev, new_ps);
3822         if (ret) {
3823                 DRM_ERROR("ni_upload_sw_state failed\n");
3824                 return ret;
3825         }
3826         if (eg_pi->dynamic_ac_timing) {
3827                 ret = ni_upload_mc_reg_table(rdev, new_ps);
3828                 if (ret) {
3829                         DRM_ERROR("ni_upload_mc_reg_table failed\n");
3830                         return ret;
3831                 }
3832         }
3833         ret = ni_program_memory_timing_parameters(rdev, new_ps);
3834         if (ret) {
3835                 DRM_ERROR("ni_program_memory_timing_parameters failed\n");
3836                 return ret;
3837         }
3838         ret = rv770_resume_smc(rdev);
3839         if (ret) {
3840                 DRM_ERROR("rv770_resume_smc failed\n");
3841                 return ret;
3842         }
3843         ret = rv770_set_sw_state(rdev);
3844         if (ret) {
3845                 DRM_ERROR("rv770_set_sw_state failed\n");
3846                 return ret;
3847         }
3848         ni_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps);
3849         ret = ni_enable_smc_cac(rdev, new_ps, true);
3850         if (ret) {
3851                 DRM_ERROR("ni_enable_smc_cac failed\n");
3852                 return ret;
3853         }
3854         ret = ni_enable_power_containment(rdev, new_ps, true);
3855         if (ret) {
3856                 DRM_ERROR("ni_enable_power_containment failed\n");
3857                 return ret;
3858         }
3859
3860         /* update tdp */
3861         ret = ni_power_control_set_level(rdev);
3862         if (ret) {
3863                 DRM_ERROR("ni_power_control_set_level failed\n");
3864                 return ret;
3865         }
3866
3867         ret = ni_dpm_force_performance_level(rdev, RADEON_DPM_FORCED_LEVEL_AUTO);
3868         if (ret) {
3869                 DRM_ERROR("ni_dpm_force_performance_level failed\n");
3870                 return ret;
3871         }
3872
3873         return 0;
3874 }
3875
3876 void ni_dpm_post_set_power_state(struct radeon_device *rdev)
3877 {
3878         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3879         struct radeon_ps *new_ps = &eg_pi->requested_rps;
3880
3881         ni_update_current_ps(rdev, new_ps);
3882 }
3883
3884 void ni_dpm_reset_asic(struct radeon_device *rdev)
3885 {
3886         ni_restrict_performance_levels_before_switch(rdev);
3887         rv770_set_boot_state(rdev);
3888 }
3889
3890 union power_info {
3891         struct _ATOM_POWERPLAY_INFO info;
3892         struct _ATOM_POWERPLAY_INFO_V2 info_2;
3893         struct _ATOM_POWERPLAY_INFO_V3 info_3;
3894         struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
3895         struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
3896         struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
3897 };
3898
3899 union pplib_clock_info {
3900         struct _ATOM_PPLIB_R600_CLOCK_INFO r600;
3901         struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780;
3902         struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
3903         struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
3904 };
3905
3906 union pplib_power_state {
3907         struct _ATOM_PPLIB_STATE v1;
3908         struct _ATOM_PPLIB_STATE_V2 v2;
3909 };
3910
3911 static void ni_parse_pplib_non_clock_info(struct radeon_device *rdev,
3912                                           struct radeon_ps *rps,
3913                                           struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info,
3914                                           u8 table_rev)
3915 {
3916         rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings);
3917         rps->class = le16_to_cpu(non_clock_info->usClassification);
3918         rps->class2 = le16_to_cpu(non_clock_info->usClassification2);
3919
3920         if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) {
3921                 rps->vclk = le32_to_cpu(non_clock_info->ulVCLK);
3922                 rps->dclk = le32_to_cpu(non_clock_info->ulDCLK);
3923         } else if (r600_is_uvd_state(rps->class, rps->class2)) {
3924                 rps->vclk = RV770_DEFAULT_VCLK_FREQ;
3925                 rps->dclk = RV770_DEFAULT_DCLK_FREQ;
3926         } else {
3927                 rps->vclk = 0;
3928                 rps->dclk = 0;
3929         }
3930
3931         if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT)
3932                 rdev->pm.dpm.boot_ps = rps;
3933         if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
3934                 rdev->pm.dpm.uvd_ps = rps;
3935 }
3936
3937 static void ni_parse_pplib_clock_info(struct radeon_device *rdev,
3938                                       struct radeon_ps *rps, int index,
3939                                       union pplib_clock_info *clock_info)
3940 {
3941         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3942         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3943         struct ni_ps *ps = ni_get_ps(rps);
3944         u16 vddc;
3945         struct rv7xx_pl *pl = &ps->performance_levels[index];
3946
3947         ps->performance_level_count = index + 1;
3948
3949         pl->sclk = le16_to_cpu(clock_info->evergreen.usEngineClockLow);
3950         pl->sclk |= clock_info->evergreen.ucEngineClockHigh << 16;
3951         pl->mclk = le16_to_cpu(clock_info->evergreen.usMemoryClockLow);
3952         pl->mclk |= clock_info->evergreen.ucMemoryClockHigh << 16;
3953
3954         pl->vddc = le16_to_cpu(clock_info->evergreen.usVDDC);
3955         pl->vddci = le16_to_cpu(clock_info->evergreen.usVDDCI);
3956         pl->flags = le32_to_cpu(clock_info->evergreen.ulFlags);
3957
3958         /* patch up vddc if necessary */
3959         if (pl->vddc == 0xff01) {
3960                 if (radeon_atom_get_max_vddc(rdev, 0, 0, &vddc) == 0)
3961                         pl->vddc = vddc;
3962         }
3963
3964         if (rps->class & ATOM_PPLIB_CLASSIFICATION_ACPI) {
3965                 pi->acpi_vddc = pl->vddc;
3966                 eg_pi->acpi_vddci = pl->vddci;
3967                 if (ps->performance_levels[0].flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2)
3968                         pi->acpi_pcie_gen2 = true;
3969                 else
3970                         pi->acpi_pcie_gen2 = false;
3971         }
3972
3973         if (rps->class2 & ATOM_PPLIB_CLASSIFICATION2_ULV) {
3974                 eg_pi->ulv.supported = true;
3975                 eg_pi->ulv.pl = pl;
3976         }
3977
3978         if (pi->min_vddc_in_table > pl->vddc)
3979                 pi->min_vddc_in_table = pl->vddc;
3980
3981         if (pi->max_vddc_in_table < pl->vddc)
3982                 pi->max_vddc_in_table = pl->vddc;
3983
3984         /* patch up boot state */
3985         if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) {
3986                 u16 vddc, vddci, mvdd;
3987                 radeon_atombios_get_default_voltages(rdev, &vddc, &vddci, &mvdd);
3988                 pl->mclk = rdev->clock.default_mclk;
3989                 pl->sclk = rdev->clock.default_sclk;
3990                 pl->vddc = vddc;
3991                 pl->vddci = vddci;
3992         }
3993
3994         if ((rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) ==
3995             ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE) {
3996                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk = pl->sclk;
3997                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.mclk = pl->mclk;
3998                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddc = pl->vddc;
3999                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddci = pl->vddci;
4000         }
4001 }
4002
4003 static int ni_parse_power_table(struct radeon_device *rdev)
4004 {
4005         struct radeon_mode_info *mode_info = &rdev->mode_info;
4006         struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
4007         union pplib_power_state *power_state;
4008         int i, j;
4009         union pplib_clock_info *clock_info;
4010         union power_info *power_info;
4011         int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
4012         u16 data_offset;
4013         u8 frev, crev;
4014         struct ni_ps *ps;
4015
4016         if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
4017                                    &frev, &crev, &data_offset))
4018                 return -EINVAL;
4019         power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
4020
4021         rdev->pm.dpm.ps = kzalloc(sizeof(struct radeon_ps) *
4022                                   power_info->pplib.ucNumStates, GFP_KERNEL);
4023         if (!rdev->pm.dpm.ps)
4024                 return -ENOMEM;
4025         rdev->pm.dpm.platform_caps = le32_to_cpu(power_info->pplib.ulPlatformCaps);
4026         rdev->pm.dpm.backbias_response_time = le16_to_cpu(power_info->pplib.usBackbiasTime);
4027         rdev->pm.dpm.voltage_response_time = le16_to_cpu(power_info->pplib.usVoltageTime);
4028
4029         for (i = 0; i < power_info->pplib.ucNumStates; i++) {
4030                 power_state = (union pplib_power_state *)
4031                         (mode_info->atom_context->bios + data_offset +
4032                          le16_to_cpu(power_info->pplib.usStateArrayOffset) +
4033                          i * power_info->pplib.ucStateEntrySize);
4034                 non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
4035                         (mode_info->atom_context->bios + data_offset +
4036                          le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset) +
4037                          (power_state->v1.ucNonClockStateIndex *
4038                           power_info->pplib.ucNonClockSize));
4039                 if (power_info->pplib.ucStateEntrySize - 1) {
4040                         ps = kzalloc(sizeof(struct ni_ps), GFP_KERNEL);
4041                         if (ps == NULL) {
4042                                 kfree(rdev->pm.dpm.ps);
4043                                 return -ENOMEM;
4044                         }
4045                         rdev->pm.dpm.ps[i].ps_priv = ps;
4046                         ni_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i],
4047                                                          non_clock_info,
4048                                                          power_info->pplib.ucNonClockSize);
4049                         for (j = 0; j < (power_info->pplib.ucStateEntrySize - 1); j++) {
4050                                 clock_info = (union pplib_clock_info *)
4051                                         (mode_info->atom_context->bios + data_offset +
4052                                          le16_to_cpu(power_info->pplib.usClockInfoArrayOffset) +
4053                                          (power_state->v1.ucClockStateIndices[j] *
4054                                           power_info->pplib.ucClockInfoSize));
4055                                 ni_parse_pplib_clock_info(rdev,
4056                                                           &rdev->pm.dpm.ps[i], j,
4057                                                           clock_info);
4058                         }
4059                 }
4060         }
4061         rdev->pm.dpm.num_ps = power_info->pplib.ucNumStates;
4062         return 0;
4063 }
4064
4065 int ni_dpm_init(struct radeon_device *rdev)
4066 {
4067         struct rv7xx_power_info *pi;
4068         struct evergreen_power_info *eg_pi;
4069         struct ni_power_info *ni_pi;
4070         int index = GetIndexIntoMasterTable(DATA, ASIC_InternalSS_Info);
4071         u16 data_offset, size;
4072         u8 frev, crev;
4073         struct atom_clock_dividers dividers;
4074         int ret;
4075
4076         ni_pi = kzalloc(sizeof(struct ni_power_info), GFP_KERNEL);
4077         if (ni_pi == NULL)
4078                 return -ENOMEM;
4079         rdev->pm.dpm.priv = ni_pi;
4080         eg_pi = &ni_pi->eg;
4081         pi = &eg_pi->rv7xx;
4082
4083         rv770_get_max_vddc(rdev);
4084
4085         eg_pi->ulv.supported = false;
4086         pi->acpi_vddc = 0;
4087         eg_pi->acpi_vddci = 0;
4088         pi->min_vddc_in_table = 0;
4089         pi->max_vddc_in_table = 0;
4090
4091         ret = ni_parse_power_table(rdev);
4092         if (ret)
4093                 return ret;
4094         ret = r600_parse_extended_power_table(rdev);
4095         if (ret)
4096                 return ret;
4097
4098         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries =
4099                 kzalloc(4 * sizeof(struct radeon_clock_voltage_dependency_entry), GFP_KERNEL);
4100         if (!rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries) {
4101                 r600_free_extended_power_table(rdev);
4102                 return -ENOMEM;
4103         }
4104         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.count = 4;
4105         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].clk = 0;
4106         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].v = 0;
4107         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].clk = 36000;
4108         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].v = 720;
4109         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].clk = 54000;
4110         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].v = 810;
4111         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].clk = 72000;
4112         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].v = 900;
4113
4114         ni_patch_dependency_tables_based_on_leakage(rdev);
4115
4116         if (rdev->pm.dpm.voltage_response_time == 0)
4117                 rdev->pm.dpm.voltage_response_time = R600_VOLTAGERESPONSETIME_DFLT;
4118         if (rdev->pm.dpm.backbias_response_time == 0)
4119                 rdev->pm.dpm.backbias_response_time = R600_BACKBIASRESPONSETIME_DFLT;
4120
4121         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
4122                                              0, false, &dividers);
4123         if (ret)
4124                 pi->ref_div = dividers.ref_div + 1;
4125         else
4126                 pi->ref_div = R600_REFERENCEDIVIDER_DFLT;
4127
4128         pi->rlp = RV770_RLP_DFLT;
4129         pi->rmp = RV770_RMP_DFLT;
4130         pi->lhp = RV770_LHP_DFLT;
4131         pi->lmp = RV770_LMP_DFLT;
4132
4133         eg_pi->ats[0].rlp = RV770_RLP_DFLT;
4134         eg_pi->ats[0].rmp = RV770_RMP_DFLT;
4135         eg_pi->ats[0].lhp = RV770_LHP_DFLT;
4136         eg_pi->ats[0].lmp = RV770_LMP_DFLT;
4137
4138         eg_pi->ats[1].rlp = BTC_RLP_UVD_DFLT;
4139         eg_pi->ats[1].rmp = BTC_RMP_UVD_DFLT;
4140         eg_pi->ats[1].lhp = BTC_LHP_UVD_DFLT;
4141         eg_pi->ats[1].lmp = BTC_LMP_UVD_DFLT;
4142
4143         eg_pi->smu_uvd_hs = true;
4144
4145         if (rdev->pdev->device == 0x6707) {
4146                 pi->mclk_strobe_mode_threshold = 55000;
4147                 pi->mclk_edc_enable_threshold = 55000;
4148                 eg_pi->mclk_edc_wr_enable_threshold = 55000;
4149         } else {
4150                 pi->mclk_strobe_mode_threshold = 40000;
4151                 pi->mclk_edc_enable_threshold = 40000;
4152                 eg_pi->mclk_edc_wr_enable_threshold = 40000;
4153         }
4154         ni_pi->mclk_rtt_mode_threshold = eg_pi->mclk_edc_wr_enable_threshold;
4155
4156         pi->voltage_control =
4157                 radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, 0);
4158
4159         pi->mvdd_control =
4160                 radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_MVDDC, 0);
4161
4162         eg_pi->vddci_control =
4163                 radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDCI, 0);
4164
4165         if (atom_parse_data_header(rdev->mode_info.atom_context, index, &size,
4166                                    &frev, &crev, &data_offset)) {
4167                 pi->sclk_ss = true;
4168                 pi->mclk_ss = true;
4169                 pi->dynamic_ss = true;
4170         } else {
4171                 pi->sclk_ss = false;
4172                 pi->mclk_ss = false;
4173                 pi->dynamic_ss = true;
4174         }
4175
4176         pi->asi = RV770_ASI_DFLT;
4177         pi->pasi = CYPRESS_HASI_DFLT;
4178         pi->vrc = CYPRESS_VRC_DFLT;
4179
4180         pi->power_gating = false;
4181
4182         pi->gfx_clock_gating = true;
4183
4184         pi->mg_clock_gating = true;
4185         pi->mgcgtssm = true;
4186         eg_pi->ls_clock_gating = false;
4187         eg_pi->sclk_deep_sleep = false;
4188
4189         pi->dynamic_pcie_gen2 = true;
4190
4191         if (rdev->pm.int_thermal_type != THERMAL_TYPE_NONE)
4192                 pi->thermal_protection = true;
4193         else
4194                 pi->thermal_protection = false;
4195
4196         pi->display_gap = true;
4197
4198         pi->dcodt = true;
4199
4200         pi->ulps = true;
4201
4202         eg_pi->dynamic_ac_timing = true;
4203         eg_pi->abm = true;
4204         eg_pi->mcls = true;
4205         eg_pi->light_sleep = true;
4206         eg_pi->memory_transition = true;
4207 #if defined(CONFIG_ACPI)
4208         eg_pi->pcie_performance_request =
4209                 radeon_acpi_is_pcie_performance_request_supported(rdev);
4210 #else
4211         eg_pi->pcie_performance_request = false;
4212 #endif
4213
4214         eg_pi->dll_default_on = false;
4215
4216         eg_pi->sclk_deep_sleep = false;
4217
4218         pi->mclk_stutter_mode_threshold = 0;
4219
4220         pi->sram_end = SMC_RAM_END;
4221
4222         rdev->pm.dpm.dyn_state.mclk_sclk_ratio = 3;
4223         rdev->pm.dpm.dyn_state.vddc_vddci_delta = 200;
4224         rdev->pm.dpm.dyn_state.min_vddc_for_pcie_gen2 = 900;
4225         rdev->pm.dpm.dyn_state.valid_sclk_values.count = ARRAY_SIZE(btc_valid_sclk);
4226         rdev->pm.dpm.dyn_state.valid_sclk_values.values = btc_valid_sclk;
4227         rdev->pm.dpm.dyn_state.valid_mclk_values.count = 0;
4228         rdev->pm.dpm.dyn_state.valid_mclk_values.values = NULL;
4229         rdev->pm.dpm.dyn_state.sclk_mclk_delta = 12500;
4230
4231         ni_pi->cac_data.leakage_coefficients.at = 516;
4232         ni_pi->cac_data.leakage_coefficients.bt = 18;
4233         ni_pi->cac_data.leakage_coefficients.av = 51;
4234         ni_pi->cac_data.leakage_coefficients.bv = 2957;
4235
4236         switch (rdev->pdev->device) {
4237         case 0x6700:
4238         case 0x6701:
4239         case 0x6702:
4240         case 0x6703:
4241         case 0x6718:
4242                 ni_pi->cac_weights = &cac_weights_cayman_xt;
4243                 break;
4244         case 0x6705:
4245         case 0x6719:
4246         case 0x671D:
4247         case 0x671C:
4248         default:
4249                 ni_pi->cac_weights = &cac_weights_cayman_pro;
4250                 break;
4251         case 0x6704:
4252         case 0x6706:
4253         case 0x6707:
4254         case 0x6708:
4255         case 0x6709:
4256                 ni_pi->cac_weights = &cac_weights_cayman_le;
4257                 break;
4258         }
4259
4260         if (ni_pi->cac_weights->enable_power_containment_by_default) {
4261                 ni_pi->enable_power_containment = true;
4262                 ni_pi->enable_cac = true;
4263                 ni_pi->enable_sq_ramping = true;
4264         } else {
4265                 ni_pi->enable_power_containment = false;
4266                 ni_pi->enable_cac = false;
4267                 ni_pi->enable_sq_ramping = false;
4268         }
4269
4270         ni_pi->driver_calculate_cac_leakage = false;
4271         ni_pi->cac_configuration_required = true;
4272
4273         if (ni_pi->cac_configuration_required) {
4274                 ni_pi->support_cac_long_term_average = true;
4275                 ni_pi->lta_window_size = ni_pi->cac_weights->l2_lta_window_size;
4276                 ni_pi->lts_truncate = ni_pi->cac_weights->lts_truncate;
4277         } else {
4278                 ni_pi->support_cac_long_term_average = false;
4279                 ni_pi->lta_window_size = 0;
4280                 ni_pi->lts_truncate = 0;
4281         }
4282
4283         ni_pi->use_power_boost_limit = true;
4284
4285         return 0;
4286 }
4287
4288 void ni_dpm_fini(struct radeon_device *rdev)
4289 {
4290         int i;
4291
4292         for (i = 0; i < rdev->pm.dpm.num_ps; i++) {
4293                 kfree(rdev->pm.dpm.ps[i].ps_priv);
4294         }
4295         kfree(rdev->pm.dpm.ps);
4296         kfree(rdev->pm.dpm.priv);
4297         kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries);
4298         r600_free_extended_power_table(rdev);
4299 }
4300
4301 void ni_dpm_print_power_state(struct radeon_device *rdev,
4302                               struct radeon_ps *rps)
4303 {
4304         struct ni_ps *ps = ni_get_ps(rps);
4305         struct rv7xx_pl *pl;
4306         int i;
4307
4308         r600_dpm_print_class_info(rps->class, rps->class2);
4309         r600_dpm_print_cap_info(rps->caps);
4310         printk("\tuvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
4311         for (i = 0; i < ps->performance_level_count; i++) {
4312                 pl = &ps->performance_levels[i];
4313                 if (rdev->family >= CHIP_TAHITI)
4314                         printk("\t\tpower level %d    sclk: %u mclk: %u vddc: %u vddci: %u pcie gen: %u\n",
4315                                i, pl->sclk, pl->mclk, pl->vddc, pl->vddci, pl->pcie_gen + 1);
4316                 else
4317                         printk("\t\tpower level %d    sclk: %u mclk: %u vddc: %u vddci: %u\n",
4318                                i, pl->sclk, pl->mclk, pl->vddc, pl->vddci);
4319         }
4320         r600_dpm_print_ps_status(rdev, rps);
4321 }
4322
4323 void ni_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev,
4324                                                     struct seq_file *m)
4325 {
4326         struct radeon_ps *rps = rdev->pm.dpm.current_ps;
4327         struct ni_ps *ps = ni_get_ps(rps);
4328         struct rv7xx_pl *pl;
4329         u32 current_index =
4330                 (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_INDEX_MASK) >>
4331                 CURRENT_STATE_INDEX_SHIFT;
4332
4333         if (current_index >= ps->performance_level_count) {
4334                 seq_printf(m, "invalid dpm profile %d\n", current_index);
4335         } else {
4336                 pl = &ps->performance_levels[current_index];
4337                 seq_printf(m, "uvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
4338                 seq_printf(m, "power level %d    sclk: %u mclk: %u vddc: %u vddci: %u\n",
4339                            current_index, pl->sclk, pl->mclk, pl->vddc, pl->vddci);
4340         }
4341 }
4342
4343 u32 ni_dpm_get_sclk(struct radeon_device *rdev, bool low)
4344 {
4345         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4346         struct ni_ps *requested_state = ni_get_ps(&eg_pi->requested_rps);
4347
4348         if (low)
4349                 return requested_state->performance_levels[0].sclk;
4350         else
4351                 return requested_state->performance_levels[requested_state->performance_level_count - 1].sclk;
4352 }
4353
4354 u32 ni_dpm_get_mclk(struct radeon_device *rdev, bool low)
4355 {
4356         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4357         struct ni_ps *requested_state = ni_get_ps(&eg_pi->requested_rps);
4358
4359         if (low)
4360                 return requested_state->performance_levels[0].mclk;
4361         else
4362                 return requested_state->performance_levels[requested_state->performance_level_count - 1].mclk;
4363 }
4364