drivers/gpu/drm/radeon/ni_dpm.c

   1 /*
   2  * Copyright 2012 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include "drmP.h"
  25 #include "radeon.h"
  26 #include "nid.h"
  27 #include "r600_dpm.h"
  28 #include "ni_dpm.h"
  29 #include "atom.h"
  30 #include <linux/math64.h>
  31 #include <linux/seq_file.h>
  32
  33 #define MC_CG_ARB_FREQ_F0           0x0a
  34 #define MC_CG_ARB_FREQ_F1           0x0b
  35 #define MC_CG_ARB_FREQ_F2           0x0c
  36 #define MC_CG_ARB_FREQ_F3           0x0d
  37
  38 #define SMC_RAM_END 0xC000
  39
  40 static const struct ni_cac_weights cac_weights_cayman_xt =
  41 {
  42         0x15,
  43         0x2,
  44         0x19,
  45         0x2,
  46         0x8,
  47         0x14,
  48         0x2,
  49         0x16,
  50         0xE,
  51         0x17,
  52         0x13,
  53         0x2B,
  54         0x10,
  55         0x7,
  56         0x5,
  57         0x5,
  58         0x5,
  59         0x2,
  60         0x3,
  61         0x9,
  62         0x10,
  63         0x10,
  64         0x2B,
  65         0xA,
  66         0x9,
  67         0x4,
  68         0xD,
  69         0xD,
  70         0x3E,
  71         0x18,
  72         0x14,
  73         0,
  74         0x3,
  75         0x3,
  76         0x5,
  77         0,
  78         0x2,
  79         0,
  80         0,
  81         0,
  82         0,
  83         0,
  84         0,
  85         0,
  86         0,
  87         0,
  88         0x1CC,
  89         0,
  90         0x164,
  91         1,
  92         1,
  93         1,
  94         1,
  95         12,
  96         12,
  97         12,
  98         0x12,
  99         0x1F,
 100         132,
 101         5,
 102         7,
 103         0,
 104         { 0, 0, 0, 0, 0, 0, 0, 0 },
 105         { 0, 0, 0, 0 },
 106         true
 107 };
 108
 109 static const struct ni_cac_weights cac_weights_cayman_pro =
 110 {
 111         0x16,
 112         0x4,
 113         0x10,
 114         0x2,
 115         0xA,
 116         0x16,
 117         0x2,
 118         0x18,
 119         0x10,
 120         0x1A,
 121         0x16,
 122         0x2D,
 123         0x12,
 124         0xA,
 125         0x6,
 126         0x6,
 127         0x6,
 128         0x2,
 129         0x4,
 130         0xB,
 131         0x11,
 132         0x11,
 133         0x2D,
 134         0xC,
 135         0xC,
 136         0x7,
 137         0x10,
 138         0x10,
 139         0x3F,
 140         0x1A,
 141         0x16,
 142         0,
 143         0x7,
 144         0x4,
 145         0x6,
 146         1,
 147         0x2,
 148         0x1,
 149         0,
 150         0,
 151         0,
 152         0,
 153         0,
 154         0,
 155         0x30,
 156         0,
 157         0x1CF,
 158         0,
 159         0x166,
 160         1,
 161         1,
 162         1,
 163         1,
 164         12,
 165         12,
 166         12,
 167         0x15,
 168         0x1F,
 169         132,
 170         6,
 171         6,
 172         0,
 173         { 0, 0, 0, 0, 0, 0, 0, 0 },
 174         { 0, 0, 0, 0 },
 175         true
 176 };
 177
 178 static const struct ni_cac_weights cac_weights_cayman_le =
 179 {
 180         0x7,
 181         0xE,
 182         0x1,
 183         0xA,
 184         0x1,
 185         0x3F,
 186         0x2,
 187         0x18,
 188         0x10,
 189         0x1A,
 190         0x1,
 191         0x3F,
 192         0x1,
 193         0xE,
 194         0x6,
 195         0x6,
 196         0x6,
 197         0x2,
 198         0x4,
 199         0x9,
 200         0x1A,
 201         0x1A,
 202         0x2C,
 203         0xA,
 204         0x11,
 205         0x8,
 206         0x19,
 207         0x19,
 208         0x1,
 209         0x1,
 210         0x1A,
 211         0,
 212         0x8,
 213         0x5,
 214         0x8,
 215         0x1,
 216         0x3,
 217         0x1,
 218         0,
 219         0,
 220         0,
 221         0,
 222         0,
 223         0,
 224         0x38,
 225         0x38,
 226         0x239,
 227         0x3,
 228         0x18A,
 229         1,
 230         1,
 231         1,
 232         1,
 233         12,
 234         12,
 235         12,
 236         0x15,
 237         0x22,
 238         132,
 239         6,
 240         6,
 241         0,
 242         { 0, 0, 0, 0, 0, 0, 0, 0 },
 243         { 0, 0, 0, 0 },
 244         true
 245 };
 246
 247 #define NISLANDS_MGCG_SEQUENCE  300
 248
 249 static const u32 cayman_cgcg_cgls_default[] =
 250 {
 251         0x000008f8, 0x00000010, 0xffffffff,
 252         0x000008fc, 0x00000000, 0xffffffff,
 253         0x000008f8, 0x00000011, 0xffffffff,
 254         0x000008fc, 0x00000000, 0xffffffff,
 255         0x000008f8, 0x00000012, 0xffffffff,
 256         0x000008fc, 0x00000000, 0xffffffff,
 257         0x000008f8, 0x00000013, 0xffffffff,
 258         0x000008fc, 0x00000000, 0xffffffff,
 259         0x000008f8, 0x00000014, 0xffffffff,
 260         0x000008fc, 0x00000000, 0xffffffff,
 261         0x000008f8, 0x00000015, 0xffffffff,
 262         0x000008fc, 0x00000000, 0xffffffff,
 263         0x000008f8, 0x00000016, 0xffffffff,
 264         0x000008fc, 0x00000000, 0xffffffff,
 265         0x000008f8, 0x00000017, 0xffffffff,
 266         0x000008fc, 0x00000000, 0xffffffff,
 267         0x000008f8, 0x00000018, 0xffffffff,
 268         0x000008fc, 0x00000000, 0xffffffff,
 269         0x000008f8, 0x00000019, 0xffffffff,
 270         0x000008fc, 0x00000000, 0xffffffff,
 271         0x000008f8, 0x0000001a, 0xffffffff,
 272         0x000008fc, 0x00000000, 0xffffffff,
 273         0x000008f8, 0x0000001b, 0xffffffff,
 274         0x000008fc, 0x00000000, 0xffffffff,
 275         0x000008f8, 0x00000020, 0xffffffff,
 276         0x000008fc, 0x00000000, 0xffffffff,
 277         0x000008f8, 0x00000021, 0xffffffff,
 278         0x000008fc, 0x00000000, 0xffffffff,
 279         0x000008f8, 0x00000022, 0xffffffff,
 280         0x000008fc, 0x00000000, 0xffffffff,
 281         0x000008f8, 0x00000023, 0xffffffff,
 282         0x000008fc, 0x00000000, 0xffffffff,
 283         0x000008f8, 0x00000024, 0xffffffff,
 284         0x000008fc, 0x00000000, 0xffffffff,
 285         0x000008f8, 0x00000025, 0xffffffff,
 286         0x000008fc, 0x00000000, 0xffffffff,
 287         0x000008f8, 0x00000026, 0xffffffff,
 288         0x000008fc, 0x00000000, 0xffffffff,
 289         0x000008f8, 0x00000027, 0xffffffff,
 290         0x000008fc, 0x00000000, 0xffffffff,
 291         0x000008f8, 0x00000028, 0xffffffff,
 292         0x000008fc, 0x00000000, 0xffffffff,
 293         0x000008f8, 0x00000029, 0xffffffff,
 294         0x000008fc, 0x00000000, 0xffffffff,
 295         0x000008f8, 0x0000002a, 0xffffffff,
 296         0x000008fc, 0x00000000, 0xffffffff,
 297         0x000008f8, 0x0000002b, 0xffffffff,
 298         0x000008fc, 0x00000000, 0xffffffff
 299 };
 300 #define CAYMAN_CGCG_CGLS_DEFAULT_LENGTH sizeof(cayman_cgcg_cgls_default) / (3 * sizeof(u32))
 301
 302 static const u32 cayman_cgcg_cgls_disable[] =
 303 {
 304         0x000008f8, 0x00000010, 0xffffffff,
 305         0x000008fc, 0xffffffff, 0xffffffff,
 306         0x000008f8, 0x00000011, 0xffffffff,
 307         0x000008fc, 0xffffffff, 0xffffffff,
 308         0x000008f8, 0x00000012, 0xffffffff,
 309         0x000008fc, 0xffffffff, 0xffffffff,
 310         0x000008f8, 0x00000013, 0xffffffff,
 311         0x000008fc, 0xffffffff, 0xffffffff,
 312         0x000008f8, 0x00000014, 0xffffffff,
 313         0x000008fc, 0xffffffff, 0xffffffff,
 314         0x000008f8, 0x00000015, 0xffffffff,
 315         0x000008fc, 0xffffffff, 0xffffffff,
 316         0x000008f8, 0x00000016, 0xffffffff,
 317         0x000008fc, 0xffffffff, 0xffffffff,
 318         0x000008f8, 0x00000017, 0xffffffff,
 319         0x000008fc, 0xffffffff, 0xffffffff,
 320         0x000008f8, 0x00000018, 0xffffffff,
 321         0x000008fc, 0xffffffff, 0xffffffff,
 322         0x000008f8, 0x00000019, 0xffffffff,
 323         0x000008fc, 0xffffffff, 0xffffffff,
 324         0x000008f8, 0x0000001a, 0xffffffff,
 325         0x000008fc, 0xffffffff, 0xffffffff,
 326         0x000008f8, 0x0000001b, 0xffffffff,
 327         0x000008fc, 0xffffffff, 0xffffffff,
 328         0x000008f8, 0x00000020, 0xffffffff,
 329         0x000008fc, 0x00000000, 0xffffffff,
 330         0x000008f8, 0x00000021, 0xffffffff,
 331         0x000008fc, 0x00000000, 0xffffffff,
 332         0x000008f8, 0x00000022, 0xffffffff,
 333         0x000008fc, 0x00000000, 0xffffffff,
 334         0x000008f8, 0x00000023, 0xffffffff,
 335         0x000008fc, 0x00000000, 0xffffffff,
 336         0x000008f8, 0x00000024, 0xffffffff,
 337         0x000008fc, 0x00000000, 0xffffffff,
 338         0x000008f8, 0x00000025, 0xffffffff,
 339         0x000008fc, 0x00000000, 0xffffffff,
 340         0x000008f8, 0x00000026, 0xffffffff,
 341         0x000008fc, 0x00000000, 0xffffffff,
 342         0x000008f8, 0x00000027, 0xffffffff,
 343         0x000008fc, 0x00000000, 0xffffffff,
 344         0x000008f8, 0x00000028, 0xffffffff,
 345         0x000008fc, 0x00000000, 0xffffffff,
 346         0x000008f8, 0x00000029, 0xffffffff,
 347         0x000008fc, 0x00000000, 0xffffffff,
 348         0x000008f8, 0x0000002a, 0xffffffff,
 349         0x000008fc, 0x00000000, 0xffffffff,
 350         0x000008f8, 0x0000002b, 0xffffffff,
 351         0x000008fc, 0x00000000, 0xffffffff,
 352         0x00000644, 0x000f7902, 0x001f4180,
 353         0x00000644, 0x000f3802, 0x001f4180
 354 };
 355 #define CAYMAN_CGCG_CGLS_DISABLE_LENGTH sizeof(cayman_cgcg_cgls_disable) / (3 * sizeof(u32))
 356
 357 static const u32 cayman_cgcg_cgls_enable[] =
 358 {
 359         0x00000644, 0x000f7882, 0x001f4080,
 360         0x000008f8, 0x00000010, 0xffffffff,
 361         0x000008fc, 0x00000000, 0xffffffff,
 362         0x000008f8, 0x00000011, 0xffffffff,
 363         0x000008fc, 0x00000000, 0xffffffff,
 364         0x000008f8, 0x00000012, 0xffffffff,
 365         0x000008fc, 0x00000000, 0xffffffff,
 366         0x000008f8, 0x00000013, 0xffffffff,
 367         0x000008fc, 0x00000000, 0xffffffff,
 368         0x000008f8, 0x00000014, 0xffffffff,
 369         0x000008fc, 0x00000000, 0xffffffff,
 370         0x000008f8, 0x00000015, 0xffffffff,
 371         0x000008fc, 0x00000000, 0xffffffff,
 372         0x000008f8, 0x00000016, 0xffffffff,
 373         0x000008fc, 0x00000000, 0xffffffff,
 374         0x000008f8, 0x00000017, 0xffffffff,
 375         0x000008fc, 0x00000000, 0xffffffff,
 376         0x000008f8, 0x00000018, 0xffffffff,
 377         0x000008fc, 0x00000000, 0xffffffff,
 378         0x000008f8, 0x00000019, 0xffffffff,
 379         0x000008fc, 0x00000000, 0xffffffff,
 380         0x000008f8, 0x0000001a, 0xffffffff,
 381         0x000008fc, 0x00000000, 0xffffffff,
 382         0x000008f8, 0x0000001b, 0xffffffff,
 383         0x000008fc, 0x00000000, 0xffffffff,
 384         0x000008f8, 0x00000020, 0xffffffff,
 385         0x000008fc, 0xffffffff, 0xffffffff,
 386         0x000008f8, 0x00000021, 0xffffffff,
 387         0x000008fc, 0xffffffff, 0xffffffff,
 388         0x000008f8, 0x00000022, 0xffffffff,
 389         0x000008fc, 0xffffffff, 0xffffffff,
 390         0x000008f8, 0x00000023, 0xffffffff,
 391         0x000008fc, 0xffffffff, 0xffffffff,
 392         0x000008f8, 0x00000024, 0xffffffff,
 393         0x000008fc, 0xffffffff, 0xffffffff,
 394         0x000008f8, 0x00000025, 0xffffffff,
 395         0x000008fc, 0xffffffff, 0xffffffff,
 396         0x000008f8, 0x00000026, 0xffffffff,
 397         0x000008fc, 0xffffffff, 0xffffffff,
 398         0x000008f8, 0x00000027, 0xffffffff,
 399         0x000008fc, 0xffffffff, 0xffffffff,
 400         0x000008f8, 0x00000028, 0xffffffff,
 401         0x000008fc, 0xffffffff, 0xffffffff,
 402         0x000008f8, 0x00000029, 0xffffffff,
 403         0x000008fc, 0xffffffff, 0xffffffff,
 404         0x000008f8, 0x0000002a, 0xffffffff,
 405         0x000008fc, 0xffffffff, 0xffffffff,
 406         0x000008f8, 0x0000002b, 0xffffffff,
 407         0x000008fc, 0xffffffff, 0xffffffff
 408 };
 409 #define CAYMAN_CGCG_CGLS_ENABLE_LENGTH  sizeof(cayman_cgcg_cgls_enable) / (3 * sizeof(u32))
 410
 411 static const u32 cayman_mgcg_default[] =
 412 {
 413         0x0000802c, 0xc0000000, 0xffffffff,
 414         0x00003fc4, 0xc0000000, 0xffffffff,
 415         0x00005448, 0x00000100, 0xffffffff,
 416         0x000055e4, 0x00000100, 0xffffffff,
 417         0x0000160c, 0x00000100, 0xffffffff,
 418         0x00008984, 0x06000100, 0xffffffff,
 419         0x0000c164, 0x00000100, 0xffffffff,
 420         0x00008a18, 0x00000100, 0xffffffff,
 421         0x0000897c, 0x06000100, 0xffffffff,
 422         0x00008b28, 0x00000100, 0xffffffff,
 423         0x00009144, 0x00800200, 0xffffffff,
 424         0x00009a60, 0x00000100, 0xffffffff,
 425         0x00009868, 0x00000100, 0xffffffff,
 426         0x00008d58, 0x00000100, 0xffffffff,
 427         0x00009510, 0x00000100, 0xffffffff,
 428         0x0000949c, 0x00000100, 0xffffffff,
 429         0x00009654, 0x00000100, 0xffffffff,
 430         0x00009030, 0x00000100, 0xffffffff,
 431         0x00009034, 0x00000100, 0xffffffff,
 432         0x00009038, 0x00000100, 0xffffffff,
 433         0x0000903c, 0x00000100, 0xffffffff,
 434         0x00009040, 0x00000100, 0xffffffff,
 435         0x0000a200, 0x00000100, 0xffffffff,
 436         0x0000a204, 0x00000100, 0xffffffff,
 437         0x0000a208, 0x00000100, 0xffffffff,
 438         0x0000a20c, 0x00000100, 0xffffffff,
 439         0x00009744, 0x00000100, 0xffffffff,
 440         0x00003f80, 0x00000100, 0xffffffff,
 441         0x0000a210, 0x00000100, 0xffffffff,
 442         0x0000a214, 0x00000100, 0xffffffff,
 443         0x000004d8, 0x00000100, 0xffffffff,
 444         0x00009664, 0x00000100, 0xffffffff,
 445         0x00009698, 0x00000100, 0xffffffff,
 446         0x000004d4, 0x00000200, 0xffffffff,
 447         0x000004d0, 0x00000000, 0xffffffff,
 448         0x000030cc, 0x00000104, 0xffffffff,
 449         0x0000d0c0, 0x00000100, 0xffffffff,
 450         0x0000d8c0, 0x00000100, 0xffffffff,
 451         0x0000802c, 0x40000000, 0xffffffff,
 452         0x00003fc4, 0x40000000, 0xffffffff,
 453         0x0000915c, 0x00010000, 0xffffffff,
 454         0x00009160, 0x00030002, 0xffffffff,
 455         0x00009164, 0x00050004, 0xffffffff,
 456         0x00009168, 0x00070006, 0xffffffff,
 457         0x00009178, 0x00070000, 0xffffffff,
 458         0x0000917c, 0x00030002, 0xffffffff,
 459         0x00009180, 0x00050004, 0xffffffff,
 460         0x0000918c, 0x00010006, 0xffffffff,
 461         0x00009190, 0x00090008, 0xffffffff,
 462         0x00009194, 0x00070000, 0xffffffff,
 463         0x00009198, 0x00030002, 0xffffffff,
 464         0x0000919c, 0x00050004, 0xffffffff,
 465         0x000091a8, 0x00010006, 0xffffffff,
 466         0x000091ac, 0x00090008, 0xffffffff,
 467         0x000091b0, 0x00070000, 0xffffffff,
 468         0x000091b4, 0x00030002, 0xffffffff,
 469         0x000091b8, 0x00050004, 0xffffffff,
 470         0x000091c4, 0x00010006, 0xffffffff,
 471         0x000091c8, 0x00090008, 0xffffffff,
 472         0x000091cc, 0x00070000, 0xffffffff,
 473         0x000091d0, 0x00030002, 0xffffffff,
 474         0x000091d4, 0x00050004, 0xffffffff,
 475         0x000091e0, 0x00010006, 0xffffffff,
 476         0x000091e4, 0x00090008, 0xffffffff,
 477         0x000091e8, 0x00000000, 0xffffffff,
 478         0x000091ec, 0x00070000, 0xffffffff,
 479         0x000091f0, 0x00030002, 0xffffffff,
 480         0x000091f4, 0x00050004, 0xffffffff,
 481         0x00009200, 0x00010006, 0xffffffff,
 482         0x00009204, 0x00090008, 0xffffffff,
 483         0x00009208, 0x00070000, 0xffffffff,
 484         0x0000920c, 0x00030002, 0xffffffff,
 485         0x00009210, 0x00050004, 0xffffffff,
 486         0x0000921c, 0x00010006, 0xffffffff,
 487         0x00009220, 0x00090008, 0xffffffff,
 488         0x00009224, 0x00070000, 0xffffffff,
 489         0x00009228, 0x00030002, 0xffffffff,
 490         0x0000922c, 0x00050004, 0xffffffff,
 491         0x00009238, 0x00010006, 0xffffffff,
 492         0x0000923c, 0x00090008, 0xffffffff,
 493         0x00009240, 0x00070000, 0xffffffff,
 494         0x00009244, 0x00030002, 0xffffffff,
 495         0x00009248, 0x00050004, 0xffffffff,
 496         0x00009254, 0x00010006, 0xffffffff,
 497         0x00009258, 0x00090008, 0xffffffff,
 498         0x0000925c, 0x00070000, 0xffffffff,
 499         0x00009260, 0x00030002, 0xffffffff,
 500         0x00009264, 0x00050004, 0xffffffff,
 501         0x00009270, 0x00010006, 0xffffffff,
 502         0x00009274, 0x00090008, 0xffffffff,
 503         0x00009278, 0x00070000, 0xffffffff,
 504         0x0000927c, 0x00030002, 0xffffffff,
 505         0x00009280, 0x00050004, 0xffffffff,
 506         0x0000928c, 0x00010006, 0xffffffff,
 507         0x00009290, 0x00090008, 0xffffffff,
 508         0x000092a8, 0x00070000, 0xffffffff,
 509         0x000092ac, 0x00030002, 0xffffffff,
 510         0x000092b0, 0x00050004, 0xffffffff,
 511         0x000092bc, 0x00010006, 0xffffffff,
 512         0x000092c0, 0x00090008, 0xffffffff,
 513         0x000092c4, 0x00070000, 0xffffffff,
 514         0x000092c8, 0x00030002, 0xffffffff,
 515         0x000092cc, 0x00050004, 0xffffffff,
 516         0x000092d8, 0x00010006, 0xffffffff,
 517         0x000092dc, 0x00090008, 0xffffffff,
 518         0x00009294, 0x00000000, 0xffffffff,
 519         0x0000802c, 0x40010000, 0xffffffff,
 520         0x00003fc4, 0x40010000, 0xffffffff,
 521         0x0000915c, 0x00010000, 0xffffffff,
 522         0x00009160, 0x00030002, 0xffffffff,
 523         0x00009164, 0x00050004, 0xffffffff,
 524         0x00009168, 0x00070006, 0xffffffff,
 525         0x00009178, 0x00070000, 0xffffffff,
 526         0x0000917c, 0x00030002, 0xffffffff,
 527         0x00009180, 0x00050004, 0xffffffff,
 528         0x0000918c, 0x00010006, 0xffffffff,
 529         0x00009190, 0x00090008, 0xffffffff,
 530         0x00009194, 0x00070000, 0xffffffff,
 531         0x00009198, 0x00030002, 0xffffffff,
 532         0x0000919c, 0x00050004, 0xffffffff,
 533         0x000091a8, 0x00010006, 0xffffffff,
 534         0x000091ac, 0x00090008, 0xffffffff,
 535         0x000091b0, 0x00070000, 0xffffffff,
 536         0x000091b4, 0x00030002, 0xffffffff,
 537         0x000091b8, 0x00050004, 0xffffffff,
 538         0x000091c4, 0x00010006, 0xffffffff,
 539         0x000091c8, 0x00090008, 0xffffffff,
 540         0x000091cc, 0x00070000, 0xffffffff,
 541         0x000091d0, 0x00030002, 0xffffffff,
 542         0x000091d4, 0x00050004, 0xffffffff,
 543         0x000091e0, 0x00010006, 0xffffffff,
 544         0x000091e4, 0x00090008, 0xffffffff,
 545         0x000091e8, 0x00000000, 0xffffffff,
 546         0x000091ec, 0x00070000, 0xffffffff,
 547         0x000091f0, 0x00030002, 0xffffffff,
 548         0x000091f4, 0x00050004, 0xffffffff,
 549         0x00009200, 0x00010006, 0xffffffff,
 550         0x00009204, 0x00090008, 0xffffffff,
 551         0x00009208, 0x00070000, 0xffffffff,
 552         0x0000920c, 0x00030002, 0xffffffff,
 553         0x00009210, 0x00050004, 0xffffffff,
 554         0x0000921c, 0x00010006, 0xffffffff,
 555         0x00009220, 0x00090008, 0xffffffff,
 556         0x00009224, 0x00070000, 0xffffffff,
 557         0x00009228, 0x00030002, 0xffffffff,
 558         0x0000922c, 0x00050004, 0xffffffff,
 559         0x00009238, 0x00010006, 0xffffffff,
 560         0x0000923c, 0x00090008, 0xffffffff,
 561         0x00009240, 0x00070000, 0xffffffff,
 562         0x00009244, 0x00030002, 0xffffffff,
 563         0x00009248, 0x00050004, 0xffffffff,
 564         0x00009254, 0x00010006, 0xffffffff,
 565         0x00009258, 0x00090008, 0xffffffff,
 566         0x0000925c, 0x00070000, 0xffffffff,
 567         0x00009260, 0x00030002, 0xffffffff,
 568         0x00009264, 0x00050004, 0xffffffff,
 569         0x00009270, 0x00010006, 0xffffffff,
 570         0x00009274, 0x00090008, 0xffffffff,
 571         0x00009278, 0x00070000, 0xffffffff,
 572         0x0000927c, 0x00030002, 0xffffffff,
 573         0x00009280, 0x00050004, 0xffffffff,
 574         0x0000928c, 0x00010006, 0xffffffff,
 575         0x00009290, 0x00090008, 0xffffffff,
 576         0x000092a8, 0x00070000, 0xffffffff,
 577         0x000092ac, 0x00030002, 0xffffffff,
 578         0x000092b0, 0x00050004, 0xffffffff,
 579         0x000092bc, 0x00010006, 0xffffffff,
 580         0x000092c0, 0x00090008, 0xffffffff,
 581         0x000092c4, 0x00070000, 0xffffffff,
 582         0x000092c8, 0x00030002, 0xffffffff,
 583         0x000092cc, 0x00050004, 0xffffffff,
 584         0x000092d8, 0x00010006, 0xffffffff,
 585         0x000092dc, 0x00090008, 0xffffffff,
 586         0x00009294, 0x00000000, 0xffffffff,
 587         0x0000802c, 0xc0000000, 0xffffffff,
 588         0x00003fc4, 0xc0000000, 0xffffffff,
 589         0x000008f8, 0x00000010, 0xffffffff,
 590         0x000008fc, 0x00000000, 0xffffffff,
 591         0x000008f8, 0x00000011, 0xffffffff,
 592         0x000008fc, 0x00000000, 0xffffffff,
 593         0x000008f8, 0x00000012, 0xffffffff,
 594         0x000008fc, 0x00000000, 0xffffffff,
 595         0x000008f8, 0x00000013, 0xffffffff,
 596         0x000008fc, 0x00000000, 0xffffffff,
 597         0x000008f8, 0x00000014, 0xffffffff,
 598         0x000008fc, 0x00000000, 0xffffffff,
 599         0x000008f8, 0x00000015, 0xffffffff,
 600         0x000008fc, 0x00000000, 0xffffffff,
 601         0x000008f8, 0x00000016, 0xffffffff,
 602         0x000008fc, 0x00000000, 0xffffffff,
 603         0x000008f8, 0x00000017, 0xffffffff,
 604         0x000008fc, 0x00000000, 0xffffffff,
 605         0x000008f8, 0x00000018, 0xffffffff,
 606         0x000008fc, 0x00000000, 0xffffffff,
 607         0x000008f8, 0x00000019, 0xffffffff,
 608         0x000008fc, 0x00000000, 0xffffffff,
 609         0x000008f8, 0x0000001a, 0xffffffff,
 610         0x000008fc, 0x00000000, 0xffffffff,
 611         0x000008f8, 0x0000001b, 0xffffffff,
 612         0x000008fc, 0x00000000, 0xffffffff
 613 };
 614 #define CAYMAN_MGCG_DEFAULT_LENGTH sizeof(cayman_mgcg_default) / (3 * sizeof(u32))
 615
 616 static const u32 cayman_mgcg_disable[] =
 617 {
 618         0x0000802c, 0xc0000000, 0xffffffff,
 619         0x000008f8, 0x00000000, 0xffffffff,
 620         0x000008fc, 0xffffffff, 0xffffffff,
 621         0x000008f8, 0x00000001, 0xffffffff,
 622         0x000008fc, 0xffffffff, 0xffffffff,
 623         0x000008f8, 0x00000002, 0xffffffff,
 624         0x000008fc, 0xffffffff, 0xffffffff,
 625         0x000008f8, 0x00000003, 0xffffffff,
 626         0x000008fc, 0xffffffff, 0xffffffff,
 627         0x00009150, 0x00600000, 0xffffffff
 628 };
 629 #define CAYMAN_MGCG_DISABLE_LENGTH   sizeof(cayman_mgcg_disable) / (3 * sizeof(u32))
 630
 631 static const u32 cayman_mgcg_enable[] =
 632 {
 633         0x0000802c, 0xc0000000, 0xffffffff,
 634         0x000008f8, 0x00000000, 0xffffffff,
 635         0x000008fc, 0x00000000, 0xffffffff,
 636         0x000008f8, 0x00000001, 0xffffffff,
 637         0x000008fc, 0x00000000, 0xffffffff,
 638         0x000008f8, 0x00000002, 0xffffffff,
 639         0x000008fc, 0x00600000, 0xffffffff,
 640         0x000008f8, 0x00000003, 0xffffffff,
 641         0x000008fc, 0x00000000, 0xffffffff,
 642         0x00009150, 0x96944200, 0xffffffff
 643 };
 644
 645 #define CAYMAN_MGCG_ENABLE_LENGTH   sizeof(cayman_mgcg_enable) / (3 * sizeof(u32))
 646
 647 #define NISLANDS_SYSLS_SEQUENCE  100
 648
 649 static const u32 cayman_sysls_default[] =
 650 {
 651         /* Register,   Value,     Mask bits */
 652         0x000055e8, 0x00000000, 0xffffffff,
 653         0x0000d0bc, 0x00000000, 0xffffffff,
 654         0x0000d8bc, 0x00000000, 0xffffffff,
 655         0x000015c0, 0x000c1401, 0xffffffff,
 656         0x0000264c, 0x000c0400, 0xffffffff,
 657         0x00002648, 0x000c0400, 0xffffffff,
 658         0x00002650, 0x000c0400, 0xffffffff,
 659         0x000020b8, 0x000c0400, 0xffffffff,
 660         0x000020bc, 0x000c0400, 0xffffffff,
 661         0x000020c0, 0x000c0c80, 0xffffffff,
 662         0x0000f4a0, 0x000000c0, 0xffffffff,
 663         0x0000f4a4, 0x00680fff, 0xffffffff,
 664         0x00002f50, 0x00000404, 0xffffffff,
 665         0x000004c8, 0x00000001, 0xffffffff,
 666         0x000064ec, 0x00000000, 0xffffffff,
 667         0x00000c7c, 0x00000000, 0xffffffff,
 668         0x00008dfc, 0x00000000, 0xffffffff
 669 };
 670 #define CAYMAN_SYSLS_DEFAULT_LENGTH sizeof(cayman_sysls_default) / (3 * sizeof(u32))
 671
 672 static const u32 cayman_sysls_disable[] =
 673 {
 674         /* Register,   Value,     Mask bits */
 675         0x0000d0c0, 0x00000000, 0xffffffff,
 676         0x0000d8c0, 0x00000000, 0xffffffff,
 677         0x000055e8, 0x00000000, 0xffffffff,
 678         0x0000d0bc, 0x00000000, 0xffffffff,
 679         0x0000d8bc, 0x00000000, 0xffffffff,
 680         0x000015c0, 0x00041401, 0xffffffff,
 681         0x0000264c, 0x00040400, 0xffffffff,
 682         0x00002648, 0x00040400, 0xffffffff,
 683         0x00002650, 0x00040400, 0xffffffff,
 684         0x000020b8, 0x00040400, 0xffffffff,
 685         0x000020bc, 0x00040400, 0xffffffff,
 686         0x000020c0, 0x00040c80, 0xffffffff,
 687         0x0000f4a0, 0x000000c0, 0xffffffff,
 688         0x0000f4a4, 0x00680000, 0xffffffff,
 689         0x00002f50, 0x00000404, 0xffffffff,
 690         0x000004c8, 0x00000001, 0xffffffff,
 691         0x000064ec, 0x00007ffd, 0xffffffff,
 692         0x00000c7c, 0x0000ff00, 0xffffffff,
 693         0x00008dfc, 0x0000007f, 0xffffffff
 694 };
 695 #define CAYMAN_SYSLS_DISABLE_LENGTH sizeof(cayman_sysls_disable) / (3 * sizeof(u32))
 696
 697 static const u32 cayman_sysls_enable[] =
 698 {
 699         /* Register,   Value,     Mask bits */
 700         0x000055e8, 0x00000001, 0xffffffff,
 701         0x0000d0bc, 0x00000100, 0xffffffff,
 702         0x0000d8bc, 0x00000100, 0xffffffff,
 703         0x000015c0, 0x000c1401, 0xffffffff,
 704         0x0000264c, 0x000c0400, 0xffffffff,
 705         0x00002648, 0x000c0400, 0xffffffff,
 706         0x00002650, 0x000c0400, 0xffffffff,
 707         0x000020b8, 0x000c0400, 0xffffffff,
 708         0x000020bc, 0x000c0400, 0xffffffff,
 709         0x000020c0, 0x000c0c80, 0xffffffff,
 710         0x0000f4a0, 0x000000c0, 0xffffffff,
 711         0x0000f4a4, 0x00680fff, 0xffffffff,
 712         0x00002f50, 0x00000903, 0xffffffff,
 713         0x000004c8, 0x00000000, 0xffffffff,
 714         0x000064ec, 0x00000000, 0xffffffff,
 715         0x00000c7c, 0x00000000, 0xffffffff,
 716         0x00008dfc, 0x00000000, 0xffffffff
 717 };
 718 #define CAYMAN_SYSLS_ENABLE_LENGTH sizeof(cayman_sysls_enable) / (3 * sizeof(u32))
 719
 720 struct rv7xx_power_info *rv770_get_pi(struct radeon_device *rdev);
 721 struct evergreen_power_info *evergreen_get_pi(struct radeon_device *rdev);
 722
 723 struct ni_power_info *ni_get_pi(struct radeon_device *rdev)
 724 {
 725         struct ni_power_info *pi = rdev->pm.dpm.priv;
 726
 727         return pi;
 728 }
 729
 730 struct ni_ps *ni_get_ps(struct radeon_ps *rps)
 731 {
 732         struct ni_ps *ps = rps->ps_priv;
 733
 734         return ps;
 735 }
 736
 737 static void ni_calculate_leakage_for_v_and_t_formula(const struct ni_leakage_coeffients *coeff,
 738                                                      u16 v, s32 t,
 739                                                      u32 ileakage,
 740                                                      u32 *leakage)
 741 {
 742         s64 kt, kv, leakage_w, i_leakage, vddc, temperature;
 743
 744         i_leakage = div64_s64(drm_int2fixp(ileakage), 1000);
 745         vddc = div64_s64(drm_int2fixp(v), 1000);
 746         temperature = div64_s64(drm_int2fixp(t), 1000);
 747
 748         kt = drm_fixp_mul(div64_s64(drm_int2fixp(coeff->at), 1000),
 749                           drm_fixp_exp(drm_fixp_mul(div64_s64(drm_int2fixp(coeff->bt), 1000), temperature)));
 750         kv = drm_fixp_mul(div64_s64(drm_int2fixp(coeff->av), 1000),
 751                           drm_fixp_exp(drm_fixp_mul(div64_s64(drm_int2fixp(coeff->bv), 1000), vddc)));
 752
 753         leakage_w = drm_fixp_mul(drm_fixp_mul(drm_fixp_mul(i_leakage, kt), kv), vddc);
 754
 755         *leakage = drm_fixp2int(leakage_w * 1000);
 756 }
 757
 758 static void ni_calculate_leakage_for_v_and_t(struct radeon_device *rdev,
 759                                              const struct ni_leakage_coeffients *coeff,
 760                                              u16 v,
 761                                              s32 t,
 762                                              u32 i_leakage,
 763                                              u32 *leakage)
 764 {
 765         ni_calculate_leakage_for_v_and_t_formula(coeff, v, t, i_leakage, leakage);
 766 }
 767
 768 static void ni_apply_state_adjust_rules(struct radeon_device *rdev,
 769                                         struct radeon_ps *rps)
 770 {
 771         struct ni_ps *ps = ni_get_ps(rps);
 772         struct radeon_clock_and_voltage_limits *max_limits;
 773         bool disable_mclk_switching;
 774         u32 mclk, sclk;
 775         u16 vddc, vddci;
 776         int i;
 777
 778         if (rdev->pm.dpm.new_active_crtc_count > 1)
 779                 disable_mclk_switching = true;
 780         else
 781                 disable_mclk_switching = false;
 782
 783         if (rdev->pm.dpm.ac_power)
 784                 max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
 785         else
 786                 max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc;
 787
 788         if (rdev->pm.dpm.ac_power == false) {
 789                 for (i = 0; i < ps->performance_level_count; i++) {
 790                         if (ps->performance_levels[i].mclk > max_limits->mclk)
 791                                 ps->performance_levels[i].mclk = max_limits->mclk;
 792                         if (ps->performance_levels[i].sclk > max_limits->sclk)
 793                                 ps->performance_levels[i].sclk = max_limits->sclk;
 794                         if (ps->performance_levels[i].vddc > max_limits->vddc)
 795                                 ps->performance_levels[i].vddc = max_limits->vddc;
 796                         if (ps->performance_levels[i].vddci > max_limits->vddci)
 797                                 ps->performance_levels[i].vddci = max_limits->vddci;
 798                 }
 799         }
 800
 801         /* XXX validate the min clocks required for display */
 802
 803         if (disable_mclk_switching) {
 804                 mclk  = ps->performance_levels[ps->performance_level_count - 1].mclk;
 805                 sclk = ps->performance_levels[0].sclk;
 806                 vddc = ps->performance_levels[0].vddc;
 807                 vddci = ps->performance_levels[ps->performance_level_count - 1].vddci;
 808         } else {
 809                 sclk = ps->performance_levels[0].sclk;
 810                 mclk = ps->performance_levels[0].mclk;
 811                 vddc = ps->performance_levels[0].vddc;
 812                 vddci = ps->performance_levels[0].vddci;
 813         }
 814
 815         /* adjusted low state */
 816         ps->performance_levels[0].sclk = sclk;
 817         ps->performance_levels[0].mclk = mclk;
 818         ps->performance_levels[0].vddc = vddc;
 819         ps->performance_levels[0].vddci = vddci;
 820
 821         btc_skip_blacklist_clocks(rdev, max_limits->sclk, max_limits->mclk,
 822                                   &ps->performance_levels[0].sclk,
 823                                   &ps->performance_levels[0].mclk);
 824
 825         for (i = 1; i < ps->performance_level_count; i++) {
 826                 if (ps->performance_levels[i].sclk < ps->performance_levels[i - 1].sclk)
 827                         ps->performance_levels[i].sclk = ps->performance_levels[i - 1].sclk;
 828                 if (ps->performance_levels[i].vddc < ps->performance_levels[i - 1].vddc)
 829                         ps->performance_levels[i].vddc = ps->performance_levels[i - 1].vddc;
 830         }
 831
 832         if (disable_mclk_switching) {
 833                 mclk = ps->performance_levels[0].mclk;
 834                 for (i = 1; i < ps->performance_level_count; i++) {
 835                         if (mclk < ps->performance_levels[i].mclk)
 836                                 mclk = ps->performance_levels[i].mclk;
 837                 }
 838                 for (i = 0; i < ps->performance_level_count; i++) {
 839                         ps->performance_levels[i].mclk = mclk;
 840                         ps->performance_levels[i].vddci = vddci;
 841                 }
 842         } else {
 843                 for (i = 1; i < ps->performance_level_count; i++) {
 844                         if (ps->performance_levels[i].mclk < ps->performance_levels[i - 1].mclk)
 845                                 ps->performance_levels[i].mclk = ps->performance_levels[i - 1].mclk;
 846                         if (ps->performance_levels[i].vddci < ps->performance_levels[i - 1].vddci)
 847                                 ps->performance_levels[i].vddci = ps->performance_levels[i - 1].vddci;
 848                 }
 849         }
 850
 851         for (i = 1; i < ps->performance_level_count; i++)
 852                 btc_skip_blacklist_clocks(rdev, max_limits->sclk, max_limits->mclk,
 853                                           &ps->performance_levels[i].sclk,
 854                                           &ps->performance_levels[i].mclk);
 855
 856         for (i = 0; i < ps->performance_level_count; i++)
 857                 btc_adjust_clock_combinations(rdev, max_limits,
 858                                               &ps->performance_levels[i]);
 859
 860         for (i = 0; i < ps->performance_level_count; i++) {
 861                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk,
 862                                                    ps->performance_levels[i].sclk,
 863                                                    max_limits->vddc,  &ps->performance_levels[i].vddc);
 864                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk,
 865                                                    ps->performance_levels[i].mclk,
 866                                                    max_limits->vddci, &ps->performance_levels[i].vddci);
 867                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk,
 868                                                    ps->performance_levels[i].mclk,
 869                                                    max_limits->vddc,  &ps->performance_levels[i].vddc);
 870                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk,
 871                                                    rdev->clock.current_dispclk,
 872                                                    max_limits->vddc,  &ps->performance_levels[i].vddc);
 873         }
 874
 875         for (i = 0; i < ps->performance_level_count; i++) {
 876                 btc_apply_voltage_delta_rules(rdev,
 877                                               max_limits->vddc, max_limits->vddci,
 878                                               &ps->performance_levels[i].vddc,
 879                                               &ps->performance_levels[i].vddci);
 880         }
 881
 882         ps->dc_compatible = true;
 883         for (i = 0; i < ps->performance_level_count; i++) {
 884                 if (ps->performance_levels[i].vddc > rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.vddc)
 885                         ps->dc_compatible = false;
 886
 887                 if (ps->performance_levels[i].vddc < rdev->pm.dpm.dyn_state.min_vddc_for_pcie_gen2)
 888                         ps->performance_levels[i].flags &= ~ATOM_PPLIB_R600_FLAGS_PCIEGEN2;
 889         }
 890 }
 891
 892 static void ni_cg_clockgating_default(struct radeon_device *rdev)
 893 {
 894         u32 count;
 895         const u32 *ps = NULL;
 896
 897         ps = (const u32 *)&cayman_cgcg_cgls_default;
 898         count = CAYMAN_CGCG_CGLS_DEFAULT_LENGTH;
 899
 900         btc_program_mgcg_hw_sequence(rdev, ps, count);
 901 }
 902
 903 static void ni_gfx_clockgating_enable(struct radeon_device *rdev,
 904                                       bool enable)
 905 {
 906         u32 count;
 907         const u32 *ps = NULL;
 908
 909         if (enable) {
 910                 ps = (const u32 *)&cayman_cgcg_cgls_enable;
 911                 count = CAYMAN_CGCG_CGLS_ENABLE_LENGTH;
 912         } else {
 913                 ps = (const u32 *)&cayman_cgcg_cgls_disable;
 914                 count = CAYMAN_CGCG_CGLS_DISABLE_LENGTH;
 915         }
 916
 917         btc_program_mgcg_hw_sequence(rdev, ps, count);
 918 }
 919
 920 static void ni_mg_clockgating_default(struct radeon_device *rdev)
 921 {
 922         u32 count;
 923         const u32 *ps = NULL;
 924
 925         ps = (const u32 *)&cayman_mgcg_default;
 926         count = CAYMAN_MGCG_DEFAULT_LENGTH;
 927
 928         btc_program_mgcg_hw_sequence(rdev, ps, count);
 929 }
 930
 931 static void ni_mg_clockgating_enable(struct radeon_device *rdev,
 932                                      bool enable)
 933 {
 934         u32 count;
 935         const u32 *ps = NULL;
 936
 937         if (enable) {
 938                 ps = (const u32 *)&cayman_mgcg_enable;
 939                 count = CAYMAN_MGCG_ENABLE_LENGTH;
 940         } else {
 941                 ps = (const u32 *)&cayman_mgcg_disable;
 942                 count = CAYMAN_MGCG_DISABLE_LENGTH;
 943         }
 944
 945         btc_program_mgcg_hw_sequence(rdev, ps, count);
 946 }
 947
 948 static void ni_ls_clockgating_default(struct radeon_device *rdev)
 949 {
 950         u32 count;
 951         const u32 *ps = NULL;
 952
 953         ps = (const u32 *)&cayman_sysls_default;
 954         count = CAYMAN_SYSLS_DEFAULT_LENGTH;
 955
 956         btc_program_mgcg_hw_sequence(rdev, ps, count);
 957 }
 958
 959 static void ni_ls_clockgating_enable(struct radeon_device *rdev,
 960                                      bool enable)
 961 {
 962         u32 count;
 963         const u32 *ps = NULL;
 964
 965         if (enable) {
 966                 ps = (const u32 *)&cayman_sysls_enable;
 967                 count = CAYMAN_SYSLS_ENABLE_LENGTH;
 968         } else {
 969                 ps = (const u32 *)&cayman_sysls_disable;
 970                 count = CAYMAN_SYSLS_DISABLE_LENGTH;
 971         }
 972
 973         btc_program_mgcg_hw_sequence(rdev, ps, count);
 974
 975 }
 976
 977 static int ni_patch_single_dependency_table_based_on_leakage(struct radeon_device *rdev,
 978                                                              struct radeon_clock_voltage_dependency_table *table)
 979 {
 980         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
 981         u32 i;
 982
 983         if (table) {
 984                 for (i = 0; i < table->count; i++) {
 985                         if (0xff01 == table->entries[i].v) {
 986                                 if (pi->max_vddc == 0)
 987                                         return -EINVAL;
 988                                 table->entries[i].v = pi->max_vddc;
 989                         }
 990                 }
 991         }
 992         return 0;
 993 }
 994
 995 static int ni_patch_dependency_tables_based_on_leakage(struct radeon_device *rdev)
 996 {
 997         int ret = 0;
 998
 999         ret = ni_patch_single_dependency_table_based_on_leakage(rdev,
1000                                                                 &rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk);
1001
1002         ret = ni_patch_single_dependency_table_based_on_leakage(rdev,
1003                                                                 &rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk);
1004         return ret;
1005 }
1006
1007 static void ni_stop_dpm(struct radeon_device *rdev)
1008 {
1009         WREG32_P(GENERAL_PWRMGT, 0, ~GLOBAL_PWRMGT_EN);
1010 }
1011
1012 #if 0
1013 static int ni_notify_hw_of_power_source(struct radeon_device *rdev,
1014                                         bool ac_power)
1015 {
1016         if (ac_power)
1017                 return (rv770_send_msg_to_smc(rdev, PPSMC_MSG_RunningOnAC) == PPSMC_Result_OK) ?
1018                         0 : -EINVAL;
1019
1020         return 0;
1021 }
1022 #endif
1023
1024 static PPSMC_Result ni_send_msg_to_smc_with_parameter(struct radeon_device *rdev,
1025                                                       PPSMC_Msg msg, u32 parameter)
1026 {
1027         WREG32(SMC_SCRATCH0, parameter);
1028         return rv770_send_msg_to_smc(rdev, msg);
1029 }
1030
1031 static int ni_restrict_performance_levels_before_switch(struct radeon_device *rdev)
1032 {
1033         if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_NoForcedLevel) != PPSMC_Result_OK)
1034                 return -EINVAL;
1035
1036         return (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 1) == PPSMC_Result_OK) ?
1037                 0 : -EINVAL;
1038 }
1039
1040 int ni_dpm_force_performance_level(struct radeon_device *rdev,
1041                                    enum radeon_dpm_forced_level level)
1042 {
1043         struct radeon_ps *rps = rdev->pm.dpm.current_ps;
1044         struct ni_ps *ps = ni_get_ps(rps);
1045         u32 levels;
1046
1047         if (level == RADEON_DPM_FORCED_LEVEL_HIGH) {
1048                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 0) != PPSMC_Result_OK)
1049                         return -EINVAL;
1050
1051                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 1) != PPSMC_Result_OK)
1052                         return -EINVAL;
1053         } else if (level == RADEON_DPM_FORCED_LEVEL_LOW) {
1054                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 0) != PPSMC_Result_OK)
1055                         return -EINVAL;
1056
1057                 levels = ps->performance_level_count - 1;
1058                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, levels) != PPSMC_Result_OK)
1059                         return -EINVAL;
1060         } else if (level == RADEON_DPM_FORCED_LEVEL_AUTO) {
1061                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 0) != PPSMC_Result_OK)
1062                         return -EINVAL;
1063
1064                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 0) != PPSMC_Result_OK)
1065                         return -EINVAL;
1066         }
1067
1068         rdev->pm.dpm.forced_level = level;
1069
1070         return 0;
1071 }
1072
1073 static void ni_stop_smc(struct radeon_device *rdev)
1074 {
1075         u32 tmp;
1076         int i;
1077
1078         for (i = 0; i < rdev->usec_timeout; i++) {
1079                 tmp = RREG32(LB_SYNC_RESET_SEL) & LB_SYNC_RESET_SEL_MASK;
1080                 if (tmp != 1)
1081                         break;
1082                 udelay(1);
1083         }
1084
1085         udelay(100);
1086
1087         r7xx_stop_smc(rdev);
1088 }
1089
1090 static int ni_process_firmware_header(struct radeon_device *rdev)
1091 {
1092         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1093         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1094         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1095         u32 tmp;
1096         int ret;
1097
1098         ret = rv770_read_smc_sram_dword(rdev,
1099                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1100                                         NISLANDS_SMC_FIRMWARE_HEADER_stateTable,
1101                                         &tmp, pi->sram_end);
1102
1103         if (ret)
1104                 return ret;
1105
1106         pi->state_table_start = (u16)tmp;
1107
1108         ret = rv770_read_smc_sram_dword(rdev,
1109                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1110                                         NISLANDS_SMC_FIRMWARE_HEADER_softRegisters,
1111                                         &tmp, pi->sram_end);
1112
1113         if (ret)
1114                 return ret;
1115
1116         pi->soft_regs_start = (u16)tmp;
1117
1118         ret = rv770_read_smc_sram_dword(rdev,
1119                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1120                                         NISLANDS_SMC_FIRMWARE_HEADER_mcRegisterTable,
1121                                         &tmp, pi->sram_end);
1122
1123         if (ret)
1124                 return ret;
1125
1126         eg_pi->mc_reg_table_start = (u16)tmp;
1127
1128         ret = rv770_read_smc_sram_dword(rdev,
1129                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1130                                         NISLANDS_SMC_FIRMWARE_HEADER_fanTable,
1131                                         &tmp, pi->sram_end);
1132
1133         if (ret)
1134                 return ret;
1135
1136         ni_pi->fan_table_start = (u16)tmp;
1137
1138         ret = rv770_read_smc_sram_dword(rdev,
1139                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1140                                         NISLANDS_SMC_FIRMWARE_HEADER_mcArbDramAutoRefreshTable,
1141                                         &tmp, pi->sram_end);
1142
1143         if (ret)
1144                 return ret;
1145
1146         ni_pi->arb_table_start = (u16)tmp;
1147
1148         ret = rv770_read_smc_sram_dword(rdev,
1149                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1150                                         NISLANDS_SMC_FIRMWARE_HEADER_cacTable,
1151                                         &tmp, pi->sram_end);
1152
1153         if (ret)
1154                 return ret;
1155
1156         ni_pi->cac_table_start = (u16)tmp;
1157
1158         ret = rv770_read_smc_sram_dword(rdev,
1159                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1160                                         NISLANDS_SMC_FIRMWARE_HEADER_spllTable,
1161                                         &tmp, pi->sram_end);
1162
1163         if (ret)
1164                 return ret;
1165
1166         ni_pi->spll_table_start = (u16)tmp;
1167
1168
1169         return ret;
1170 }
1171
1172 static void ni_read_clock_registers(struct radeon_device *rdev)
1173 {
1174         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1175
1176         ni_pi->clock_registers.cg_spll_func_cntl = RREG32(CG_SPLL_FUNC_CNTL);
1177         ni_pi->clock_registers.cg_spll_func_cntl_2 = RREG32(CG_SPLL_FUNC_CNTL_2);
1178         ni_pi->clock_registers.cg_spll_func_cntl_3 = RREG32(CG_SPLL_FUNC_CNTL_3);
1179         ni_pi->clock_registers.cg_spll_func_cntl_4 = RREG32(CG_SPLL_FUNC_CNTL_4);
1180         ni_pi->clock_registers.cg_spll_spread_spectrum = RREG32(CG_SPLL_SPREAD_SPECTRUM);
1181         ni_pi->clock_registers.cg_spll_spread_spectrum_2 = RREG32(CG_SPLL_SPREAD_SPECTRUM_2);
1182         ni_pi->clock_registers.mpll_ad_func_cntl = RREG32(MPLL_AD_FUNC_CNTL);
1183         ni_pi->clock_registers.mpll_ad_func_cntl_2 = RREG32(MPLL_AD_FUNC_CNTL_2);
1184         ni_pi->clock_registers.mpll_dq_func_cntl = RREG32(MPLL_DQ_FUNC_CNTL);
1185         ni_pi->clock_registers.mpll_dq_func_cntl_2 = RREG32(MPLL_DQ_FUNC_CNTL_2);
1186         ni_pi->clock_registers.mclk_pwrmgt_cntl = RREG32(MCLK_PWRMGT_CNTL);
1187         ni_pi->clock_registers.dll_cntl = RREG32(DLL_CNTL);
1188         ni_pi->clock_registers.mpll_ss1 = RREG32(MPLL_SS1);
1189         ni_pi->clock_registers.mpll_ss2 = RREG32(MPLL_SS2);
1190 }
1191
1192 #if 0
1193 static int ni_enter_ulp_state(struct radeon_device *rdev)
1194 {
1195         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1196
1197         if (pi->gfx_clock_gating) {
1198                 WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN);
1199                 WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON);
1200                 WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON);
1201                 RREG32(GB_ADDR_CONFIG);
1202         }
1203
1204         WREG32_P(SMC_MSG, HOST_SMC_MSG(PPSMC_MSG_SwitchToMinimumPower),
1205                  ~HOST_SMC_MSG_MASK);
1206
1207         udelay(25000);
1208
1209         return 0;
1210 }
1211 #endif
1212
1213 static void ni_program_response_times(struct radeon_device *rdev)
1214 {
1215         u32 voltage_response_time, backbias_response_time, acpi_delay_time, vbi_time_out;
1216         u32 vddc_dly, bb_dly, acpi_dly, vbi_dly, mclk_switch_limit;
1217         u32 reference_clock;
1218
1219         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mvdd_chg_time, 1);
1220
1221         voltage_response_time = (u32)rdev->pm.dpm.voltage_response_time;
1222         backbias_response_time = (u32)rdev->pm.dpm.backbias_response_time;
1223
1224         if (voltage_response_time == 0)
1225                 voltage_response_time = 1000;
1226
1227         if (backbias_response_time == 0)
1228                 backbias_response_time = 1000;
1229
1230         acpi_delay_time = 15000;
1231         vbi_time_out = 100000;
1232
1233         reference_clock = radeon_get_xclk(rdev);
1234
1235         vddc_dly = (voltage_response_time  * reference_clock) / 1600;
1236         bb_dly   = (backbias_response_time * reference_clock) / 1600;
1237         acpi_dly = (acpi_delay_time * reference_clock) / 1600;
1238         vbi_dly  = (vbi_time_out * reference_clock) / 1600;
1239
1240         mclk_switch_limit = (460 * reference_clock) / 100;
1241
1242         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_vreg,  vddc_dly);
1243         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_bbias, bb_dly);
1244         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_acpi,  acpi_dly);
1245         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mclk_chg_timeout, vbi_dly);
1246         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mc_block_delay, 0xAA);
1247         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mclk_switch_lim, mclk_switch_limit);
1248 }
1249
1250 static void ni_populate_smc_voltage_table(struct radeon_device *rdev,
1251                                           struct atom_voltage_table *voltage_table,
1252                                           NISLANDS_SMC_STATETABLE *table)
1253 {
1254         unsigned int i;
1255
1256         for (i = 0; i < voltage_table->count; i++) {
1257                 table->highSMIO[i] = 0;
1258                 table->lowSMIO[i] |= cpu_to_be32(voltage_table->entries[i].smio_low);
1259         }
1260 }
1261
1262 static void ni_populate_smc_voltage_tables(struct radeon_device *rdev,
1263                                            NISLANDS_SMC_STATETABLE *table)
1264 {
1265         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1266         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1267         unsigned char i;
1268
1269         if (eg_pi->vddc_voltage_table.count) {
1270                 ni_populate_smc_voltage_table(rdev, &eg_pi->vddc_voltage_table, table);
1271                 table->voltageMaskTable.highMask[NISLANDS_SMC_VOLTAGEMASK_VDDC] = 0;
1272                 table->voltageMaskTable.lowMask[NISLANDS_SMC_VOLTAGEMASK_VDDC] =
1273                         cpu_to_be32(eg_pi->vddc_voltage_table.mask_low);
1274
1275                 for (i = 0; i < eg_pi->vddc_voltage_table.count; i++) {
1276                         if (pi->max_vddc_in_table <= eg_pi->vddc_voltage_table.entries[i].value) {
1277                                 table->maxVDDCIndexInPPTable = i;
1278                                 break;
1279                         }
1280                 }
1281         }
1282
1283         if (eg_pi->vddci_voltage_table.count) {
1284                 ni_populate_smc_voltage_table(rdev, &eg_pi->vddci_voltage_table, table);
1285
1286                 table->voltageMaskTable.highMask[NISLANDS_SMC_VOLTAGEMASK_VDDCI] = 0;
1287                 table->voltageMaskTable.lowMask[NISLANDS_SMC_VOLTAGEMASK_VDDCI] =
1288                         cpu_to_be32(eg_pi->vddc_voltage_table.mask_low);
1289         }
1290 }
1291
1292 static int ni_populate_voltage_value(struct radeon_device *rdev,
1293                                      struct atom_voltage_table *table,
1294                                      u16 value,
1295                                      NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1296 {
1297         unsigned int i;
1298
1299         for (i = 0; i < table->count; i++) {
1300                 if (value <= table->entries[i].value) {
1301                         voltage->index = (u8)i;
1302                         voltage->value = cpu_to_be16(table->entries[i].value);
1303                         break;
1304                 }
1305         }
1306
1307         if (i >= table->count)
1308                 return -EINVAL;
1309
1310         return 0;
1311 }
1312
1313 static void ni_populate_mvdd_value(struct radeon_device *rdev,
1314                                    u32 mclk,
1315                                    NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1316 {
1317         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1318         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1319
1320         if (!pi->mvdd_control) {
1321                 voltage->index = eg_pi->mvdd_high_index;
1322                 voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1323                 return;
1324         }
1325
1326         if (mclk <= pi->mvdd_split_frequency) {
1327                 voltage->index = eg_pi->mvdd_low_index;
1328                 voltage->value = cpu_to_be16(MVDD_LOW_VALUE);
1329         } else {
1330                 voltage->index = eg_pi->mvdd_high_index;
1331                 voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1332         }
1333 }
1334
1335 static int ni_get_std_voltage_value(struct radeon_device *rdev,
1336                                     NISLANDS_SMC_VOLTAGE_VALUE *voltage,
1337                                     u16 *std_voltage)
1338 {
1339         if (rdev->pm.dpm.dyn_state.cac_leakage_table.entries &&
1340             ((u32)voltage->index < rdev->pm.dpm.dyn_state.cac_leakage_table.count))
1341                 *std_voltage = rdev->pm.dpm.dyn_state.cac_leakage_table.entries[voltage->index].vddc;
1342         else
1343                 *std_voltage = be16_to_cpu(voltage->value);
1344
1345         return 0;
1346 }
1347
1348 static void ni_populate_std_voltage_value(struct radeon_device *rdev,
1349                                           u16 value, u8 index,
1350                                           NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1351 {
1352         voltage->index = index;
1353         voltage->value = cpu_to_be16(value);
1354 }
1355
1356 static u32 ni_get_smc_power_scaling_factor(struct radeon_device *rdev)
1357 {
1358         u32 xclk_period;
1359         u32 xclk = radeon_get_xclk(rdev);
1360         u32 tmp = RREG32(CG_CAC_CTRL) & TID_CNT_MASK;
1361
1362         xclk_period = (1000000000UL / xclk);
1363         xclk_period /= 10000UL;
1364
1365         return tmp * xclk_period;
1366 }
1367
1368 static u32 ni_scale_power_for_smc(u32 power_in_watts, u32 scaling_factor)
1369 {
1370         return (power_in_watts * scaling_factor) << 2;
1371 }
1372
1373 static u32 ni_calculate_power_boost_limit(struct radeon_device *rdev,
1374                                           struct radeon_ps *radeon_state,
1375                                           u32 near_tdp_limit)
1376 {
1377         struct ni_ps *state = ni_get_ps(radeon_state);
1378         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1379         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1380         u32 power_boost_limit = 0;
1381         int ret;
1382
1383         if (ni_pi->enable_power_containment &&
1384             ni_pi->use_power_boost_limit) {
1385                 NISLANDS_SMC_VOLTAGE_VALUE vddc;
1386                 u16 std_vddc_med;
1387                 u16 std_vddc_high;
1388                 u64 tmp, n, d;
1389
1390                 if (state->performance_level_count < 3)
1391                         return 0;
1392
1393                 ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
1394                                                 state->performance_levels[state->performance_level_count - 2].vddc,
1395                                                 &vddc);
1396                 if (ret)
1397                         return 0;
1398
1399                 ret = ni_get_std_voltage_value(rdev, &vddc, &std_vddc_med);
1400                 if (ret)
1401                         return 0;
1402
1403                 ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
1404                                                 state->performance_levels[state->performance_level_count - 1].vddc,
1405                                                 &vddc);
1406                 if (ret)
1407                         return 0;
1408
1409                 ret = ni_get_std_voltage_value(rdev, &vddc, &std_vddc_high);
1410                 if (ret)
1411                         return 0;
1412
1413                 n = ((u64)near_tdp_limit * ((u64)std_vddc_med * (u64)std_vddc_med) * 90);
1414                 d = ((u64)std_vddc_high * (u64)std_vddc_high * 100);
1415                 tmp = div64_u64(n, d);
1416
1417                 if (tmp >> 32)
1418                         return 0;
1419                 power_boost_limit = (u32)tmp;
1420         }
1421
1422         return power_boost_limit;
1423 }
1424
1425 static int ni_calculate_adjusted_tdp_limits(struct radeon_device *rdev,
1426                                             bool adjust_polarity,
1427                                             u32 tdp_adjustment,
1428                                             u32 *tdp_limit,
1429                                             u32 *near_tdp_limit)
1430 {
1431         if (tdp_adjustment > (u32)rdev->pm.dpm.tdp_od_limit)
1432                 return -EINVAL;
1433
1434         if (adjust_polarity) {
1435                 *tdp_limit = ((100 + tdp_adjustment) * rdev->pm.dpm.tdp_limit) / 100;
1436                 *near_tdp_limit = rdev->pm.dpm.near_tdp_limit + (*tdp_limit - rdev->pm.dpm.tdp_limit);
1437         } else {
1438                 *tdp_limit = ((100 - tdp_adjustment) * rdev->pm.dpm.tdp_limit) / 100;
1439                 *near_tdp_limit = rdev->pm.dpm.near_tdp_limit - (rdev->pm.dpm.tdp_limit - *tdp_limit);
1440         }
1441
1442         return 0;
1443 }
1444
1445 static int ni_populate_smc_tdp_limits(struct radeon_device *rdev,
1446                                       struct radeon_ps *radeon_state)
1447 {
1448         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1449         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1450
1451         if (ni_pi->enable_power_containment) {
1452                 NISLANDS_SMC_STATETABLE *smc_table = &ni_pi->smc_statetable;
1453                 u32 scaling_factor = ni_get_smc_power_scaling_factor(rdev);
1454                 u32 tdp_limit;
1455                 u32 near_tdp_limit;
1456                 u32 power_boost_limit;
1457                 int ret;
1458
1459                 if (scaling_factor == 0)
1460                         return -EINVAL;
1461
1462                 memset(smc_table, 0, sizeof(NISLANDS_SMC_STATETABLE));
1463
1464                 ret = ni_calculate_adjusted_tdp_limits(rdev,
1465                                                        false, /* ??? */
1466                                                        rdev->pm.dpm.tdp_adjustment,
1467                                                        &tdp_limit,
1468                                                        &near_tdp_limit);
1469                 if (ret)
1470                         return ret;
1471
1472                 power_boost_limit = ni_calculate_power_boost_limit(rdev, radeon_state,
1473                                                                    near_tdp_limit);
1474
1475                 smc_table->dpm2Params.TDPLimit =
1476                         cpu_to_be32(ni_scale_power_for_smc(tdp_limit, scaling_factor));
1477                 smc_table->dpm2Params.NearTDPLimit =
1478                         cpu_to_be32(ni_scale_power_for_smc(near_tdp_limit, scaling_factor));
1479                 smc_table->dpm2Params.SafePowerLimit =
1480                         cpu_to_be32(ni_scale_power_for_smc((near_tdp_limit * NISLANDS_DPM2_TDP_SAFE_LIMIT_PERCENT) / 100,
1481                                                            scaling_factor));
1482                 smc_table->dpm2Params.PowerBoostLimit =
1483                         cpu_to_be32(ni_scale_power_for_smc(power_boost_limit, scaling_factor));
1484
1485                 ret = rv770_copy_bytes_to_smc(rdev,
1486                                               (u16)(pi->state_table_start + offsetof(NISLANDS_SMC_STATETABLE, dpm2Params) +
1487                                                     offsetof(PP_NIslands_DPM2Parameters, TDPLimit)),
1488                                               (u8 *)(&smc_table->dpm2Params.TDPLimit),
1489                                               sizeof(u32) * 4, pi->sram_end);
1490                 if (ret)
1491                         return ret;
1492         }
1493
1494         return 0;
1495 }
1496
1497 int ni_copy_and_switch_arb_sets(struct radeon_device *rdev,
1498                                 u32 arb_freq_src, u32 arb_freq_dest)
1499 {
1500         u32 mc_arb_dram_timing;
1501         u32 mc_arb_dram_timing2;
1502         u32 burst_time;
1503         u32 mc_cg_config;
1504
1505         switch (arb_freq_src) {
1506         case MC_CG_ARB_FREQ_F0:
1507                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING);
1508                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2);
1509                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE0_MASK) >> STATE0_SHIFT;
1510                 break;
1511         case MC_CG_ARB_FREQ_F1:
1512                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_1);
1513                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_1);
1514                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE1_MASK) >> STATE1_SHIFT;
1515                 break;
1516         case MC_CG_ARB_FREQ_F2:
1517                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_2);
1518                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_2);
1519                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE2_MASK) >> STATE2_SHIFT;
1520                 break;
1521         case MC_CG_ARB_FREQ_F3:
1522                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_3);
1523                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_3);
1524                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE3_MASK) >> STATE3_SHIFT;
1525                 break;
1526         default:
1527                 return -EINVAL;
1528         }
1529
1530         switch (arb_freq_dest) {
1531         case MC_CG_ARB_FREQ_F0:
1532                 WREG32(MC_ARB_DRAM_TIMING, mc_arb_dram_timing);
1533                 WREG32(MC_ARB_DRAM_TIMING2, mc_arb_dram_timing2);
1534                 WREG32_P(MC_ARB_BURST_TIME, STATE0(burst_time), ~STATE0_MASK);
1535                 break;
1536         case MC_CG_ARB_FREQ_F1:
1537                 WREG32(MC_ARB_DRAM_TIMING_1, mc_arb_dram_timing);
1538                 WREG32(MC_ARB_DRAM_TIMING2_1, mc_arb_dram_timing2);
1539                 WREG32_P(MC_ARB_BURST_TIME, STATE1(burst_time), ~STATE1_MASK);
1540                 break;
1541         case MC_CG_ARB_FREQ_F2:
1542                 WREG32(MC_ARB_DRAM_TIMING_2, mc_arb_dram_timing);
1543                 WREG32(MC_ARB_DRAM_TIMING2_2, mc_arb_dram_timing2);
1544                 WREG32_P(MC_ARB_BURST_TIME, STATE2(burst_time), ~STATE2_MASK);
1545                 break;
1546         case MC_CG_ARB_FREQ_F3:
1547                 WREG32(MC_ARB_DRAM_TIMING_3, mc_arb_dram_timing);
1548                 WREG32(MC_ARB_DRAM_TIMING2_3, mc_arb_dram_timing2);
1549                 WREG32_P(MC_ARB_BURST_TIME, STATE3(burst_time), ~STATE3_MASK);
1550                 break;
1551         default:
1552                 return -EINVAL;
1553         }
1554
1555         mc_cg_config = RREG32(MC_CG_CONFIG) | 0x0000000F;
1556         WREG32(MC_CG_CONFIG, mc_cg_config);
1557         WREG32_P(MC_ARB_CG, CG_ARB_REQ(arb_freq_dest), ~CG_ARB_REQ_MASK);
1558
1559         return 0;
1560 }
1561
1562 static int ni_init_arb_table_index(struct radeon_device *rdev)
1563 {
1564         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1565         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1566         u32 tmp;
1567         int ret;
1568
1569         ret = rv770_read_smc_sram_dword(rdev, ni_pi->arb_table_start,
1570                                         &tmp, pi->sram_end);
1571         if (ret)
1572                 return ret;
1573
1574         tmp &= 0x00FFFFFF;
1575         tmp |= ((u32)MC_CG_ARB_FREQ_F1) << 24;
1576
1577         return rv770_write_smc_sram_dword(rdev, ni_pi->arb_table_start,
1578                                           tmp, pi->sram_end);
1579 }
1580
1581 static int ni_initial_switch_from_arb_f0_to_f1(struct radeon_device *rdev)
1582 {
1583         return ni_copy_and_switch_arb_sets(rdev, MC_CG_ARB_FREQ_F0, MC_CG_ARB_FREQ_F1);
1584 }
1585
1586 static int ni_force_switch_to_arb_f0(struct radeon_device *rdev)
1587 {
1588         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1589         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1590         u32 tmp;
1591         int ret;
1592
1593         ret = rv770_read_smc_sram_dword(rdev, ni_pi->arb_table_start,
1594                                         &tmp, pi->sram_end);
1595         if (ret)
1596                 return ret;
1597
1598         tmp = (tmp >> 24) & 0xff;
1599
1600         if (tmp == MC_CG_ARB_FREQ_F0)
1601                 return 0;
1602
1603         return ni_copy_and_switch_arb_sets(rdev, tmp, MC_CG_ARB_FREQ_F0);
1604 }
1605
1606 static int ni_populate_memory_timing_parameters(struct radeon_device *rdev,
1607                                                 struct rv7xx_pl *pl,
1608                                                 SMC_NIslands_MCArbDramTimingRegisterSet *arb_regs)
1609 {
1610         u32 dram_timing;
1611         u32 dram_timing2;
1612
1613         arb_regs->mc_arb_rfsh_rate =
1614                 (u8)rv770_calculate_memory_refresh_rate(rdev, pl->sclk);
1615
1616
1617         radeon_atom_set_engine_dram_timings(rdev,
1618                                             pl->sclk,
1619                                             pl->mclk);
1620
1621         dram_timing = RREG32(MC_ARB_DRAM_TIMING);
1622         dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2);
1623
1624         arb_regs->mc_arb_dram_timing  = cpu_to_be32(dram_timing);
1625         arb_regs->mc_arb_dram_timing2 = cpu_to_be32(dram_timing2);
1626
1627         return 0;
1628 }
1629
1630 static int ni_do_program_memory_timing_parameters(struct radeon_device *rdev,
1631                                                   struct radeon_ps *radeon_state,
1632                                                   unsigned int first_arb_set)
1633 {
1634         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1635         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1636         struct ni_ps *state = ni_get_ps(radeon_state);
1637         SMC_NIslands_MCArbDramTimingRegisterSet arb_regs = { 0 };
1638         int i, ret = 0;
1639
1640         for (i = 0; i < state->performance_level_count; i++) {
1641                 ret = ni_populate_memory_timing_parameters(rdev, &state->performance_levels[i], &arb_regs);
1642                 if (ret)
1643                         break;
1644
1645                 ret = rv770_copy_bytes_to_smc(rdev,
1646                                               (u16)(ni_pi->arb_table_start +
1647                                                     offsetof(SMC_NIslands_MCArbDramTimingRegisters, data) +
1648                                                     sizeof(SMC_NIslands_MCArbDramTimingRegisterSet) * (first_arb_set + i)),
1649                                               (u8 *)&arb_regs,
1650                                               (u16)sizeof(SMC_NIslands_MCArbDramTimingRegisterSet),
1651                                               pi->sram_end);
1652                 if (ret)
1653                         break;
1654         }
1655         return ret;
1656 }
1657
1658 static int ni_program_memory_timing_parameters(struct radeon_device *rdev,
1659                                                struct radeon_ps *radeon_new_state)
1660 {
1661         return ni_do_program_memory_timing_parameters(rdev, radeon_new_state,
1662                                                       NISLANDS_DRIVER_STATE_ARB_INDEX);
1663 }
1664
1665 static void ni_populate_initial_mvdd_value(struct radeon_device *rdev,
1666                                            struct NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1667 {
1668         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1669
1670         voltage->index = eg_pi->mvdd_high_index;
1671         voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1672 }
1673
1674 static int ni_populate_smc_initial_state(struct radeon_device *rdev,
1675                                          struct radeon_ps *radeon_initial_state,
1676                                          NISLANDS_SMC_STATETABLE *table)
1677 {
1678         struct ni_ps *initial_state = ni_get_ps(radeon_initial_state);
1679         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1680         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1681         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1682         u32 reg;
1683         int ret;
1684
1685         table->initialState.levels[0].mclk.vMPLL_AD_FUNC_CNTL =
1686                 cpu_to_be32(ni_pi->clock_registers.mpll_ad_func_cntl);
1687         table->initialState.levels[0].mclk.vMPLL_AD_FUNC_CNTL_2 =
1688                 cpu_to_be32(ni_pi->clock_registers.mpll_ad_func_cntl_2);
1689         table->initialState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL =
1690                 cpu_to_be32(ni_pi->clock_registers.mpll_dq_func_cntl);
1691         table->initialState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL_2 =
1692                 cpu_to_be32(ni_pi->clock_registers.mpll_dq_func_cntl_2);
1693         table->initialState.levels[0].mclk.vMCLK_PWRMGT_CNTL =
1694                 cpu_to_be32(ni_pi->clock_registers.mclk_pwrmgt_cntl);
1695         table->initialState.levels[0].mclk.vDLL_CNTL =
1696                 cpu_to_be32(ni_pi->clock_registers.dll_cntl);
1697         table->initialState.levels[0].mclk.vMPLL_SS =
1698                 cpu_to_be32(ni_pi->clock_registers.mpll_ss1);
1699         table->initialState.levels[0].mclk.vMPLL_SS2 =
1700                 cpu_to_be32(ni_pi->clock_registers.mpll_ss2);
1701         table->initialState.levels[0].mclk.mclk_value =
1702                 cpu_to_be32(initial_state->performance_levels[0].mclk);
1703
1704         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL =
1705                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl);
1706         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 =
1707                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_2);
1708         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 =
1709                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_3);
1710         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_4 =
1711                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_4);
1712         table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM =
1713                 cpu_to_be32(ni_pi->clock_registers.cg_spll_spread_spectrum);
1714         table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM_2 =
1715                 cpu_to_be32(ni_pi->clock_registers.cg_spll_spread_spectrum_2);
1716         table->initialState.levels[0].sclk.sclk_value =
1717                 cpu_to_be32(initial_state->performance_levels[0].sclk);
1718         table->initialState.levels[0].arbRefreshState =
1719                 NISLANDS_INITIAL_STATE_ARB_INDEX;
1720
1721         table->initialState.levels[0].ACIndex = 0;
1722
1723         ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
1724                                         initial_state->performance_levels[0].vddc,
1725                                         &table->initialState.levels[0].vddc);
1726         if (!ret) {
1727                 u16 std_vddc;
1728
1729                 ret = ni_get_std_voltage_value(rdev,
1730                                                &table->initialState.levels[0].vddc,
1731                                                &std_vddc);
1732                 if (!ret)
1733                         ni_populate_std_voltage_value(rdev, std_vddc,
1734                                                       table->initialState.levels[0].vddc.index,
1735                                                       &table->initialState.levels[0].std_vddc);
1736         }
1737
1738         if (eg_pi->vddci_control)
1739                 ni_populate_voltage_value(rdev,
1740                                           &eg_pi->vddci_voltage_table,
1741                                           initial_state->performance_levels[0].vddci,
1742                                           &table->initialState.levels[0].vddci);
1743
1744         ni_populate_initial_mvdd_value(rdev, &table->initialState.levels[0].mvdd);
1745
1746         reg = CG_R(0xffff) | CG_L(0);
1747         table->initialState.levels[0].aT = cpu_to_be32(reg);
1748
1749         table->initialState.levels[0].bSP = cpu_to_be32(pi->dsp);
1750
1751         if (pi->boot_in_gen2)
1752                 table->initialState.levels[0].gen2PCIE = 1;
1753         else
1754                 table->initialState.levels[0].gen2PCIE = 0;
1755
1756         if (pi->mem_gddr5) {
1757                 table->initialState.levels[0].strobeMode =
1758                         cypress_get_strobe_mode_settings(rdev,
1759                                                          initial_state->performance_levels[0].mclk);
1760
1761                 if (initial_state->performance_levels[0].mclk > pi->mclk_edc_enable_threshold)
1762                         table->initialState.levels[0].mcFlags = NISLANDS_SMC_MC_EDC_RD_FLAG | NISLANDS_SMC_MC_EDC_WR_FLAG;
1763                 else
1764                         table->initialState.levels[0].mcFlags =  0;
1765         }
1766
1767         table->initialState.levelCount = 1;
1768
1769         table->initialState.flags |= PPSMC_SWSTATE_FLAG_DC;
1770
1771         table->initialState.levels[0].dpm2.MaxPS = 0;
1772         table->initialState.levels[0].dpm2.NearTDPDec = 0;
1773         table->initialState.levels[0].dpm2.AboveSafeInc = 0;
1774         table->initialState.levels[0].dpm2.BelowSafeInc = 0;
1775
1776         reg = MIN_POWER_MASK | MAX_POWER_MASK;
1777         table->initialState.levels[0].SQPowerThrottle = cpu_to_be32(reg);
1778
1779         reg = MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
1780         table->initialState.levels[0].SQPowerThrottle_2 = cpu_to_be32(reg);
1781
1782         return 0;
1783 }
1784
1785 static int ni_populate_smc_acpi_state(struct radeon_device *rdev,
1786                                       NISLANDS_SMC_STATETABLE *table)
1787 {
1788         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1789         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1790         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1791         u32 mpll_ad_func_cntl   = ni_pi->clock_registers.mpll_ad_func_cntl;
1792         u32 mpll_ad_func_cntl_2 = ni_pi->clock_registers.mpll_ad_func_cntl_2;
1793         u32 mpll_dq_func_cntl   = ni_pi->clock_registers.mpll_dq_func_cntl;
1794         u32 mpll_dq_func_cntl_2 = ni_pi->clock_registers.mpll_dq_func_cntl_2;
1795         u32 spll_func_cntl      = ni_pi->clock_registers.cg_spll_func_cntl;
1796         u32 spll_func_cntl_2    = ni_pi->clock_registers.cg_spll_func_cntl_2;
1797         u32 spll_func_cntl_3    = ni_pi->clock_registers.cg_spll_func_cntl_3;
1798         u32 spll_func_cntl_4    = ni_pi->clock_registers.cg_spll_func_cntl_4;
1799         u32 mclk_pwrmgt_cntl    = ni_pi->clock_registers.mclk_pwrmgt_cntl;
1800         u32 dll_cntl            = ni_pi->clock_registers.dll_cntl;
1801         u32 reg;
1802         int ret;
1803
1804         table->ACPIState = table->initialState;
1805
1806         table->ACPIState.flags &= ~PPSMC_SWSTATE_FLAG_DC;
1807
1808         if (pi->acpi_vddc) {
1809                 ret = ni_populate_voltage_value(rdev,
1810                                                 &eg_pi->vddc_voltage_table,
1811                                                 pi->acpi_vddc, &table->ACPIState.levels[0].vddc);
1812                 if (!ret) {
1813                         u16 std_vddc;
1814
1815                         ret = ni_get_std_voltage_value(rdev,
1816                                                        &table->ACPIState.levels[0].vddc, &std_vddc);
1817                         if (!ret)
1818                                 ni_populate_std_voltage_value(rdev, std_vddc,
1819                                                               table->ACPIState.levels[0].vddc.index,
1820                                                               &table->ACPIState.levels[0].std_vddc);
1821                 }
1822
1823                 if (pi->pcie_gen2) {
1824                         if (pi->acpi_pcie_gen2)
1825                                 table->ACPIState.levels[0].gen2PCIE = 1;
1826                         else
1827                                 table->ACPIState.levels[0].gen2PCIE = 0;
1828                 } else {
1829                         table->ACPIState.levels[0].gen2PCIE = 0;
1830                 }
1831         } else {
1832                 ret = ni_populate_voltage_value(rdev,
1833                                                 &eg_pi->vddc_voltage_table,
1834                                                 pi->min_vddc_in_table,
1835                                                 &table->ACPIState.levels[0].vddc);
1836                 if (!ret) {
1837                         u16 std_vddc;
1838
1839                         ret = ni_get_std_voltage_value(rdev,
1840                                                        &table->ACPIState.levels[0].vddc,
1841                                                        &std_vddc);
1842                         if (!ret)
1843                                 ni_populate_std_voltage_value(rdev, std_vddc,
1844                                                               table->ACPIState.levels[0].vddc.index,
1845                                                               &table->ACPIState.levels[0].std_vddc);
1846                 }
1847                 table->ACPIState.levels[0].gen2PCIE = 0;
1848         }
1849
1850         if (eg_pi->acpi_vddci) {
1851                 if (eg_pi->vddci_control)
1852                         ni_populate_voltage_value(rdev,
1853                                                   &eg_pi->vddci_voltage_table,
1854                                                   eg_pi->acpi_vddci,
1855                                                   &table->ACPIState.levels[0].vddci);
1856         }
1857
1858
1859         mpll_ad_func_cntl &= ~PDNB;
1860
1861         mpll_ad_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN;
1862
1863         if (pi->mem_gddr5)
1864                 mpll_dq_func_cntl &= ~PDNB;
1865         mpll_dq_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN | BYPASS;
1866
1867
1868         mclk_pwrmgt_cntl |= (MRDCKA0_RESET |
1869                              MRDCKA1_RESET |
1870                              MRDCKB0_RESET |
1871                              MRDCKB1_RESET |
1872                              MRDCKC0_RESET |
1873                              MRDCKC1_RESET |
1874                              MRDCKD0_RESET |
1875                              MRDCKD1_RESET);
1876
1877         mclk_pwrmgt_cntl &= ~(MRDCKA0_PDNB |
1878                               MRDCKA1_PDNB |
1879                               MRDCKB0_PDNB |
1880                               MRDCKB1_PDNB |
1881                               MRDCKC0_PDNB |
1882                               MRDCKC1_PDNB |
1883                               MRDCKD0_PDNB |
1884                               MRDCKD1_PDNB);
1885
1886         dll_cntl |= (MRDCKA0_BYPASS |
1887                      MRDCKA1_BYPASS |
1888                      MRDCKB0_BYPASS |
1889                      MRDCKB1_BYPASS |
1890                      MRDCKC0_BYPASS |
1891                      MRDCKC1_BYPASS |
1892                      MRDCKD0_BYPASS |
1893                      MRDCKD1_BYPASS);
1894
1895         spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
1896         spll_func_cntl_2 |= SCLK_MUX_SEL(4);
1897
1898         table->ACPIState.levels[0].mclk.vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
1899         table->ACPIState.levels[0].mclk.vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
1900         table->ACPIState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
1901         table->ACPIState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
1902         table->ACPIState.levels[0].mclk.vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
1903         table->ACPIState.levels[0].mclk.vDLL_CNTL = cpu_to_be32(dll_cntl);
1904
1905         table->ACPIState.levels[0].mclk.mclk_value = 0;
1906
1907         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL = cpu_to_be32(spll_func_cntl);
1908         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(spll_func_cntl_2);
1909         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(spll_func_cntl_3);
1910         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_4 = cpu_to_be32(spll_func_cntl_4);
1911
1912         table->ACPIState.levels[0].sclk.sclk_value = 0;
1913
1914         ni_populate_mvdd_value(rdev, 0, &table->ACPIState.levels[0].mvdd);
1915
1916         if (eg_pi->dynamic_ac_timing)
1917                 table->ACPIState.levels[0].ACIndex = 1;
1918
1919         table->ACPIState.levels[0].dpm2.MaxPS = 0;
1920         table->ACPIState.levels[0].dpm2.NearTDPDec = 0;
1921         table->ACPIState.levels[0].dpm2.AboveSafeInc = 0;
1922         table->ACPIState.levels[0].dpm2.BelowSafeInc = 0;
1923
1924         reg = MIN_POWER_MASK | MAX_POWER_MASK;
1925         table->ACPIState.levels[0].SQPowerThrottle = cpu_to_be32(reg);
1926
1927         reg = MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
1928         table->ACPIState.levels[0].SQPowerThrottle_2 = cpu_to_be32(reg);
1929
1930         return 0;
1931 }
1932
1933 static int ni_init_smc_table(struct radeon_device *rdev)
1934 {
1935         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1936         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1937         int ret;
1938         struct radeon_ps *radeon_boot_state = rdev->pm.dpm.boot_ps;
1939         NISLANDS_SMC_STATETABLE *table = &ni_pi->smc_statetable;
1940
1941         memset(table, 0, sizeof(NISLANDS_SMC_STATETABLE));
1942
1943         ni_populate_smc_voltage_tables(rdev, table);
1944
1945         switch (rdev->pm.int_thermal_type) {
1946         case THERMAL_TYPE_NI:
1947         case THERMAL_TYPE_EMC2103_WITH_INTERNAL:
1948                 table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_INTERNAL;
1949                 break;
1950         case THERMAL_TYPE_NONE:
1951                 table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_NONE;
1952                 break;
1953         default:
1954                 table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_EXTERNAL;
1955                 break;
1956         }
1957
1958         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC)
1959                 table->systemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC;
1960
1961         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_REGULATOR_HOT)
1962                 table->systemFlags |= PPSMC_SYSTEMFLAG_REGULATOR_HOT;
1963
1964         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC)
1965                 table->systemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC;
1966
1967         if (pi->mem_gddr5)
1968                 table->systemFlags |= PPSMC_SYSTEMFLAG_GDDR5;
1969
1970         ret = ni_populate_smc_initial_state(rdev, radeon_boot_state, table);
1971         if (ret)
1972                 return ret;
1973
1974         ret = ni_populate_smc_acpi_state(rdev, table);
1975         if (ret)
1976                 return ret;
1977
1978         table->driverState = table->initialState;
1979
1980         table->ULVState = table->initialState;
1981
1982         ret = ni_do_program_memory_timing_parameters(rdev, radeon_boot_state,
1983                                                      NISLANDS_INITIAL_STATE_ARB_INDEX);
1984         if (ret)
1985                 return ret;
1986
1987         return rv770_copy_bytes_to_smc(rdev, pi->state_table_start, (u8 *)table,
1988                                        sizeof(NISLANDS_SMC_STATETABLE), pi->sram_end);
1989 }
1990
1991 static int ni_calculate_sclk_params(struct radeon_device *rdev,
1992                                     u32 engine_clock,
1993                                     NISLANDS_SMC_SCLK_VALUE *sclk)
1994 {
1995         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1996         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1997         struct atom_clock_dividers dividers;
1998         u32 spll_func_cntl = ni_pi->clock_registers.cg_spll_func_cntl;
1999         u32 spll_func_cntl_2 = ni_pi->clock_registers.cg_spll_func_cntl_2;
2000         u32 spll_func_cntl_3 = ni_pi->clock_registers.cg_spll_func_cntl_3;
2001         u32 spll_func_cntl_4 = ni_pi->clock_registers.cg_spll_func_cntl_4;
2002         u32 cg_spll_spread_spectrum = ni_pi->clock_registers.cg_spll_spread_spectrum;
2003         u32 cg_spll_spread_spectrum_2 = ni_pi->clock_registers.cg_spll_spread_spectrum_2;
2004         u64 tmp;
2005         u32 reference_clock = rdev->clock.spll.reference_freq;
2006         u32 reference_divider;
2007         u32 fbdiv;
2008         int ret;
2009
2010         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
2011                                              engine_clock, false, &dividers);
2012         if (ret)
2013                 return ret;
2014
2015         reference_divider = 1 + dividers.ref_div;
2016
2017
2018         tmp = (u64) engine_clock * reference_divider * dividers.post_div * 16834;
2019         do_div(tmp, reference_clock);
2020         fbdiv = (u32) tmp;
2021
2022         spll_func_cntl &= ~(SPLL_PDIV_A_MASK | SPLL_REF_DIV_MASK);
2023         spll_func_cntl |= SPLL_REF_DIV(dividers.ref_div);
2024         spll_func_cntl |= SPLL_PDIV_A(dividers.post_div);
2025
2026         spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
2027         spll_func_cntl_2 |= SCLK_MUX_SEL(2);
2028
2029         spll_func_cntl_3 &= ~SPLL_FB_DIV_MASK;
2030         spll_func_cntl_3 |= SPLL_FB_DIV(fbdiv);
2031         spll_func_cntl_3 |= SPLL_DITHEN;
2032
2033         if (pi->sclk_ss) {
2034                 struct radeon_atom_ss ss;
2035                 u32 vco_freq = engine_clock * dividers.post_div;
2036
2037                 if (radeon_atombios_get_asic_ss_info(rdev, &ss,
2038                                                      ASIC_INTERNAL_ENGINE_SS, vco_freq)) {
2039                         u32 clk_s = reference_clock * 5 / (reference_divider * ss.rate);
2040                         u32 clk_v = 4 * ss.percentage * fbdiv / (clk_s * 10000);
2041
2042                         cg_spll_spread_spectrum &= ~CLK_S_MASK;
2043                         cg_spll_spread_spectrum |= CLK_S(clk_s);
2044                         cg_spll_spread_spectrum |= SSEN;
2045
2046                         cg_spll_spread_spectrum_2 &= ~CLK_V_MASK;
2047                         cg_spll_spread_spectrum_2 |= CLK_V(clk_v);
2048                 }
2049         }
2050
2051         sclk->sclk_value = engine_clock;
2052         sclk->vCG_SPLL_FUNC_CNTL = spll_func_cntl;
2053         sclk->vCG_SPLL_FUNC_CNTL_2 = spll_func_cntl_2;
2054         sclk->vCG_SPLL_FUNC_CNTL_3 = spll_func_cntl_3;
2055         sclk->vCG_SPLL_FUNC_CNTL_4 = spll_func_cntl_4;
2056         sclk->vCG_SPLL_SPREAD_SPECTRUM = cg_spll_spread_spectrum;
2057         sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cg_spll_spread_spectrum_2;
2058
2059         return 0;
2060 }
2061
2062 static int ni_populate_sclk_value(struct radeon_device *rdev,
2063                                   u32 engine_clock,
2064                                   NISLANDS_SMC_SCLK_VALUE *sclk)
2065 {
2066         NISLANDS_SMC_SCLK_VALUE sclk_tmp;
2067         int ret;
2068
2069         ret = ni_calculate_sclk_params(rdev, engine_clock, &sclk_tmp);
2070         if (!ret) {
2071                 sclk->sclk_value = cpu_to_be32(sclk_tmp.sclk_value);
2072                 sclk->vCG_SPLL_FUNC_CNTL = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL);
2073                 sclk->vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_2);
2074                 sclk->vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_3);
2075                 sclk->vCG_SPLL_FUNC_CNTL_4 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_4);
2076                 sclk->vCG_SPLL_SPREAD_SPECTRUM = cpu_to_be32(sclk_tmp.vCG_SPLL_SPREAD_SPECTRUM);
2077                 sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cpu_to_be32(sclk_tmp.vCG_SPLL_SPREAD_SPECTRUM_2);
2078         }
2079
2080         return ret;
2081 }
2082
2083 static int ni_init_smc_spll_table(struct radeon_device *rdev)
2084 {
2085         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2086         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2087         SMC_NISLANDS_SPLL_DIV_TABLE *spll_table;
2088         NISLANDS_SMC_SCLK_VALUE sclk_params;
2089         u32 fb_div;
2090         u32 p_div;
2091         u32 clk_s;
2092         u32 clk_v;
2093         u32 sclk = 0;
2094         int i, ret;
2095         u32 tmp;
2096
2097         if (ni_pi->spll_table_start == 0)
2098                 return -EINVAL;
2099
2100         spll_table = kzalloc(sizeof(SMC_NISLANDS_SPLL_DIV_TABLE), GFP_KERNEL);
2101         if (spll_table == NULL)
2102                 return -ENOMEM;
2103
2104         for (i = 0; i < 256; i++) {
2105                 ret = ni_calculate_sclk_params(rdev, sclk, &sclk_params);
2106                 if (ret)
2107                         break;
2108
2109                 p_div = (sclk_params.vCG_SPLL_FUNC_CNTL & SPLL_PDIV_A_MASK) >> SPLL_PDIV_A_SHIFT;
2110                 fb_div = (sclk_params.vCG_SPLL_FUNC_CNTL_3 & SPLL_FB_DIV_MASK) >> SPLL_FB_DIV_SHIFT;
2111                 clk_s = (sclk_params.vCG_SPLL_SPREAD_SPECTRUM & CLK_S_MASK) >> CLK_S_SHIFT;
2112                 clk_v = (sclk_params.vCG_SPLL_SPREAD_SPECTRUM_2 & CLK_V_MASK) >> CLK_V_SHIFT;
2113
2114                 fb_div &= ~0x00001FFF;
2115                 fb_div >>= 1;
2116                 clk_v >>= 6;
2117
2118                 if (p_div & ~(SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT))
2119                         ret = -EINVAL;
2120
2121                 if (clk_s & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT))
2122                         ret = -EINVAL;
2123
2124                 if (clk_s & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT))
2125                         ret = -EINVAL;
2126
2127                 if (clk_v & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT))
2128                         ret = -EINVAL;
2129
2130                 if (ret)
2131                         break;
2132
2133                 tmp = ((fb_div << SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_MASK) |
2134                         ((p_div << SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_MASK);
2135                 spll_table->freq[i] = cpu_to_be32(tmp);
2136
2137                 tmp = ((clk_v << SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_MASK) |
2138                         ((clk_s << SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK);
2139                 spll_table->ss[i] = cpu_to_be32(tmp);
2140
2141                 sclk += 512;
2142         }
2143
2144         if (!ret)
2145                 ret = rv770_copy_bytes_to_smc(rdev, ni_pi->spll_table_start, (u8 *)spll_table,
2146                                               sizeof(SMC_NISLANDS_SPLL_DIV_TABLE), pi->sram_end);
2147
2148         kfree(spll_table);
2149
2150         return ret;
2151 }
2152
2153 static int ni_populate_mclk_value(struct radeon_device *rdev,
2154                                   u32 engine_clock,
2155                                   u32 memory_clock,
2156                                   NISLANDS_SMC_MCLK_VALUE *mclk,
2157                                   bool strobe_mode,
2158                                   bool dll_state_on)
2159 {
2160         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2161         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2162         u32 mpll_ad_func_cntl = ni_pi->clock_registers.mpll_ad_func_cntl;
2163         u32 mpll_ad_func_cntl_2 = ni_pi->clock_registers.mpll_ad_func_cntl_2;
2164         u32 mpll_dq_func_cntl = ni_pi->clock_registers.mpll_dq_func_cntl;
2165         u32 mpll_dq_func_cntl_2 = ni_pi->clock_registers.mpll_dq_func_cntl_2;
2166         u32 mclk_pwrmgt_cntl = ni_pi->clock_registers.mclk_pwrmgt_cntl;
2167         u32 dll_cntl = ni_pi->clock_registers.dll_cntl;
2168         u32 mpll_ss1 = ni_pi->clock_registers.mpll_ss1;
2169         u32 mpll_ss2 = ni_pi->clock_registers.mpll_ss2;
2170         struct atom_clock_dividers dividers;
2171         u32 ibias;
2172         u32 dll_speed;
2173         int ret;
2174         u32 mc_seq_misc7;
2175
2176         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_MEMORY_PLL_PARAM,
2177                                              memory_clock, strobe_mode, &dividers);
2178         if (ret)
2179                 return ret;
2180
2181         if (!strobe_mode) {
2182                 mc_seq_misc7 = RREG32(MC_SEQ_MISC7);
2183
2184                 if (mc_seq_misc7 & 0x8000000)
2185                         dividers.post_div = 1;
2186         }
2187
2188         ibias = cypress_map_clkf_to_ibias(rdev, dividers.whole_fb_div);
2189
2190         mpll_ad_func_cntl &= ~(CLKR_MASK |
2191                                YCLK_POST_DIV_MASK |
2192                                CLKF_MASK |
2193                                CLKFRAC_MASK |
2194                                IBIAS_MASK);
2195         mpll_ad_func_cntl |= CLKR(dividers.ref_div);
2196         mpll_ad_func_cntl |= YCLK_POST_DIV(dividers.post_div);
2197         mpll_ad_func_cntl |= CLKF(dividers.whole_fb_div);
2198         mpll_ad_func_cntl |= CLKFRAC(dividers.frac_fb_div);
2199         mpll_ad_func_cntl |= IBIAS(ibias);
2200
2201         if (dividers.vco_mode)
2202                 mpll_ad_func_cntl_2 |= VCO_MODE;
2203         else
2204                 mpll_ad_func_cntl_2 &= ~VCO_MODE;
2205
2206         if (pi->mem_gddr5) {
2207                 mpll_dq_func_cntl &= ~(CLKR_MASK |
2208                                        YCLK_POST_DIV_MASK |
2209                                        CLKF_MASK |
2210                                        CLKFRAC_MASK |
2211                                        IBIAS_MASK);
2212                 mpll_dq_func_cntl |= CLKR(dividers.ref_div);
2213                 mpll_dq_func_cntl |= YCLK_POST_DIV(dividers.post_div);
2214                 mpll_dq_func_cntl |= CLKF(dividers.whole_fb_div);
2215                 mpll_dq_func_cntl |= CLKFRAC(dividers.frac_fb_div);
2216                 mpll_dq_func_cntl |= IBIAS(ibias);
2217
2218                 if (strobe_mode)
2219                         mpll_dq_func_cntl &= ~PDNB;
2220                 else
2221                         mpll_dq_func_cntl |= PDNB;
2222
2223                 if (dividers.vco_mode)
2224                         mpll_dq_func_cntl_2 |= VCO_MODE;
2225                 else
2226                         mpll_dq_func_cntl_2 &= ~VCO_MODE;
2227         }
2228
2229         if (pi->mclk_ss) {
2230                 struct radeon_atom_ss ss;
2231                 u32 vco_freq = memory_clock * dividers.post_div;
2232
2233                 if (radeon_atombios_get_asic_ss_info(rdev, &ss,
2234                                                      ASIC_INTERNAL_MEMORY_SS, vco_freq)) {
2235                         u32 reference_clock = rdev->clock.mpll.reference_freq;
2236                         u32 decoded_ref = rv740_get_decoded_reference_divider(dividers.ref_div);
2237                         u32 clk_s = reference_clock * 5 / (decoded_ref * ss.rate);
2238                         u32 clk_v = ss.percentage *
2239                                 (0x4000 * dividers.whole_fb_div + 0x800 * dividers.frac_fb_div) / (clk_s * 625);
2240
2241                         mpll_ss1 &= ~CLKV_MASK;
2242                         mpll_ss1 |= CLKV(clk_v);
2243
2244                         mpll_ss2 &= ~CLKS_MASK;
2245                         mpll_ss2 |= CLKS(clk_s);
2246                 }
2247         }
2248
2249         dll_speed = rv740_get_dll_speed(pi->mem_gddr5,
2250                                         memory_clock);
2251
2252         mclk_pwrmgt_cntl &= ~DLL_SPEED_MASK;
2253         mclk_pwrmgt_cntl |= DLL_SPEED(dll_speed);
2254         if (dll_state_on)
2255                 mclk_pwrmgt_cntl |= (MRDCKA0_PDNB |
2256                                      MRDCKA1_PDNB |
2257                                      MRDCKB0_PDNB |
2258                                      MRDCKB1_PDNB |
2259                                      MRDCKC0_PDNB |
2260                                      MRDCKC1_PDNB |
2261                                      MRDCKD0_PDNB |
2262                                      MRDCKD1_PDNB);
2263         else
2264                 mclk_pwrmgt_cntl &= ~(MRDCKA0_PDNB |
2265                                       MRDCKA1_PDNB |
2266                                       MRDCKB0_PDNB |
2267                                       MRDCKB1_PDNB |
2268                                       MRDCKC0_PDNB |
2269                                       MRDCKC1_PDNB |
2270                                       MRDCKD0_PDNB |
2271                                       MRDCKD1_PDNB);
2272
2273
2274         mclk->mclk_value = cpu_to_be32(memory_clock);
2275         mclk->vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
2276         mclk->vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
2277         mclk->vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
2278         mclk->vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
2279         mclk->vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
2280         mclk->vDLL_CNTL = cpu_to_be32(dll_cntl);
2281         mclk->vMPLL_SS = cpu_to_be32(mpll_ss1);
2282         mclk->vMPLL_SS2 = cpu_to_be32(mpll_ss2);
2283
2284         return 0;
2285 }
2286
2287 static void ni_populate_smc_sp(struct radeon_device *rdev,
2288                                struct radeon_ps *radeon_state,
2289                                NISLANDS_SMC_SWSTATE *smc_state)
2290 {
2291         struct ni_ps *ps = ni_get_ps(radeon_state);
2292         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2293         int i;
2294
2295         for (i = 0; i < ps->performance_level_count - 1; i++)
2296                 smc_state->levels[i].bSP = cpu_to_be32(pi->dsp);
2297
2298         smc_state->levels[ps->performance_level_count - 1].bSP =
2299                 cpu_to_be32(pi->psp);
2300 }
2301
2302 static int ni_convert_power_level_to_smc(struct radeon_device *rdev,
2303                                          struct rv7xx_pl *pl,
2304                                          NISLANDS_SMC_HW_PERFORMANCE_LEVEL *level)
2305 {
2306         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2307         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2308         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2309         int ret;
2310         bool dll_state_on;
2311         u16 std_vddc;
2312         u32 tmp = RREG32(DC_STUTTER_CNTL);
2313
2314         level->gen2PCIE = pi->pcie_gen2 ?
2315                 ((pl->flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? 1 : 0) : 0;
2316
2317         ret = ni_populate_sclk_value(rdev, pl->sclk, &level->sclk);
2318         if (ret)
2319                 return ret;
2320
2321         level->mcFlags =  0;
2322         if (pi->mclk_stutter_mode_threshold &&
2323             (pl->mclk <= pi->mclk_stutter_mode_threshold) &&
2324             !eg_pi->uvd_enabled &&
2325             (tmp & DC_STUTTER_ENABLE_A) &&
2326             (tmp & DC_STUTTER_ENABLE_B))
2327                 level->mcFlags |= NISLANDS_SMC_MC_STUTTER_EN;
2328
2329         if (pi->mem_gddr5) {
2330                 if (pl->mclk > pi->mclk_edc_enable_threshold)
2331                         level->mcFlags |= NISLANDS_SMC_MC_EDC_RD_FLAG;
2332                 if (pl->mclk > eg_pi->mclk_edc_wr_enable_threshold)
2333                         level->mcFlags |= NISLANDS_SMC_MC_EDC_WR_FLAG;
2334
2335                 level->strobeMode = cypress_get_strobe_mode_settings(rdev, pl->mclk);
2336
2337                 if (level->strobeMode & NISLANDS_SMC_STROBE_ENABLE) {
2338                         if (cypress_get_mclk_frequency_ratio(rdev, pl->mclk, true) >=
2339                             ((RREG32(MC_SEQ_MISC7) >> 16) & 0xf))
2340                                 dll_state_on = ((RREG32(MC_SEQ_MISC5) >> 1) & 0x1) ? true : false;
2341                         else
2342                                 dll_state_on = ((RREG32(MC_SEQ_MISC6) >> 1) & 0x1) ? true : false;
2343                 } else {
2344                         dll_state_on = false;
2345                         if (pl->mclk > ni_pi->mclk_rtt_mode_threshold)
2346                                 level->mcFlags |= NISLANDS_SMC_MC_RTT_ENABLE;
2347                 }
2348
2349                 ret = ni_populate_mclk_value(rdev, pl->sclk, pl->mclk,
2350                                              &level->mclk,
2351                                              (level->strobeMode & NISLANDS_SMC_STROBE_ENABLE) != 0,
2352                                              dll_state_on);
2353         } else
2354                 ret = ni_populate_mclk_value(rdev, pl->sclk, pl->mclk, &level->mclk, 1, 1);
2355
2356         if (ret)
2357                 return ret;
2358
2359         ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
2360                                         pl->vddc, &level->vddc);
2361         if (ret)
2362                 return ret;
2363
2364         ret = ni_get_std_voltage_value(rdev, &level->vddc, &std_vddc);
2365         if (ret)
2366                 return ret;
2367
2368         ni_populate_std_voltage_value(rdev, std_vddc,
2369                                       level->vddc.index, &level->std_vddc);
2370
2371         if (eg_pi->vddci_control) {
2372                 ret = ni_populate_voltage_value(rdev, &eg_pi->vddci_voltage_table,
2373                                                 pl->vddci, &level->vddci);
2374                 if (ret)
2375                         return ret;
2376         }
2377
2378         ni_populate_mvdd_value(rdev, pl->mclk, &level->mvdd);
2379
2380         return ret;
2381 }
2382
2383 static int ni_populate_smc_t(struct radeon_device *rdev,
2384                              struct radeon_ps *radeon_state,
2385                              NISLANDS_SMC_SWSTATE *smc_state)
2386 {
2387         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2388         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2389         struct ni_ps *state = ni_get_ps(radeon_state);
2390         u32 a_t;
2391         u32 t_l, t_h;
2392         u32 high_bsp;
2393         int i, ret;
2394
2395         if (state->performance_level_count >= 9)
2396                 return -EINVAL;
2397
2398         if (state->performance_level_count < 2) {
2399                 a_t = CG_R(0xffff) | CG_L(0);
2400                 smc_state->levels[0].aT = cpu_to_be32(a_t);
2401                 return 0;
2402         }
2403
2404         smc_state->levels[0].aT = cpu_to_be32(0);
2405
2406         for (i = 0; i <= state->performance_level_count - 2; i++) {
2407                 if (eg_pi->uvd_enabled)
2408                         ret = r600_calculate_at(
2409                                 1000 * (i * (eg_pi->smu_uvd_hs ? 2 : 8) + 2),
2410                                 100 * R600_AH_DFLT,
2411                                 state->performance_levels[i + 1].sclk,
2412                                 state->performance_levels[i].sclk,
2413                                 &t_l,
2414                                 &t_h);
2415                 else
2416                         ret = r600_calculate_at(
2417                                 1000 * (i + 1),
2418                                 100 * R600_AH_DFLT,
2419                                 state->performance_levels[i + 1].sclk,
2420                                 state->performance_levels[i].sclk,
2421                                 &t_l,
2422                                 &t_h);
2423
2424                 if (ret) {
2425                         t_h = (i + 1) * 1000 - 50 * R600_AH_DFLT;
2426                         t_l = (i + 1) * 1000 + 50 * R600_AH_DFLT;
2427                 }
2428
2429                 a_t = be32_to_cpu(smc_state->levels[i].aT) & ~CG_R_MASK;
2430                 a_t |= CG_R(t_l * pi->bsp / 20000);
2431                 smc_state->levels[i].aT = cpu_to_be32(a_t);
2432
2433                 high_bsp = (i == state->performance_level_count - 2) ?
2434                         pi->pbsp : pi->bsp;
2435
2436                 a_t = CG_R(0xffff) | CG_L(t_h * high_bsp / 20000);
2437                 smc_state->levels[i + 1].aT = cpu_to_be32(a_t);
2438         }
2439
2440         return 0;
2441 }
2442
2443 static int ni_populate_power_containment_values(struct radeon_device *rdev,
2444                                                 struct radeon_ps *radeon_state,
2445                                                 NISLANDS_SMC_SWSTATE *smc_state)
2446 {
2447         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2448         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2449         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2450         struct ni_ps *state = ni_get_ps(radeon_state);
2451         u32 prev_sclk;
2452         u32 max_sclk;
2453         u32 min_sclk;
2454         int i, ret;
2455         u32 tdp_limit;
2456         u32 near_tdp_limit;
2457         u32 power_boost_limit;
2458         u8 max_ps_percent;
2459
2460         if (ni_pi->enable_power_containment == false)
2461                 return 0;
2462
2463         if (state->performance_level_count == 0)
2464                 return -EINVAL;
2465
2466         if (smc_state->levelCount != state->performance_level_count)
2467                 return -EINVAL;
2468
2469         ret = ni_calculate_adjusted_tdp_limits(rdev,
2470                                                false, /* ??? */
2471                                                rdev->pm.dpm.tdp_adjustment,
2472                                                &tdp_limit,
2473                                                &near_tdp_limit);
2474         if (ret)
2475                 return ret;
2476
2477         power_boost_limit = ni_calculate_power_boost_limit(rdev, radeon_state, near_tdp_limit);
2478
2479         ret = rv770_write_smc_sram_dword(rdev,
2480                                          pi->state_table_start +
2481                                          offsetof(NISLANDS_SMC_STATETABLE, dpm2Params) +
2482                                          offsetof(PP_NIslands_DPM2Parameters, PowerBoostLimit),
2483                                          ni_scale_power_for_smc(power_boost_limit, ni_get_smc_power_scaling_factor(rdev)),
2484                                          pi->sram_end);
2485         if (ret)
2486                 power_boost_limit = 0;
2487
2488         smc_state->levels[0].dpm2.MaxPS = 0;
2489         smc_state->levels[0].dpm2.NearTDPDec = 0;
2490         smc_state->levels[0].dpm2.AboveSafeInc = 0;
2491         smc_state->levels[0].dpm2.BelowSafeInc = 0;
2492         smc_state->levels[0].stateFlags |= power_boost_limit ? PPSMC_STATEFLAG_POWERBOOST : 0;
2493
2494         for (i = 1; i < state->performance_level_count; i++) {
2495                 prev_sclk = state->performance_levels[i-1].sclk;
2496                 max_sclk  = state->performance_levels[i].sclk;
2497                 max_ps_percent = (i != (state->performance_level_count - 1)) ?
2498                         NISLANDS_DPM2_MAXPS_PERCENT_M : NISLANDS_DPM2_MAXPS_PERCENT_H;
2499
2500                 if (max_sclk < prev_sclk)
2501                         return -EINVAL;
2502
2503                 if ((max_ps_percent == 0) || (prev_sclk == max_sclk) || eg_pi->uvd_enabled)
2504                         min_sclk = max_sclk;
2505                 else if (1 == i)
2506                         min_sclk = prev_sclk;
2507                 else
2508                         min_sclk = (prev_sclk * (u32)max_ps_percent) / 100;
2509
2510                 if (min_sclk < state->performance_levels[0].sclk)
2511                         min_sclk = state->performance_levels[0].sclk;
2512
2513                 if (min_sclk == 0)
2514                         return -EINVAL;
2515
2516                 smc_state->levels[i].dpm2.MaxPS =
2517                         (u8)((NISLANDS_DPM2_MAX_PULSE_SKIP * (max_sclk - min_sclk)) / max_sclk);
2518                 smc_state->levels[i].dpm2.NearTDPDec = NISLANDS_DPM2_NEAR_TDP_DEC;
2519                 smc_state->levels[i].dpm2.AboveSafeInc = NISLANDS_DPM2_ABOVE_SAFE_INC;
2520                 smc_state->levels[i].dpm2.BelowSafeInc = NISLANDS_DPM2_BELOW_SAFE_INC;
2521                 smc_state->levels[i].stateFlags |=
2522                         ((i != (state->performance_level_count - 1)) && power_boost_limit) ?
2523                         PPSMC_STATEFLAG_POWERBOOST : 0;
2524         }
2525
2526         return 0;
2527 }
2528
2529 static int ni_populate_sq_ramping_values(struct radeon_device *rdev,
2530                                          struct radeon_ps *radeon_state,
2531                                          NISLANDS_SMC_SWSTATE *smc_state)
2532 {
2533         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2534         struct ni_ps *state = ni_get_ps(radeon_state);
2535         u32 sq_power_throttle;
2536         u32 sq_power_throttle2;
2537         bool enable_sq_ramping = ni_pi->enable_sq_ramping;
2538         int i;
2539
2540         if (state->performance_level_count == 0)
2541                 return -EINVAL;
2542
2543         if (smc_state->levelCount != state->performance_level_count)
2544                 return -EINVAL;
2545
2546         if (rdev->pm.dpm.sq_ramping_threshold == 0)
2547                 return -EINVAL;
2548
2549         if (NISLANDS_DPM2_SQ_RAMP_MAX_POWER > (MAX_POWER_MASK >> MAX_POWER_SHIFT))
2550                 enable_sq_ramping = false;
2551
2552         if (NISLANDS_DPM2_SQ_RAMP_MIN_POWER > (MIN_POWER_MASK >> MIN_POWER_SHIFT))
2553                 enable_sq_ramping = false;
2554
2555         if (NISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA > (MAX_POWER_DELTA_MASK >> MAX_POWER_DELTA_SHIFT))
2556                 enable_sq_ramping = false;
2557
2558         if (NISLANDS_DPM2_SQ_RAMP_STI_SIZE > (STI_SIZE_MASK >> STI_SIZE_SHIFT))
2559                 enable_sq_ramping = false;
2560
2561         if (NISLANDS_DPM2_SQ_RAMP_LTI_RATIO <= (LTI_RATIO_MASK >> LTI_RATIO_SHIFT))
2562                 enable_sq_ramping = false;
2563
2564         for (i = 0; i < state->performance_level_count; i++) {
2565                 sq_power_throttle  = 0;
2566                 sq_power_throttle2 = 0;
2567
2568                 if ((state->performance_levels[i].sclk >= rdev->pm.dpm.sq_ramping_threshold) &&
2569                     enable_sq_ramping) {
2570                         sq_power_throttle |= MAX_POWER(NISLANDS_DPM2_SQ_RAMP_MAX_POWER);
2571                         sq_power_throttle |= MIN_POWER(NISLANDS_DPM2_SQ_RAMP_MIN_POWER);
2572                         sq_power_throttle2 |= MAX_POWER_DELTA(NISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA);
2573                         sq_power_throttle2 |= STI_SIZE(NISLANDS_DPM2_SQ_RAMP_STI_SIZE);
2574                         sq_power_throttle2 |= LTI_RATIO(NISLANDS_DPM2_SQ_RAMP_LTI_RATIO);
2575                 } else {
2576                         sq_power_throttle |= MAX_POWER_MASK | MIN_POWER_MASK;
2577                         sq_power_throttle2 |= MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
2578                 }
2579
2580                 smc_state->levels[i].SQPowerThrottle   = cpu_to_be32(sq_power_throttle);
2581                 smc_state->levels[i].SQPowerThrottle_2 = cpu_to_be32(sq_power_throttle2);
2582         }
2583
2584         return 0;
2585 }
2586
2587 static int ni_enable_power_containment(struct radeon_device *rdev,
2588                                        struct radeon_ps *radeon_new_state,
2589                                        bool enable)
2590 {
2591         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2592         PPSMC_Result smc_result;
2593         int ret = 0;
2594
2595         if (ni_pi->enable_power_containment) {
2596                 if (enable) {
2597                         if (!r600_is_uvd_state(radeon_new_state->class, radeon_new_state->class2)) {
2598                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_TDPClampingActive);
2599                                 if (smc_result != PPSMC_Result_OK) {
2600                                         ret = -EINVAL;
2601                                         ni_pi->pc_enabled = false;
2602                                 } else {
2603                                         ni_pi->pc_enabled = true;
2604                                 }
2605                         }
2606                 } else {
2607                         smc_result = rv770_send_msg_to_smc(rdev, PPSMC_TDPClampingInactive);
2608                         if (smc_result != PPSMC_Result_OK)
2609                                 ret = -EINVAL;
2610                         ni_pi->pc_enabled = false;
2611                 }
2612         }
2613
2614         return ret;
2615 }
2616
2617 static int ni_convert_power_state_to_smc(struct radeon_device *rdev,
2618                                          struct radeon_ps *radeon_state,
2619                                          NISLANDS_SMC_SWSTATE *smc_state)
2620 {
2621         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2622         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2623         struct ni_ps *state = ni_get_ps(radeon_state);
2624         int i, ret;
2625         u32 threshold = state->performance_levels[state->performance_level_count - 1].sclk * 100 / 100;
2626
2627         if (!(radeon_state->caps & ATOM_PPLIB_DISALLOW_ON_DC))
2628                 smc_state->flags |= PPSMC_SWSTATE_FLAG_DC;
2629
2630         smc_state->levelCount = 0;
2631
2632         if (state->performance_level_count > NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE)
2633                 return -EINVAL;
2634
2635         for (i = 0; i < state->performance_level_count; i++) {
2636                 ret = ni_convert_power_level_to_smc(rdev, &state->performance_levels[i],
2637                                                     &smc_state->levels[i]);
2638                 smc_state->levels[i].arbRefreshState =
2639                         (u8)(NISLANDS_DRIVER_STATE_ARB_INDEX + i);
2640
2641                 if (ret)
2642                         return ret;
2643
2644                 if (ni_pi->enable_power_containment)
2645                         smc_state->levels[i].displayWatermark =
2646                                 (state->performance_levels[i].sclk < threshold) ?
2647                                 PPSMC_DISPLAY_WATERMARK_LOW : PPSMC_DISPLAY_WATERMARK_HIGH;
2648                 else
2649                         smc_state->levels[i].displayWatermark = (i < 2) ?
2650                                 PPSMC_DISPLAY_WATERMARK_LOW : PPSMC_DISPLAY_WATERMARK_HIGH;
2651
2652                 if (eg_pi->dynamic_ac_timing)
2653                         smc_state->levels[i].ACIndex = NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT + i;
2654                 else
2655                         smc_state->levels[i].ACIndex = 0;
2656
2657                 smc_state->levelCount++;
2658         }
2659
2660         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_watermark_threshold,
2661                                       cpu_to_be32(threshold / 512));
2662
2663         ni_populate_smc_sp(rdev, radeon_state, smc_state);
2664
2665         ret = ni_populate_power_containment_values(rdev, radeon_state, smc_state);
2666         if (ret)
2667                 ni_pi->enable_power_containment = false;
2668
2669         ret = ni_populate_sq_ramping_values(rdev, radeon_state, smc_state);
2670         if (ret)
2671                 ni_pi->enable_sq_ramping = false;
2672
2673         return ni_populate_smc_t(rdev, radeon_state, smc_state);
2674 }
2675
2676 static int ni_upload_sw_state(struct radeon_device *rdev,
2677                               struct radeon_ps *radeon_new_state)
2678 {
2679         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2680         u16 address = pi->state_table_start +
2681                 offsetof(NISLANDS_SMC_STATETABLE, driverState);
2682         u16 state_size = sizeof(NISLANDS_SMC_SWSTATE) +
2683                 ((NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1) * sizeof(NISLANDS_SMC_HW_PERFORMANCE_LEVEL));
2684         int ret;
2685         NISLANDS_SMC_SWSTATE *smc_state = kzalloc(state_size, GFP_KERNEL);
2686
2687         if (smc_state == NULL)
2688                 return -ENOMEM;
2689
2690         ret = ni_convert_power_state_to_smc(rdev, radeon_new_state, smc_state);
2691         if (ret)
2692                 goto done;
2693
2694         ret = rv770_copy_bytes_to_smc(rdev, address, (u8 *)smc_state, state_size, pi->sram_end);
2695
2696 done:
2697         kfree(smc_state);
2698
2699         return ret;
2700 }
2701
2702 static int ni_set_mc_special_registers(struct radeon_device *rdev,
2703                                        struct ni_mc_reg_table *table)
2704 {
2705         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2706         u8 i, j, k;
2707         u32 temp_reg;
2708
2709         for (i = 0, j = table->last; i < table->last; i++) {
2710                 switch (table->mc_reg_address[i].s1) {
2711                 case MC_SEQ_MISC1 >> 2:
2712                         if (j >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2713                                 return -EINVAL;
2714                         temp_reg = RREG32(MC_PMG_CMD_EMRS);
2715                         table->mc_reg_address[j].s1 = MC_PMG_CMD_EMRS >> 2;
2716                         table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_EMRS_LP >> 2;
2717                         for (k = 0; k < table->num_entries; k++)
2718                                 table->mc_reg_table_entry[k].mc_data[j] =
2719                                         ((temp_reg & 0xffff0000)) |
2720                                         ((table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16);
2721                         j++;
2722                         if (j >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2723                                 return -EINVAL;
2724
2725                         temp_reg = RREG32(MC_PMG_CMD_MRS);
2726                         table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS >> 2;
2727                         table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS_LP >> 2;
2728                         for(k = 0; k < table->num_entries; k++) {
2729                                 table->mc_reg_table_entry[k].mc_data[j] =
2730                                         (temp_reg & 0xffff0000) |
2731                                         (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
2732                                 if (!pi->mem_gddr5)
2733                                         table->mc_reg_table_entry[k].mc_data[j] |= 0x100;
2734                         }
2735                         j++;
2736                         if (j > SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2737                                 return -EINVAL;
2738                         break;
2739                 case MC_SEQ_RESERVE_M >> 2:
2740                         temp_reg = RREG32(MC_PMG_CMD_MRS1);
2741                         table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS1 >> 2;
2742                         table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS1_LP >> 2;
2743                         for (k = 0; k < table->num_entries; k++)
2744                                 table->mc_reg_table_entry[k].mc_data[j] =
2745                                         (temp_reg & 0xffff0000) |
2746                                         (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
2747                         j++;
2748                         if (j > SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2749                                 return -EINVAL;
2750                         break;
2751                 default:
2752                         break;
2753                 }
2754         }
2755
2756         table->last = j;
2757
2758         return 0;
2759 }
2760
2761 static bool ni_check_s0_mc_reg_index(u16 in_reg, u16 *out_reg)
2762 {
2763         bool result = true;
2764
2765         switch (in_reg) {
2766         case  MC_SEQ_RAS_TIMING >> 2:
2767                 *out_reg = MC_SEQ_RAS_TIMING_LP >> 2;
2768                 break;
2769         case MC_SEQ_CAS_TIMING >> 2:
2770                 *out_reg = MC_SEQ_CAS_TIMING_LP >> 2;
2771                 break;
2772         case MC_SEQ_MISC_TIMING >> 2:
2773                 *out_reg = MC_SEQ_MISC_TIMING_LP >> 2;
2774                 break;
2775         case MC_SEQ_MISC_TIMING2 >> 2:
2776                 *out_reg = MC_SEQ_MISC_TIMING2_LP >> 2;
2777                 break;
2778         case MC_SEQ_RD_CTL_D0 >> 2:
2779                 *out_reg = MC_SEQ_RD_CTL_D0_LP >> 2;
2780                 break;
2781         case MC_SEQ_RD_CTL_D1 >> 2:
2782                 *out_reg = MC_SEQ_RD_CTL_D1_LP >> 2;
2783                 break;
2784         case MC_SEQ_WR_CTL_D0 >> 2:
2785                 *out_reg = MC_SEQ_WR_CTL_D0_LP >> 2;
2786                 break;
2787         case MC_SEQ_WR_CTL_D1 >> 2:
2788                 *out_reg = MC_SEQ_WR_CTL_D1_LP >> 2;
2789                 break;
2790         case MC_PMG_CMD_EMRS >> 2:
2791                 *out_reg = MC_SEQ_PMG_CMD_EMRS_LP >> 2;
2792                 break;
2793         case MC_PMG_CMD_MRS >> 2:
2794                 *out_reg = MC_SEQ_PMG_CMD_MRS_LP >> 2;
2795                 break;
2796         case MC_PMG_CMD_MRS1 >> 2:
2797                 *out_reg = MC_SEQ_PMG_CMD_MRS1_LP >> 2;
2798                 break;
2799         case MC_SEQ_PMG_TIMING >> 2:
2800                 *out_reg = MC_SEQ_PMG_TIMING_LP >> 2;
2801                 break;
2802         case MC_PMG_CMD_MRS2 >> 2:
2803                 *out_reg = MC_SEQ_PMG_CMD_MRS2_LP >> 2;
2804                 break;
2805         default:
2806                 result = false;
2807                 break;
2808         }
2809
2810         return result;
2811 }
2812
2813 static void ni_set_valid_flag(struct ni_mc_reg_table *table)
2814 {
2815         u8 i, j;
2816
2817         for (i = 0; i < table->last; i++) {
2818                 for (j = 1; j < table->num_entries; j++) {
2819                         if (table->mc_reg_table_entry[j-1].mc_data[i] != table->mc_reg_table_entry[j].mc_data[i]) {
2820                                 table->valid_flag |= 1 << i;
2821                                 break;
2822                         }
2823                 }
2824         }
2825 }
2826
2827 static void ni_set_s0_mc_reg_index(struct ni_mc_reg_table *table)
2828 {
2829         u32 i;
2830         u16 address;
2831
2832         for (i = 0; i < table->last; i++)
2833                 table->mc_reg_address[i].s0 =
2834                         ni_check_s0_mc_reg_index(table->mc_reg_address[i].s1, &address) ?
2835                         address : table->mc_reg_address[i].s1;
2836 }
2837
2838 static int ni_copy_vbios_mc_reg_table(struct atom_mc_reg_table *table,
2839                                       struct ni_mc_reg_table *ni_table)
2840 {
2841         u8 i, j;
2842
2843         if (table->last > SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2844                 return -EINVAL;
2845         if (table->num_entries > MAX_AC_TIMING_ENTRIES)
2846                 return -EINVAL;
2847
2848         for (i = 0; i < table->last; i++)
2849                 ni_table->mc_reg_address[i].s1 = table->mc_reg_address[i].s1;
2850         ni_table->last = table->last;
2851
2852         for (i = 0; i < table->num_entries; i++) {
2853                 ni_table->mc_reg_table_entry[i].mclk_max =
2854                         table->mc_reg_table_entry[i].mclk_max;
2855                 for (j = 0; j < table->last; j++)
2856                         ni_table->mc_reg_table_entry[i].mc_data[j] =
2857                                 table->mc_reg_table_entry[i].mc_data[j];
2858         }
2859         ni_table->num_entries = table->num_entries;
2860
2861         return 0;
2862 }
2863
2864 static int ni_initialize_mc_reg_table(struct radeon_device *rdev)
2865 {
2866         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2867         int ret;
2868         struct atom_mc_reg_table *table;
2869         struct ni_mc_reg_table *ni_table = &ni_pi->mc_reg_table;
2870         u8 module_index = rv770_get_memory_module_index(rdev);
2871
2872         table = kzalloc(sizeof(struct atom_mc_reg_table), GFP_KERNEL);
2873         if (!table)
2874                 return -ENOMEM;
2875
2876         WREG32(MC_SEQ_RAS_TIMING_LP, RREG32(MC_SEQ_RAS_TIMING));
2877         WREG32(MC_SEQ_CAS_TIMING_LP, RREG32(MC_SEQ_CAS_TIMING));
2878         WREG32(MC_SEQ_MISC_TIMING_LP, RREG32(MC_SEQ_MISC_TIMING));
2879         WREG32(MC_SEQ_MISC_TIMING2_LP, RREG32(MC_SEQ_MISC_TIMING2));
2880         WREG32(MC_SEQ_PMG_CMD_EMRS_LP, RREG32(MC_PMG_CMD_EMRS));
2881         WREG32(MC_SEQ_PMG_CMD_MRS_LP, RREG32(MC_PMG_CMD_MRS));
2882         WREG32(MC_SEQ_PMG_CMD_MRS1_LP, RREG32(MC_PMG_CMD_MRS1));
2883         WREG32(MC_SEQ_WR_CTL_D0_LP, RREG32(MC_SEQ_WR_CTL_D0));
2884         WREG32(MC_SEQ_WR_CTL_D1_LP, RREG32(MC_SEQ_WR_CTL_D1));
2885         WREG32(MC_SEQ_RD_CTL_D0_LP, RREG32(MC_SEQ_RD_CTL_D0));
2886         WREG32(MC_SEQ_RD_CTL_D1_LP, RREG32(MC_SEQ_RD_CTL_D1));
2887         WREG32(MC_SEQ_PMG_TIMING_LP, RREG32(MC_SEQ_PMG_TIMING));
2888         WREG32(MC_SEQ_PMG_CMD_MRS2_LP, RREG32(MC_PMG_CMD_MRS2));
2889
2890         ret = radeon_atom_init_mc_reg_table(rdev, module_index, table);
2891
2892         if (ret)
2893                 goto init_mc_done;
2894
2895         ret = ni_copy_vbios_mc_reg_table(table, ni_table);
2896
2897         if (ret)
2898                 goto init_mc_done;
2899
2900         ni_set_s0_mc_reg_index(ni_table);
2901
2902         ret = ni_set_mc_special_registers(rdev, ni_table);
2903
2904         if (ret)
2905                 goto init_mc_done;
2906
2907         ni_set_valid_flag(ni_table);
2908
2909 init_mc_done:
2910         kfree(table);
2911
2912         return ret;
2913 }
2914
2915 static void ni_populate_mc_reg_addresses(struct radeon_device *rdev,
2916                                          SMC_NIslands_MCRegisters *mc_reg_table)
2917 {
2918         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2919         u32 i, j;
2920
2921         for (i = 0, j = 0; j < ni_pi->mc_reg_table.last; j++) {
2922                 if (ni_pi->mc_reg_table.valid_flag & (1 << j)) {
2923                         if (i >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2924                                 break;
2925                         mc_reg_table->address[i].s0 =
2926                                 cpu_to_be16(ni_pi->mc_reg_table.mc_reg_address[j].s0);
2927                         mc_reg_table->address[i].s1 =
2928                                 cpu_to_be16(ni_pi->mc_reg_table.mc_reg_address[j].s1);
2929                         i++;
2930                 }
2931         }
2932         mc_reg_table->last = (u8)i;
2933 }
2934
2935
2936 static void ni_convert_mc_registers(struct ni_mc_reg_entry *entry,
2937                                     SMC_NIslands_MCRegisterSet *data,
2938                                     u32 num_entries, u32 valid_flag)
2939 {
2940         u32 i, j;
2941
2942         for (i = 0, j = 0; j < num_entries; j++) {
2943                 if (valid_flag & (1 << j)) {
2944                         data->value[i] = cpu_to_be32(entry->mc_data[j]);
2945                         i++;
2946                 }
2947         }
2948 }
2949
2950 static void ni_convert_mc_reg_table_entry_to_smc(struct radeon_device *rdev,
2951                                                  struct rv7xx_pl *pl,
2952                                                  SMC_NIslands_MCRegisterSet *mc_reg_table_data)
2953 {
2954         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2955         u32 i = 0;
2956
2957         for (i = 0; i < ni_pi->mc_reg_table.num_entries; i++) {
2958                 if (pl->mclk <= ni_pi->mc_reg_table.mc_reg_table_entry[i].mclk_max)
2959                         break;
2960         }
2961
2962         if ((i == ni_pi->mc_reg_table.num_entries) && (i > 0))
2963                 --i;
2964
2965         ni_convert_mc_registers(&ni_pi->mc_reg_table.mc_reg_table_entry[i],
2966                                 mc_reg_table_data,
2967                                 ni_pi->mc_reg_table.last,
2968                                 ni_pi->mc_reg_table.valid_flag);
2969 }
2970
2971 static void ni_convert_mc_reg_table_to_smc(struct radeon_device *rdev,
2972                                            struct radeon_ps *radeon_state,
2973                                            SMC_NIslands_MCRegisters *mc_reg_table)
2974 {
2975         struct ni_ps *state = ni_get_ps(radeon_state);
2976         int i;
2977
2978         for (i = 0; i < state->performance_level_count; i++) {
2979                 ni_convert_mc_reg_table_entry_to_smc(rdev,
2980                                                      &state->performance_levels[i],
2981                                                      &mc_reg_table->data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT + i]);
2982         }
2983 }
2984
2985 static int ni_populate_mc_reg_table(struct radeon_device *rdev,
2986                                     struct radeon_ps *radeon_boot_state)
2987 {
2988         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2989         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2990         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2991         struct ni_ps *boot_state = ni_get_ps(radeon_boot_state);
2992         SMC_NIslands_MCRegisters *mc_reg_table = &ni_pi->smc_mc_reg_table;
2993
2994         memset(mc_reg_table, 0, sizeof(SMC_NIslands_MCRegisters));
2995
2996         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_seq_index, 1);
2997
2998         ni_populate_mc_reg_addresses(rdev, mc_reg_table);
2999
3000         ni_convert_mc_reg_table_entry_to_smc(rdev, &boot_state->performance_levels[0],
3001                                              &mc_reg_table->data[0]);
3002
3003         ni_convert_mc_registers(&ni_pi->mc_reg_table.mc_reg_table_entry[0],
3004                                 &mc_reg_table->data[1],
3005                                 ni_pi->mc_reg_table.last,
3006                                 ni_pi->mc_reg_table.valid_flag);
3007
3008         ni_convert_mc_reg_table_to_smc(rdev, radeon_boot_state, mc_reg_table);
3009
3010         return rv770_copy_bytes_to_smc(rdev, eg_pi->mc_reg_table_start,
3011                                        (u8 *)mc_reg_table,
3012                                        sizeof(SMC_NIslands_MCRegisters),
3013                                        pi->sram_end);
3014 }
3015
3016 static int ni_upload_mc_reg_table(struct radeon_device *rdev,
3017                                   struct radeon_ps *radeon_new_state)
3018 {
3019         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3020         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3021         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3022         struct ni_ps *ni_new_state = ni_get_ps(radeon_new_state);
3023         SMC_NIslands_MCRegisters *mc_reg_table = &ni_pi->smc_mc_reg_table;
3024         u16 address;
3025
3026         memset(mc_reg_table, 0, sizeof(SMC_NIslands_MCRegisters));
3027
3028         ni_convert_mc_reg_table_to_smc(rdev, radeon_new_state, mc_reg_table);
3029
3030         address = eg_pi->mc_reg_table_start +
3031                 (u16)offsetof(SMC_NIslands_MCRegisters, data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT]);
3032
3033         return rv770_copy_bytes_to_smc(rdev, address,
3034                                        (u8 *)&mc_reg_table->data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT],
3035                                        sizeof(SMC_NIslands_MCRegisterSet) * ni_new_state->performance_level_count,
3036                                        pi->sram_end);
3037 }
3038
3039 static int ni_init_driver_calculated_leakage_table(struct radeon_device *rdev,
3040                                                    PP_NIslands_CACTABLES *cac_tables)
3041 {
3042         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3043         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3044         u32 leakage = 0;
3045         unsigned int i, j, table_size;
3046         s32 t;
3047         u32 smc_leakage, max_leakage = 0;
3048         u32 scaling_factor;
3049
3050         table_size = eg_pi->vddc_voltage_table.count;
3051
3052         if (SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES < table_size)
3053                 table_size = SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES;
3054
3055         scaling_factor = ni_get_smc_power_scaling_factor(rdev);
3056
3057         for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++) {
3058                 for (j = 0; j < table_size; j++) {
3059                         t = (1000 * ((i + 1) * 8));
3060
3061                         if (t < ni_pi->cac_data.leakage_minimum_temperature)
3062                                 t = ni_pi->cac_data.leakage_minimum_temperature;
3063
3064                         ni_calculate_leakage_for_v_and_t(rdev,
3065                                                          &ni_pi->cac_data.leakage_coefficients,
3066                                                          eg_pi->vddc_voltage_table.entries[j].value,
3067                                                          t,
3068                                                          ni_pi->cac_data.i_leakage,
3069                                                          &leakage);
3070
3071                         smc_leakage = ni_scale_power_for_smc(leakage, scaling_factor) / 1000;
3072                         if (smc_leakage > max_leakage)
3073                                 max_leakage = smc_leakage;
3074
3075                         cac_tables->cac_lkge_lut[i][j] = cpu_to_be32(smc_leakage);
3076                 }
3077         }
3078
3079         for (j = table_size; j < SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; j++) {
3080                 for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
3081                         cac_tables->cac_lkge_lut[i][j] = cpu_to_be32(max_leakage);
3082         }
3083         return 0;
3084 }
3085
3086 static int ni_init_simplified_leakage_table(struct radeon_device *rdev,
3087                                             PP_NIslands_CACTABLES *cac_tables)
3088 {
3089         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3090         struct radeon_cac_leakage_table *leakage_table =
3091                 &rdev->pm.dpm.dyn_state.cac_leakage_table;
3092         u32 i, j, table_size;
3093         u32 smc_leakage, max_leakage = 0;
3094         u32 scaling_factor;
3095
3096         if (!leakage_table)
3097                 return -EINVAL;
3098
3099         table_size = leakage_table->count;
3100
3101         if (eg_pi->vddc_voltage_table.count != table_size)
3102                 table_size = (eg_pi->vddc_voltage_table.count < leakage_table->count) ?
3103                         eg_pi->vddc_voltage_table.count : leakage_table->count;
3104
3105         if (SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES < table_size)
3106                 table_size = SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES;
3107
3108         if (table_size == 0)
3109                 return -EINVAL;
3110
3111         scaling_factor = ni_get_smc_power_scaling_factor(rdev);
3112
3113         for (j = 0; j < table_size; j++) {
3114                 smc_leakage = leakage_table->entries[j].leakage;
3115
3116                 if (smc_leakage > max_leakage)
3117                         max_leakage = smc_leakage;
3118
3119                 for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
3120                         cac_tables->cac_lkge_lut[i][j] =
3121                                 cpu_to_be32(ni_scale_power_for_smc(smc_leakage, scaling_factor));
3122         }
3123
3124         for (j = table_size; j < SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; j++) {
3125                 for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
3126                         cac_tables->cac_lkge_lut[i][j] =
3127                                 cpu_to_be32(ni_scale_power_for_smc(max_leakage, scaling_factor));
3128         }
3129         return 0;
3130 }
3131
3132 static int ni_initialize_smc_cac_tables(struct radeon_device *rdev)
3133 {
3134         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3135         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3136         PP_NIslands_CACTABLES *cac_tables = NULL;
3137         int i, ret;
3138         u32 reg;
3139
3140         if (ni_pi->enable_cac == false)
3141                 return 0;
3142
3143         cac_tables = kzalloc(sizeof(PP_NIslands_CACTABLES), GFP_KERNEL);
3144         if (!cac_tables)
3145                 return -ENOMEM;
3146
3147         reg = RREG32(CG_CAC_CTRL) & ~(TID_CNT_MASK | TID_UNIT_MASK);
3148         reg |= (TID_CNT(ni_pi->cac_weights->tid_cnt) |
3149                 TID_UNIT(ni_pi->cac_weights->tid_unit));
3150         WREG32(CG_CAC_CTRL, reg);
3151
3152         for (i = 0; i < NISLANDS_DCCAC_MAX_LEVELS; i++)
3153                 ni_pi->dc_cac_table[i] = ni_pi->cac_weights->dc_cac[i];
3154
3155         for (i = 0; i < SMC_NISLANDS_BIF_LUT_NUM_OF_ENTRIES; i++)
3156                 cac_tables->cac_bif_lut[i] = ni_pi->cac_weights->pcie_cac[i];
3157
3158         ni_pi->cac_data.i_leakage = rdev->pm.dpm.cac_leakage;
3159         ni_pi->cac_data.pwr_const = 0;
3160         ni_pi->cac_data.dc_cac_value = ni_pi->dc_cac_table[NISLANDS_DCCAC_LEVEL_0];
3161         ni_pi->cac_data.bif_cac_value = 0;
3162         ni_pi->cac_data.mc_wr_weight = ni_pi->cac_weights->mc_write_weight;
3163         ni_pi->cac_data.mc_rd_weight = ni_pi->cac_weights->mc_read_weight;
3164         ni_pi->cac_data.allow_ovrflw = 0;
3165         ni_pi->cac_data.l2num_win_tdp = ni_pi->lta_window_size;
3166         ni_pi->cac_data.num_win_tdp = 0;
3167         ni_pi->cac_data.lts_truncate_n = ni_pi->lts_truncate;
3168
3169         if (ni_pi->driver_calculate_cac_leakage)
3170                 ret = ni_init_driver_calculated_leakage_table(rdev, cac_tables);
3171         else
3172                 ret = ni_init_simplified_leakage_table(rdev, cac_tables);
3173
3174         if (ret)
3175                 goto done_free;
3176
3177         cac_tables->pwr_const      = cpu_to_be32(ni_pi->cac_data.pwr_const);
3178         cac_tables->dc_cacValue    = cpu_to_be32(ni_pi->cac_data.dc_cac_value);
3179         cac_tables->bif_cacValue   = cpu_to_be32(ni_pi->cac_data.bif_cac_value);
3180         cac_tables->AllowOvrflw    = ni_pi->cac_data.allow_ovrflw;
3181         cac_tables->MCWrWeight     = ni_pi->cac_data.mc_wr_weight;
3182         cac_tables->MCRdWeight     = ni_pi->cac_data.mc_rd_weight;
3183         cac_tables->numWin_TDP     = ni_pi->cac_data.num_win_tdp;
3184         cac_tables->l2numWin_TDP   = ni_pi->cac_data.l2num_win_tdp;
3185         cac_tables->lts_truncate_n = ni_pi->cac_data.lts_truncate_n;
3186
3187         ret = rv770_copy_bytes_to_smc(rdev, ni_pi->cac_table_start, (u8 *)cac_tables,
3188                                       sizeof(PP_NIslands_CACTABLES), pi->sram_end);
3189
3190 done_free:
3191         if (ret) {
3192                 ni_pi->enable_cac = false;
3193                 ni_pi->enable_power_containment = false;
3194         }
3195
3196         kfree(cac_tables);
3197
3198         return 0;
3199 }
3200
3201 static int ni_initialize_hardware_cac_manager(struct radeon_device *rdev)
3202 {
3203         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3204         u32 reg;
3205
3206         if (!ni_pi->enable_cac ||
3207             !ni_pi->cac_configuration_required)
3208                 return 0;
3209
3210         if (ni_pi->cac_weights == NULL)
3211                 return -EINVAL;
3212
3213         reg = RREG32_CG(CG_CAC_REGION_1_WEIGHT_0) & ~(WEIGHT_TCP_SIG0_MASK |
3214                                                       WEIGHT_TCP_SIG1_MASK |
3215                                                       WEIGHT_TA_SIG_MASK);
3216         reg |= (WEIGHT_TCP_SIG0(ni_pi->cac_weights->weight_tcp_sig0) |
3217                 WEIGHT_TCP_SIG1(ni_pi->cac_weights->weight_tcp_sig1) |
3218                 WEIGHT_TA_SIG(ni_pi->cac_weights->weight_ta_sig));
3219         WREG32_CG(CG_CAC_REGION_1_WEIGHT_0, reg);
3220
3221         reg = RREG32_CG(CG_CAC_REGION_1_WEIGHT_1) & ~(WEIGHT_TCC_EN0_MASK |
3222                                                       WEIGHT_TCC_EN1_MASK |
3223                                                       WEIGHT_TCC_EN2_MASK);
3224         reg |= (WEIGHT_TCC_EN0(ni_pi->cac_weights->weight_tcc_en0) |
3225                 WEIGHT_TCC_EN1(ni_pi->cac_weights->weight_tcc_en1) |
3226                 WEIGHT_TCC_EN2(ni_pi->cac_weights->weight_tcc_en2));
3227         WREG32_CG(CG_CAC_REGION_1_WEIGHT_1, reg);
3228
3229         reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_0) & ~(WEIGHT_CB_EN0_MASK |
3230                                                       WEIGHT_CB_EN1_MASK |
3231                                                       WEIGHT_CB_EN2_MASK |
3232                                                       WEIGHT_CB_EN3_MASK);
3233         reg |= (WEIGHT_CB_EN0(ni_pi->cac_weights->weight_cb_en0) |
3234                 WEIGHT_CB_EN1(ni_pi->cac_weights->weight_cb_en1) |
3235                 WEIGHT_CB_EN2(ni_pi->cac_weights->weight_cb_en2) |
3236                 WEIGHT_CB_EN3(ni_pi->cac_weights->weight_cb_en3));
3237         WREG32_CG(CG_CAC_REGION_2_WEIGHT_0, reg);
3238
3239         reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_1) & ~(WEIGHT_DB_SIG0_MASK |
3240                                                       WEIGHT_DB_SIG1_MASK |
3241                                                       WEIGHT_DB_SIG2_MASK |
3242                                                       WEIGHT_DB_SIG3_MASK);
3243         reg |= (WEIGHT_DB_SIG0(ni_pi->cac_weights->weight_db_sig0) |
3244                 WEIGHT_DB_SIG1(ni_pi->cac_weights->weight_db_sig1) |
3245                 WEIGHT_DB_SIG2(ni_pi->cac_weights->weight_db_sig2) |
3246                 WEIGHT_DB_SIG3(ni_pi->cac_weights->weight_db_sig3));
3247         WREG32_CG(CG_CAC_REGION_2_WEIGHT_1, reg);
3248
3249         reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_2) & ~(WEIGHT_SXM_SIG0_MASK |
3250                                                       WEIGHT_SXM_SIG1_MASK |
3251                                                       WEIGHT_SXM_SIG2_MASK |
3252                                                       WEIGHT_SXS_SIG0_MASK |
3253                                                       WEIGHT_SXS_SIG1_MASK);
3254         reg |= (WEIGHT_SXM_SIG0(ni_pi->cac_weights->weight_sxm_sig0) |
3255                 WEIGHT_SXM_SIG1(ni_pi->cac_weights->weight_sxm_sig1) |
3256                 WEIGHT_SXM_SIG2(ni_pi->cac_weights->weight_sxm_sig2) |
3257                 WEIGHT_SXS_SIG0(ni_pi->cac_weights->weight_sxs_sig0) |
3258                 WEIGHT_SXS_SIG1(ni_pi->cac_weights->weight_sxs_sig1));
3259         WREG32_CG(CG_CAC_REGION_2_WEIGHT_2, reg);
3260
3261         reg = RREG32_CG(CG_CAC_REGION_3_WEIGHT_0) & ~(WEIGHT_XBR_0_MASK |
3262                                                       WEIGHT_XBR_1_MASK |
3263                                                       WEIGHT_XBR_2_MASK |
3264                                                       WEIGHT_SPI_SIG0_MASK);
3265         reg |= (WEIGHT_XBR_0(ni_pi->cac_weights->weight_xbr_0) |
3266                 WEIGHT_XBR_1(ni_pi->cac_weights->weight_xbr_1) |
3267                 WEIGHT_XBR_2(ni_pi->cac_weights->weight_xbr_2) |
3268                 WEIGHT_SPI_SIG0(ni_pi->cac_weights->weight_spi_sig0));
3269         WREG32_CG(CG_CAC_REGION_3_WEIGHT_0, reg);
3270
3271         reg = RREG32_CG(CG_CAC_REGION_3_WEIGHT_1) & ~(WEIGHT_SPI_SIG1_MASK |
3272                                                       WEIGHT_SPI_SIG2_MASK |
3273                                                       WEIGHT_SPI_SIG3_MASK |
3274                                                       WEIGHT_SPI_SIG4_MASK |
3275                                                       WEIGHT_SPI_SIG5_MASK);
3276         reg |= (WEIGHT_SPI_SIG1(ni_pi->cac_weights->weight_spi_sig1) |
3277                 WEIGHT_SPI_SIG2(ni_pi->cac_weights->weight_spi_sig2) |
3278                 WEIGHT_SPI_SIG3(ni_pi->cac_weights->weight_spi_sig3) |
3279                 WEIGHT_SPI_SIG4(ni_pi->cac_weights->weight_spi_sig4) |
3280                 WEIGHT_SPI_SIG5(ni_pi->cac_weights->weight_spi_sig5));
3281         WREG32_CG(CG_CAC_REGION_3_WEIGHT_1, reg);
3282
3283         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_0) & ~(WEIGHT_LDS_SIG0_MASK |
3284                                                       WEIGHT_LDS_SIG1_MASK |
3285                                                       WEIGHT_SC_MASK);
3286         reg |= (WEIGHT_LDS_SIG0(ni_pi->cac_weights->weight_lds_sig0) |
3287                 WEIGHT_LDS_SIG1(ni_pi->cac_weights->weight_lds_sig1) |
3288                 WEIGHT_SC(ni_pi->cac_weights->weight_sc));
3289         WREG32_CG(CG_CAC_REGION_4_WEIGHT_0, reg);
3290
3291         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_1) & ~(WEIGHT_BIF_MASK |
3292                                                       WEIGHT_CP_MASK |
3293                                                       WEIGHT_PA_SIG0_MASK |
3294                                                       WEIGHT_PA_SIG1_MASK |
3295                                                       WEIGHT_VGT_SIG0_MASK);
3296         reg |= (WEIGHT_BIF(ni_pi->cac_weights->weight_bif) |
3297                 WEIGHT_CP(ni_pi->cac_weights->weight_cp) |
3298                 WEIGHT_PA_SIG0(ni_pi->cac_weights->weight_pa_sig0) |
3299                 WEIGHT_PA_SIG1(ni_pi->cac_weights->weight_pa_sig1) |
3300                 WEIGHT_VGT_SIG0(ni_pi->cac_weights->weight_vgt_sig0));
3301         WREG32_CG(CG_CAC_REGION_4_WEIGHT_1, reg);
3302
3303         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_2) & ~(WEIGHT_VGT_SIG1_MASK |
3304                                                       WEIGHT_VGT_SIG2_MASK |
3305                                                       WEIGHT_DC_SIG0_MASK |
3306                                                       WEIGHT_DC_SIG1_MASK |
3307                                                       WEIGHT_DC_SIG2_MASK);
3308         reg |= (WEIGHT_VGT_SIG1(ni_pi->cac_weights->weight_vgt_sig1) |
3309                 WEIGHT_VGT_SIG2(ni_pi->cac_weights->weight_vgt_sig2) |
3310                 WEIGHT_DC_SIG0(ni_pi->cac_weights->weight_dc_sig0) |
3311                 WEIGHT_DC_SIG1(ni_pi->cac_weights->weight_dc_sig1) |
3312                 WEIGHT_DC_SIG2(ni_pi->cac_weights->weight_dc_sig2));
3313         WREG32_CG(CG_CAC_REGION_4_WEIGHT_2, reg);
3314
3315         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_3) & ~(WEIGHT_DC_SIG3_MASK |
3316                                                       WEIGHT_UVD_SIG0_MASK |
3317                                                       WEIGHT_UVD_SIG1_MASK |
3318                                                       WEIGHT_SPARE0_MASK |
3319                                                       WEIGHT_SPARE1_MASK);
3320         reg |= (WEIGHT_DC_SIG3(ni_pi->cac_weights->weight_dc_sig3) |
3321                 WEIGHT_UVD_SIG0(ni_pi->cac_weights->weight_uvd_sig0) |
3322                 WEIGHT_UVD_SIG1(ni_pi->cac_weights->weight_uvd_sig1) |
3323                 WEIGHT_SPARE0(ni_pi->cac_weights->weight_spare0) |
3324                 WEIGHT_SPARE1(ni_pi->cac_weights->weight_spare1));
3325         WREG32_CG(CG_CAC_REGION_4_WEIGHT_3, reg);
3326
3327         reg = RREG32_CG(CG_CAC_REGION_5_WEIGHT_0) & ~(WEIGHT_SQ_VSP_MASK |
3328                                                       WEIGHT_SQ_VSP0_MASK);
3329         reg |= (WEIGHT_SQ_VSP(ni_pi->cac_weights->weight_sq_vsp) |
3330                 WEIGHT_SQ_VSP0(ni_pi->cac_weights->weight_sq_vsp0));
3331         WREG32_CG(CG_CAC_REGION_5_WEIGHT_0, reg);
3332
3333         reg = RREG32_CG(CG_CAC_REGION_5_WEIGHT_1) & ~(WEIGHT_SQ_GPR_MASK);
3334         reg |= WEIGHT_SQ_GPR(ni_pi->cac_weights->weight_sq_gpr);
3335         WREG32_CG(CG_CAC_REGION_5_WEIGHT_1, reg);
3336
3337         reg = RREG32_CG(CG_CAC_REGION_4_OVERRIDE_4) & ~(OVR_MODE_SPARE_0_MASK |
3338                                                         OVR_VAL_SPARE_0_MASK |
3339                                                         OVR_MODE_SPARE_1_MASK |
3340                                                         OVR_VAL_SPARE_1_MASK);
3341         reg |= (OVR_MODE_SPARE_0(ni_pi->cac_weights->ovr_mode_spare_0) |
3342                 OVR_VAL_SPARE_0(ni_pi->cac_weights->ovr_val_spare_0) |
3343                 OVR_MODE_SPARE_1(ni_pi->cac_weights->ovr_mode_spare_1) |
3344                 OVR_VAL_SPARE_1(ni_pi->cac_weights->ovr_val_spare_1));
3345         WREG32_CG(CG_CAC_REGION_4_OVERRIDE_4, reg);
3346
3347         reg = RREG32(SQ_CAC_THRESHOLD) & ~(VSP_MASK |
3348                                            VSP0_MASK |
3349                                            GPR_MASK);
3350         reg |= (VSP(ni_pi->cac_weights->vsp) |
3351                 VSP0(ni_pi->cac_weights->vsp0) |
3352                 GPR(ni_pi->cac_weights->gpr));
3353         WREG32(SQ_CAC_THRESHOLD, reg);
3354
3355         reg = (MCDW_WR_ENABLE |
3356                MCDX_WR_ENABLE |
3357                MCDY_WR_ENABLE |
3358                MCDZ_WR_ENABLE |
3359                INDEX(0x09D4));
3360         WREG32(MC_CG_CONFIG, reg);
3361
3362         reg = (READ_WEIGHT(ni_pi->cac_weights->mc_read_weight) |
3363                WRITE_WEIGHT(ni_pi->cac_weights->mc_write_weight) |
3364                ALLOW_OVERFLOW);
3365         WREG32(MC_CG_DATAPORT, reg);
3366
3367         return 0;
3368 }
3369
3370 static int ni_enable_smc_cac(struct radeon_device *rdev,
3371                              struct radeon_ps *radeon_new_state,
3372                              bool enable)
3373 {
3374         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3375         int ret = 0;
3376         PPSMC_Result smc_result;
3377
3378         if (ni_pi->enable_cac) {
3379                 if (enable) {
3380                         if (!r600_is_uvd_state(radeon_new_state->class, radeon_new_state->class2)) {
3381                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_CollectCAC_PowerCorreln);
3382
3383                                 if (ni_pi->support_cac_long_term_average) {
3384                                         smc_result = rv770_send_msg_to_smc(rdev, PPSMC_CACLongTermAvgEnable);
3385                                         if (PPSMC_Result_OK != smc_result)
3386                                                 ni_pi->support_cac_long_term_average = false;
3387                                 }
3388
3389                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_EnableCac);
3390                                 if (PPSMC_Result_OK != smc_result)
3391                                         ret = -EINVAL;
3392
3393                                 ni_pi->cac_enabled = (PPSMC_Result_OK == smc_result) ? true : false;
3394                         }
3395                 } else if (ni_pi->cac_enabled) {
3396                         smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_DisableCac);
3397
3398                         ni_pi->cac_enabled = false;
3399
3400                         if (ni_pi->support_cac_long_term_average) {
3401                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_CACLongTermAvgDisable);
3402                                 if (PPSMC_Result_OK != smc_result)
3403                                         ni_pi->support_cac_long_term_average = false;
3404                         }
3405                 }
3406         }
3407
3408         return ret;
3409 }
3410
3411 static int ni_pcie_performance_request(struct radeon_device *rdev,
3412                                        u8 perf_req, bool advertise)
3413 {
3414         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3415
3416 #if defined(CONFIG_ACPI)
3417         if ((perf_req == PCIE_PERF_REQ_PECI_GEN1) ||
3418             (perf_req == PCIE_PERF_REQ_PECI_GEN2)) {
3419                 if (eg_pi->pcie_performance_request_registered == false)
3420                         radeon_acpi_pcie_notify_device_ready(rdev);
3421                 eg_pi->pcie_performance_request_registered = true;
3422                 return radeon_acpi_pcie_performance_request(rdev, perf_req, advertise);
3423         } else if ((perf_req == PCIE_PERF_REQ_REMOVE_REGISTRY) &&
3424                    eg_pi->pcie_performance_request_registered) {
3425                 eg_pi->pcie_performance_request_registered = false;
3426                 return radeon_acpi_pcie_performance_request(rdev, perf_req, advertise);
3427         }
3428 #endif
3429         return 0;
3430 }
3431
3432 static int ni_advertise_gen2_capability(struct radeon_device *rdev)
3433 {
3434         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3435         u32 tmp;
3436
3437         tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3438
3439         if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
3440             (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2))
3441                 pi->pcie_gen2 = true;
3442         else
3443                 pi->pcie_gen2 = false;
3444
3445         if (!pi->pcie_gen2)
3446                 ni_pcie_performance_request(rdev, PCIE_PERF_REQ_PECI_GEN2, true);
3447
3448         return 0;
3449 }
3450
3451 static void ni_enable_bif_dynamic_pcie_gen2(struct radeon_device *rdev,
3452                                             bool enable)
3453 {
3454         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3455         u32 tmp, bif;
3456
3457         tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3458
3459         if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
3460             (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
3461                 if (enable) {
3462                         if (!pi->boot_in_gen2) {
3463                                 bif = RREG32(CG_BIF_REQ_AND_RSP) & ~CG_CLIENT_REQ_MASK;
3464                                 bif |= CG_CLIENT_REQ(0xd);
3465                                 WREG32(CG_BIF_REQ_AND_RSP, bif);
3466                         }
3467                         tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
3468                         tmp |= LC_HW_VOLTAGE_IF_CONTROL(1);
3469                         tmp |= LC_GEN2_EN_STRAP;
3470
3471                         tmp |= LC_CLR_FAILED_SPD_CHANGE_CNT;
3472                         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
3473                         udelay(10);
3474                         tmp &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
3475                         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
3476                 } else {
3477                         if (!pi->boot_in_gen2) {
3478                                 bif = RREG32(CG_BIF_REQ_AND_RSP) & ~CG_CLIENT_REQ_MASK;
3479                                 bif |= CG_CLIENT_REQ(0xd);
3480                                 WREG32(CG_BIF_REQ_AND_RSP, bif);
3481
3482                                 tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
3483                                 tmp &= ~LC_GEN2_EN_STRAP;
3484                         }
3485                         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
3486                 }
3487         }
3488 }
3489
3490 static void ni_enable_dynamic_pcie_gen2(struct radeon_device *rdev,
3491                                         bool enable)
3492 {
3493         ni_enable_bif_dynamic_pcie_gen2(rdev, enable);
3494
3495         if (enable)
3496                 WREG32_P(GENERAL_PWRMGT, ENABLE_GEN2PCIE, ~ENABLE_GEN2PCIE);
3497         else
3498                 WREG32_P(GENERAL_PWRMGT, 0, ~ENABLE_GEN2PCIE);
3499 }
3500
3501 void ni_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev,
3502                                            struct radeon_ps *new_ps,
3503                                            struct radeon_ps *old_ps)
3504 {
3505         struct ni_ps *new_state = ni_get_ps(new_ps);
3506         struct ni_ps *current_state = ni_get_ps(old_ps);
3507
3508         if ((new_ps->vclk == old_ps->vclk) &&
3509             (new_ps->dclk == old_ps->dclk))
3510                 return;
3511
3512         if (new_state->performance_levels[new_state->performance_level_count - 1].sclk >=
3513             current_state->performance_levels[current_state->performance_level_count - 1].sclk)
3514                 return;
3515
3516         radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
3517 }
3518
3519 void ni_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev,
3520                                           struct radeon_ps *new_ps,
3521                                           struct radeon_ps *old_ps)
3522 {
3523         struct ni_ps *new_state = ni_get_ps(new_ps);
3524         struct ni_ps *current_state = ni_get_ps(old_ps);
3525
3526         if ((new_ps->vclk == old_ps->vclk) &&
3527             (new_ps->dclk == old_ps->dclk))
3528                 return;
3529
3530         if (new_state->performance_levels[new_state->performance_level_count - 1].sclk <
3531             current_state->performance_levels[current_state->performance_level_count - 1].sclk)
3532                 return;
3533
3534         radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
3535 }
3536
3537 void ni_dpm_setup_asic(struct radeon_device *rdev)
3538 {
3539         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3540
3541         ni_read_clock_registers(rdev);
3542         btc_read_arb_registers(rdev);
3543         rv770_get_memory_type(rdev);
3544         if (eg_pi->pcie_performance_request)
3545                 ni_advertise_gen2_capability(rdev);
3546         rv770_get_pcie_gen2_status(rdev);
3547         rv770_enable_acpi_pm(rdev);
3548 }
3549
3550 void ni_update_current_ps(struct radeon_device *rdev,
3551                           struct radeon_ps *rps)
3552 {
3553         struct ni_ps *new_ps = ni_get_ps(rps);
3554         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3555         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3556
3557         eg_pi->current_rps = *rps;
3558         ni_pi->current_ps = *new_ps;
3559         eg_pi->current_rps.ps_priv = &ni_pi->current_ps;
3560 }
3561
3562 void ni_update_requested_ps(struct radeon_device *rdev,
3563                             struct radeon_ps *rps)
3564 {
3565         struct ni_ps *new_ps = ni_get_ps(rps);
3566         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3567         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3568
3569         eg_pi->requested_rps = *rps;
3570         ni_pi->requested_ps = *new_ps;
3571         eg_pi->requested_rps.ps_priv = &ni_pi->requested_ps;
3572 }
3573
3574 int ni_dpm_enable(struct radeon_device *rdev)
3575 {
3576         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3577         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3578         struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
3579         int ret;
3580
3581         if (pi->gfx_clock_gating)
3582                 ni_cg_clockgating_default(rdev);
3583         if (btc_dpm_enabled(rdev))
3584                 return -EINVAL;
3585         if (pi->mg_clock_gating)
3586                 ni_mg_clockgating_default(rdev);
3587         if (eg_pi->ls_clock_gating)
3588                 ni_ls_clockgating_default(rdev);
3589         if (pi->voltage_control) {
3590                 rv770_enable_voltage_control(rdev, true);
3591                 ret = cypress_construct_voltage_tables(rdev);
3592                 if (ret) {
3593                         DRM_ERROR("cypress_construct_voltage_tables failed\n");
3594                         return ret;
3595                 }
3596         }
3597         if (eg_pi->dynamic_ac_timing) {
3598                 ret = ni_initialize_mc_reg_table(rdev);
3599                 if (ret)
3600                         eg_pi->dynamic_ac_timing = false;
3601         }
3602         if (pi->dynamic_ss)
3603                 cypress_enable_spread_spectrum(rdev, true);
3604         if (pi->thermal_protection)
3605                 rv770_enable_thermal_protection(rdev, true);
3606         rv770_setup_bsp(rdev);
3607         rv770_program_git(rdev);
3608         rv770_program_tp(rdev);
3609         rv770_program_tpp(rdev);
3610         rv770_program_sstp(rdev);
3611         cypress_enable_display_gap(rdev);
3612         rv770_program_vc(rdev);
3613         if (pi->dynamic_pcie_gen2)
3614                 ni_enable_dynamic_pcie_gen2(rdev, true);
3615         ret = rv770_upload_firmware(rdev);
3616         if (ret) {
3617                 DRM_ERROR("rv770_upload_firmware failed\n");
3618                 return ret;
3619         }
3620         ret = ni_process_firmware_header(rdev);
3621         if (ret) {
3622                 DRM_ERROR("ni_process_firmware_header failed\n");
3623                 return ret;
3624         }
3625         ret = ni_initial_switch_from_arb_f0_to_f1(rdev);
3626         if (ret) {
3627                 DRM_ERROR("ni_initial_switch_from_arb_f0_to_f1 failed\n");
3628                 return ret;
3629         }
3630         ret = ni_init_smc_table(rdev);
3631         if (ret) {
3632                 DRM_ERROR("ni_init_smc_table failed\n");
3633                 return ret;
3634         }
3635         ret = ni_init_smc_spll_table(rdev);
3636         if (ret) {
3637                 DRM_ERROR("ni_init_smc_spll_table failed\n");
3638                 return ret;
3639         }
3640         ret = ni_init_arb_table_index(rdev);
3641         if (ret) {
3642                 DRM_ERROR("ni_init_arb_table_index failed\n");
3643                 return ret;
3644         }
3645         if (eg_pi->dynamic_ac_timing) {
3646                 ret = ni_populate_mc_reg_table(rdev, boot_ps);
3647                 if (ret) {
3648                         DRM_ERROR("ni_populate_mc_reg_table failed\n");
3649                         return ret;
3650                 }
3651         }
3652         ret = ni_initialize_smc_cac_tables(rdev);
3653         if (ret) {
3654                 DRM_ERROR("ni_initialize_smc_cac_tables failed\n");
3655                 return ret;
3656         }
3657         ret = ni_initialize_hardware_cac_manager(rdev);
3658         if (ret) {
3659                 DRM_ERROR("ni_initialize_hardware_cac_manager failed\n");
3660                 return ret;
3661         }
3662         ret = ni_populate_smc_tdp_limits(rdev, boot_ps);
3663         if (ret) {
3664                 DRM_ERROR("ni_populate_smc_tdp_limits failed\n");
3665                 return ret;
3666         }
3667         ni_program_response_times(rdev);
3668         r7xx_start_smc(rdev);
3669         ret = cypress_notify_smc_display_change(rdev, false);
3670         if (ret) {
3671                 DRM_ERROR("cypress_notify_smc_display_change failed\n");
3672                 return ret;
3673         }
3674         cypress_enable_sclk_control(rdev, true);
3675         if (eg_pi->memory_transition)
3676                 cypress_enable_mclk_control(rdev, true);
3677         cypress_start_dpm(rdev);
3678         if (pi->gfx_clock_gating)
3679                 ni_gfx_clockgating_enable(rdev, true);
3680         if (pi->mg_clock_gating)
3681                 ni_mg_clockgating_enable(rdev, true);
3682         if (eg_pi->ls_clock_gating)
3683                 ni_ls_clockgating_enable(rdev, true);
3684
3685         if (rdev->irq.installed &&
3686             r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
3687                 PPSMC_Result result;
3688
3689                 ret = rv770_set_thermal_temperature_range(rdev, R600_TEMP_RANGE_MIN, 0xff * 1000);
3690                 if (ret)
3691                         return ret;
3692                 rdev->irq.dpm_thermal = true;
3693                 radeon_irq_set(rdev);
3694                 result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_EnableThermalInterrupt);
3695
3696                 if (result != PPSMC_Result_OK)
3697                         DRM_DEBUG_KMS("Could not enable thermal interrupts.\n");
3698         }
3699
3700         rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, true);
3701
3702         ni_update_current_ps(rdev, boot_ps);
3703
3704         return 0;
3705 }
3706
3707 void ni_dpm_disable(struct radeon_device *rdev)
3708 {
3709         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3710         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3711         struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
3712
3713         if (!btc_dpm_enabled(rdev))
3714                 return;
3715         rv770_clear_vc(rdev);
3716         if (pi->thermal_protection)
3717                 rv770_enable_thermal_protection(rdev, false);
3718         ni_enable_power_containment(rdev, boot_ps, false);
3719         ni_enable_smc_cac(rdev, boot_ps, false);
3720         cypress_enable_spread_spectrum(rdev, false);
3721         rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, false);
3722         if (pi->dynamic_pcie_gen2)
3723                 ni_enable_dynamic_pcie_gen2(rdev, false);
3724
3725         if (rdev->irq.installed &&
3726             r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
3727                 rdev->irq.dpm_thermal = false;
3728                 radeon_irq_set(rdev);
3729         }
3730
3731         if (pi->gfx_clock_gating)
3732                 ni_gfx_clockgating_enable(rdev, false);
3733         if (pi->mg_clock_gating)
3734                 ni_mg_clockgating_enable(rdev, false);
3735         if (eg_pi->ls_clock_gating)
3736                 ni_ls_clockgating_enable(rdev, false);
3737         ni_stop_dpm(rdev);
3738         btc_reset_to_default(rdev);
3739         ni_stop_smc(rdev);
3740         ni_force_switch_to_arb_f0(rdev);
3741
3742         ni_update_current_ps(rdev, boot_ps);
3743 }
3744
3745 static int ni_power_control_set_level(struct radeon_device *rdev)
3746 {
3747         struct radeon_ps *new_ps = rdev->pm.dpm.requested_ps;
3748         int ret;
3749
3750         ret = ni_restrict_performance_levels_before_switch(rdev);
3751         if (ret)
3752                 return ret;
3753         ret = rv770_halt_smc(rdev);
3754         if (ret)
3755                 return ret;
3756         ret = ni_populate_smc_tdp_limits(rdev, new_ps);
3757         if (ret)
3758                 return ret;
3759         ret = rv770_resume_smc(rdev);
3760         if (ret)
3761                 return ret;
3762         ret = rv770_set_sw_state(rdev);
3763         if (ret)
3764                 return ret;
3765
3766         return 0;
3767 }
3768
3769 int ni_dpm_pre_set_power_state(struct radeon_device *rdev)
3770 {
3771         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3772         struct radeon_ps requested_ps = *rdev->pm.dpm.requested_ps;
3773         struct radeon_ps *new_ps = &requested_ps;
3774
3775         ni_update_requested_ps(rdev, new_ps);
3776
3777         ni_apply_state_adjust_rules(rdev, &eg_pi->requested_rps);
3778
3779         return 0;
3780 }
3781
3782 int ni_dpm_set_power_state(struct radeon_device *rdev)
3783 {
3784         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3785         struct radeon_ps *new_ps = &eg_pi->requested_rps;
3786         struct radeon_ps *old_ps = &eg_pi->current_rps;
3787         int ret;
3788
3789         ret = ni_restrict_performance_levels_before_switch(rdev);
3790         if (ret) {
3791                 DRM_ERROR("ni_restrict_performance_levels_before_switch failed\n");
3792                 return ret;
3793         }
3794         ni_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps);
3795         ret = ni_enable_power_containment(rdev, new_ps, false);
3796         if (ret) {
3797                 DRM_ERROR("ni_enable_power_containment failed\n");
3798                 return ret;
3799         }
3800         ret = ni_enable_smc_cac(rdev, new_ps, false);
3801         if (ret) {
3802                 DRM_ERROR("ni_enable_smc_cac failed\n");
3803                 return ret;
3804         }
3805         ret = rv770_halt_smc(rdev);
3806         if (ret) {
3807                 DRM_ERROR("rv770_halt_smc failed\n");
3808                 return ret;
3809         }
3810         if (eg_pi->smu_uvd_hs)
3811                 btc_notify_uvd_to_smc(rdev, new_ps);
3812         ret = ni_upload_sw_state(rdev, new_ps);
3813         if (ret) {
3814                 DRM_ERROR("ni_upload_sw_state failed\n");
3815                 return ret;
3816         }
3817         if (eg_pi->dynamic_ac_timing) {
3818                 ret = ni_upload_mc_reg_table(rdev, new_ps);
3819                 if (ret) {
3820                         DRM_ERROR("ni_upload_mc_reg_table failed\n");
3821                         return ret;
3822                 }
3823         }
3824         ret = ni_program_memory_timing_parameters(rdev, new_ps);
3825         if (ret) {
3826                 DRM_ERROR("ni_program_memory_timing_parameters failed\n");
3827                 return ret;
3828         }
3829         ret = rv770_resume_smc(rdev);
3830         if (ret) {
3831                 DRM_ERROR("rv770_resume_smc failed\n");
3832                 return ret;
3833         }
3834         ret = rv770_set_sw_state(rdev);
3835         if (ret) {
3836                 DRM_ERROR("rv770_set_sw_state failed\n");
3837                 return ret;
3838         }
3839         ni_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps);
3840         ret = ni_enable_smc_cac(rdev, new_ps, true);
3841         if (ret) {
3842                 DRM_ERROR("ni_enable_smc_cac failed\n");
3843                 return ret;
3844         }
3845         ret = ni_enable_power_containment(rdev, new_ps, true);
3846         if (ret) {
3847                 DRM_ERROR("ni_enable_power_containment failed\n");
3848                 return ret;
3849         }
3850
3851         /* update tdp */
3852         ret = ni_power_control_set_level(rdev);
3853         if (ret) {
3854                 DRM_ERROR("ni_power_control_set_level failed\n");
3855                 return ret;
3856         }
3857
3858         ret = ni_dpm_force_performance_level(rdev, RADEON_DPM_FORCED_LEVEL_AUTO);
3859         if (ret) {
3860                 DRM_ERROR("ni_dpm_force_performance_level failed\n");
3861                 return ret;
3862         }
3863
3864         return 0;
3865 }
3866
3867 void ni_dpm_post_set_power_state(struct radeon_device *rdev)
3868 {
3869         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3870         struct radeon_ps *new_ps = &eg_pi->requested_rps;
3871
3872         ni_update_current_ps(rdev, new_ps);
3873 }
3874
3875 void ni_dpm_reset_asic(struct radeon_device *rdev)
3876 {
3877         ni_restrict_performance_levels_before_switch(rdev);
3878         rv770_set_boot_state(rdev);
3879 }
3880
3881 union power_info {
3882         struct _ATOM_POWERPLAY_INFO info;
3883         struct _ATOM_POWERPLAY_INFO_V2 info_2;
3884         struct _ATOM_POWERPLAY_INFO_V3 info_3;
3885         struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
3886         struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
3887         struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
3888 };
3889
3890 union pplib_clock_info {
3891         struct _ATOM_PPLIB_R600_CLOCK_INFO r600;
3892         struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780;
3893         struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
3894         struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
3895 };
3896
3897 union pplib_power_state {
3898         struct _ATOM_PPLIB_STATE v1;
3899         struct _ATOM_PPLIB_STATE_V2 v2;
3900 };
3901
3902 static void ni_parse_pplib_non_clock_info(struct radeon_device *rdev,
3903                                           struct radeon_ps *rps,
3904                                           struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info,
3905                                           u8 table_rev)
3906 {
3907         rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings);
3908         rps->class = le16_to_cpu(non_clock_info->usClassification);
3909         rps->class2 = le16_to_cpu(non_clock_info->usClassification2);
3910
3911         if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) {
3912                 rps->vclk = le32_to_cpu(non_clock_info->ulVCLK);
3913                 rps->dclk = le32_to_cpu(non_clock_info->ulDCLK);
3914         } else if (r600_is_uvd_state(rps->class, rps->class2)) {
3915                 rps->vclk = RV770_DEFAULT_VCLK_FREQ;
3916                 rps->dclk = RV770_DEFAULT_DCLK_FREQ;
3917         } else {
3918                 rps->vclk = 0;
3919                 rps->dclk = 0;
3920         }
3921
3922         if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT)
3923                 rdev->pm.dpm.boot_ps = rps;
3924         if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
3925                 rdev->pm.dpm.uvd_ps = rps;
3926 }
3927
3928 static void ni_parse_pplib_clock_info(struct radeon_device *rdev,
3929                                       struct radeon_ps *rps, int index,
3930                                       union pplib_clock_info *clock_info)
3931 {
3932         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3933         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3934         struct ni_ps *ps = ni_get_ps(rps);
3935         u16 vddc;
3936         struct rv7xx_pl *pl = &ps->performance_levels[index];
3937
3938         ps->performance_level_count = index + 1;
3939
3940         pl->sclk = le16_to_cpu(clock_info->evergreen.usEngineClockLow);
3941         pl->sclk |= clock_info->evergreen.ucEngineClockHigh << 16;
3942         pl->mclk = le16_to_cpu(clock_info->evergreen.usMemoryClockLow);
3943         pl->mclk |= clock_info->evergreen.ucMemoryClockHigh << 16;
3944
3945         pl->vddc = le16_to_cpu(clock_info->evergreen.usVDDC);
3946         pl->vddci = le16_to_cpu(clock_info->evergreen.usVDDCI);
3947         pl->flags = le32_to_cpu(clock_info->evergreen.ulFlags);
3948
3949         /* patch up vddc if necessary */
3950         if (pl->vddc == 0xff01) {
3951                 if (radeon_atom_get_max_vddc(rdev, 0, 0, &vddc) == 0)
3952                         pl->vddc = vddc;
3953         }
3954
3955         if (rps->class & ATOM_PPLIB_CLASSIFICATION_ACPI) {
3956                 pi->acpi_vddc = pl->vddc;
3957                 eg_pi->acpi_vddci = pl->vddci;
3958                 if (ps->performance_levels[0].flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2)
3959                         pi->acpi_pcie_gen2 = true;
3960                 else
3961                         pi->acpi_pcie_gen2 = false;
3962         }
3963
3964         if (rps->class2 & ATOM_PPLIB_CLASSIFICATION2_ULV) {
3965                 eg_pi->ulv.supported = true;
3966                 eg_pi->ulv.pl = pl;
3967         }
3968
3969         if (pi->min_vddc_in_table > pl->vddc)
3970                 pi->min_vddc_in_table = pl->vddc;
3971
3972         if (pi->max_vddc_in_table < pl->vddc)
3973                 pi->max_vddc_in_table = pl->vddc;
3974
3975         /* patch up boot state */
3976         if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) {
3977                 u16 vddc, vddci, mvdd;
3978                 radeon_atombios_get_default_voltages(rdev, &vddc, &vddci, &mvdd);
3979                 pl->mclk = rdev->clock.default_mclk;
3980                 pl->sclk = rdev->clock.default_sclk;
3981                 pl->vddc = vddc;
3982                 pl->vddci = vddci;
3983         }
3984
3985         if ((rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) ==
3986             ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE) {
3987                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk = pl->sclk;
3988                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.mclk = pl->mclk;
3989                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddc = pl->vddc;
3990                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddci = pl->vddci;
3991         }
3992 }
3993
3994 static int ni_parse_power_table(struct radeon_device *rdev)
3995 {
3996         struct radeon_mode_info *mode_info = &rdev->mode_info;
3997         struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
3998         union pplib_power_state *power_state;
3999         int i, j;
4000         union pplib_clock_info *clock_info;
4001         union power_info *power_info;
4002         int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
4003         u16 data_offset;
4004         u8 frev, crev;
4005         struct ni_ps *ps;
4006
4007         if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
4008                                    &frev, &crev, &data_offset))
4009                 return -EINVAL;
4010         power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
4011
4012         rdev->pm.dpm.ps = kzalloc(sizeof(struct radeon_ps) *
4013                                   power_info->pplib.ucNumStates, GFP_KERNEL);
4014         if (!rdev->pm.dpm.ps)
4015                 return -ENOMEM;
4016         rdev->pm.dpm.platform_caps = le32_to_cpu(power_info->pplib.ulPlatformCaps);
4017         rdev->pm.dpm.backbias_response_time = le16_to_cpu(power_info->pplib.usBackbiasTime);
4018         rdev->pm.dpm.voltage_response_time = le16_to_cpu(power_info->pplib.usVoltageTime);
4019
4020         for (i = 0; i < power_info->pplib.ucNumStates; i++) {
4021                 power_state = (union pplib_power_state *)
4022                         (mode_info->atom_context->bios + data_offset +
4023                          le16_to_cpu(power_info->pplib.usStateArrayOffset) +
4024                          i * power_info->pplib.ucStateEntrySize);
4025                 non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
4026                         (mode_info->atom_context->bios + data_offset +
4027                          le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset) +
4028                          (power_state->v1.ucNonClockStateIndex *
4029                           power_info->pplib.ucNonClockSize));
4030                 if (power_info->pplib.ucStateEntrySize - 1) {
4031                         ps = kzalloc(sizeof(struct ni_ps), GFP_KERNEL);
4032                         if (ps == NULL) {
4033                                 kfree(rdev->pm.dpm.ps);
4034                                 return -ENOMEM;
4035                         }
4036                         rdev->pm.dpm.ps[i].ps_priv = ps;
4037                         ni_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i],
4038                                                          non_clock_info,
4039                                                          power_info->pplib.ucNonClockSize);
4040                         for (j = 0; j < (power_info->pplib.ucStateEntrySize - 1); j++) {
4041                                 clock_info = (union pplib_clock_info *)
4042                                         (mode_info->atom_context->bios + data_offset +
4043                                          le16_to_cpu(power_info->pplib.usClockInfoArrayOffset) +
4044                                          (power_state->v1.ucClockStateIndices[j] *
4045                                           power_info->pplib.ucClockInfoSize));
4046                                 ni_parse_pplib_clock_info(rdev,
4047                                                           &rdev->pm.dpm.ps[i], j,
4048                                                           clock_info);
4049                         }
4050                 }
4051         }
4052         rdev->pm.dpm.num_ps = power_info->pplib.ucNumStates;
4053         return 0;
4054 }
4055
4056 int ni_dpm_init(struct radeon_device *rdev)
4057 {
4058         struct rv7xx_power_info *pi;
4059         struct evergreen_power_info *eg_pi;
4060         struct ni_power_info *ni_pi;
4061         int index = GetIndexIntoMasterTable(DATA, ASIC_InternalSS_Info);
4062         u16 data_offset, size;
4063         u8 frev, crev;
4064         struct atom_clock_dividers dividers;
4065         int ret;
4066
4067         ni_pi = kzalloc(sizeof(struct ni_power_info), GFP_KERNEL);
4068         if (ni_pi == NULL)
4069                 return -ENOMEM;
4070         rdev->pm.dpm.priv = ni_pi;
4071         eg_pi = &ni_pi->eg;
4072         pi = &eg_pi->rv7xx;
4073
4074         rv770_get_max_vddc(rdev);
4075
4076         eg_pi->ulv.supported = false;
4077         pi->acpi_vddc = 0;
4078         eg_pi->acpi_vddci = 0;
4079         pi->min_vddc_in_table = 0;
4080         pi->max_vddc_in_table = 0;
4081
4082         ret = ni_parse_power_table(rdev);
4083         if (ret)
4084                 return ret;
4085         ret = r600_parse_extended_power_table(rdev);
4086         if (ret)
4087                 return ret;
4088
4089         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries =
4090                 kzalloc(4 * sizeof(struct radeon_clock_voltage_dependency_entry), GFP_KERNEL);
4091         if (!rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries) {
4092                 r600_free_extended_power_table(rdev);
4093                 return -ENOMEM;
4094         }
4095         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.count = 4;
4096         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].clk = 0;
4097         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].v = 0;
4098         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].clk = 36000;
4099         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].v = 720;
4100         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].clk = 54000;
4101         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].v = 810;
4102         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].clk = 72000;
4103         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].v = 900;
4104
4105         ni_patch_dependency_tables_based_on_leakage(rdev);
4106
4107         if (rdev->pm.dpm.voltage_response_time == 0)
4108                 rdev->pm.dpm.voltage_response_time = R600_VOLTAGERESPONSETIME_DFLT;
4109         if (rdev->pm.dpm.backbias_response_time == 0)
4110                 rdev->pm.dpm.backbias_response_time = R600_BACKBIASRESPONSETIME_DFLT;
4111
4112         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
4113                                              0, false, &dividers);
4114         if (ret)
4115                 pi->ref_div = dividers.ref_div + 1;
4116         else
4117                 pi->ref_div = R600_REFERENCEDIVIDER_DFLT;
4118
4119         pi->rlp = RV770_RLP_DFLT;
4120         pi->rmp = RV770_RMP_DFLT;
4121         pi->lhp = RV770_LHP_DFLT;
4122         pi->lmp = RV770_LMP_DFLT;
4123
4124         eg_pi->ats[0].rlp = RV770_RLP_DFLT;
4125         eg_pi->ats[0].rmp = RV770_RMP_DFLT;
4126         eg_pi->ats[0].lhp = RV770_LHP_DFLT;
4127         eg_pi->ats[0].lmp = RV770_LMP_DFLT;
4128
4129         eg_pi->ats[1].rlp = BTC_RLP_UVD_DFLT;
4130         eg_pi->ats[1].rmp = BTC_RMP_UVD_DFLT;
4131         eg_pi->ats[1].lhp = BTC_LHP_UVD_DFLT;
4132         eg_pi->ats[1].lmp = BTC_LMP_UVD_DFLT;
4133
4134         eg_pi->smu_uvd_hs = true;
4135
4136         if (rdev->pdev->device == 0x6707) {
4137                 pi->mclk_strobe_mode_threshold = 55000;
4138                 pi->mclk_edc_enable_threshold = 55000;
4139                 eg_pi->mclk_edc_wr_enable_threshold = 55000;
4140         } else {
4141                 pi->mclk_strobe_mode_threshold = 40000;
4142                 pi->mclk_edc_enable_threshold = 40000;
4143                 eg_pi->mclk_edc_wr_enable_threshold = 40000;
4144         }
4145         ni_pi->mclk_rtt_mode_threshold = eg_pi->mclk_edc_wr_enable_threshold;
4146
4147         pi->voltage_control =
4148                 radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, 0);
4149
4150         pi->mvdd_control =
4151                 radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_MVDDC, 0);
4152
4153         eg_pi->vddci_control =
4154                 radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDCI, 0);
4155
4156         if (atom_parse_data_header(rdev->mode_info.atom_context, index, &size,
4157                                    &frev, &crev, &data_offset)) {
4158                 pi->sclk_ss = true;
4159                 pi->mclk_ss = true;
4160                 pi->dynamic_ss = true;
4161         } else {
4162                 pi->sclk_ss = false;
4163                 pi->mclk_ss = false;
4164                 pi->dynamic_ss = true;
4165         }
4166
4167         pi->asi = RV770_ASI_DFLT;
4168         pi->pasi = CYPRESS_HASI_DFLT;
4169         pi->vrc = CYPRESS_VRC_DFLT;
4170
4171         pi->power_gating = false;
4172
4173         pi->gfx_clock_gating = true;
4174
4175         pi->mg_clock_gating = true;
4176         pi->mgcgtssm = true;
4177         eg_pi->ls_clock_gating = false;
4178         eg_pi->sclk_deep_sleep = false;
4179
4180         pi->dynamic_pcie_gen2 = true;
4181
4182         if (pi->gfx_clock_gating &&
4183             (rdev->pm.int_thermal_type != THERMAL_TYPE_NONE))
4184                 pi->thermal_protection = true;
4185         else
4186                 pi->thermal_protection = false;
4187
4188         pi->display_gap = true;
4189
4190         pi->dcodt = true;
4191
4192         pi->ulps = true;
4193
4194         eg_pi->dynamic_ac_timing = true;
4195         eg_pi->abm = true;
4196         eg_pi->mcls = true;
4197         eg_pi->light_sleep = true;
4198         eg_pi->memory_transition = true;
4199 #if defined(CONFIG_ACPI)
4200         eg_pi->pcie_performance_request =
4201                 radeon_acpi_is_pcie_performance_request_supported(rdev);
4202 #else
4203         eg_pi->pcie_performance_request = false;
4204 #endif
4205
4206         eg_pi->dll_default_on = false;
4207
4208         eg_pi->sclk_deep_sleep = false;
4209
4210         pi->mclk_stutter_mode_threshold = 0;
4211
4212         pi->sram_end = SMC_RAM_END;
4213
4214         rdev->pm.dpm.dyn_state.mclk_sclk_ratio = 3;
4215         rdev->pm.dpm.dyn_state.vddc_vddci_delta = 200;
4216         rdev->pm.dpm.dyn_state.min_vddc_for_pcie_gen2 = 900;
4217         rdev->pm.dpm.dyn_state.valid_sclk_values.count = ARRAY_SIZE(btc_valid_sclk);
4218         rdev->pm.dpm.dyn_state.valid_sclk_values.values = btc_valid_sclk;
4219         rdev->pm.dpm.dyn_state.valid_mclk_values.count = 0;
4220         rdev->pm.dpm.dyn_state.valid_mclk_values.values = NULL;
4221         rdev->pm.dpm.dyn_state.sclk_mclk_delta = 12500;
4222
4223         ni_pi->cac_data.leakage_coefficients.at = 516;
4224         ni_pi->cac_data.leakage_coefficients.bt = 18;
4225         ni_pi->cac_data.leakage_coefficients.av = 51;
4226         ni_pi->cac_data.leakage_coefficients.bv = 2957;
4227
4228         switch (rdev->pdev->device) {
4229         case 0x6700:
4230         case 0x6701:
4231         case 0x6702:
4232         case 0x6703:
4233         case 0x6718:
4234                 ni_pi->cac_weights = &cac_weights_cayman_xt;
4235                 break;
4236         case 0x6705:
4237         case 0x6719:
4238         case 0x671D:
4239         case 0x671C:
4240         default:
4241                 ni_pi->cac_weights = &cac_weights_cayman_pro;
4242                 break;
4243         case 0x6704:
4244         case 0x6706:
4245         case 0x6707:
4246         case 0x6708:
4247         case 0x6709:
4248                 ni_pi->cac_weights = &cac_weights_cayman_le;
4249                 break;
4250         }
4251
4252         if (ni_pi->cac_weights->enable_power_containment_by_default) {
4253                 ni_pi->enable_power_containment = true;
4254                 ni_pi->enable_cac = true;
4255                 ni_pi->enable_sq_ramping = true;
4256         } else {
4257                 ni_pi->enable_power_containment = false;
4258                 ni_pi->enable_cac = false;
4259                 ni_pi->enable_sq_ramping = false;
4260         }
4261
4262         ni_pi->driver_calculate_cac_leakage = false;
4263         ni_pi->cac_configuration_required = true;
4264
4265         if (ni_pi->cac_configuration_required) {
4266                 ni_pi->support_cac_long_term_average = true;
4267                 ni_pi->lta_window_size = ni_pi->cac_weights->l2_lta_window_size;
4268                 ni_pi->lts_truncate = ni_pi->cac_weights->lts_truncate;
4269         } else {
4270                 ni_pi->support_cac_long_term_average = false;
4271                 ni_pi->lta_window_size = 0;
4272                 ni_pi->lts_truncate = 0;
4273         }
4274
4275         ni_pi->use_power_boost_limit = true;
4276
4277         return 0;
4278 }
4279
4280 void ni_dpm_fini(struct radeon_device *rdev)
4281 {
4282         int i;
4283
4284         for (i = 0; i < rdev->pm.dpm.num_ps; i++) {
4285                 kfree(rdev->pm.dpm.ps[i].ps_priv);
4286         }
4287         kfree(rdev->pm.dpm.ps);
4288         kfree(rdev->pm.dpm.priv);
4289         kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries);
4290         r600_free_extended_power_table(rdev);
4291 }
4292
4293 void ni_dpm_print_power_state(struct radeon_device *rdev,
4294                               struct radeon_ps *rps)
4295 {
4296         struct ni_ps *ps = ni_get_ps(rps);
4297         struct rv7xx_pl *pl;
4298         int i;
4299
4300         r600_dpm_print_class_info(rps->class, rps->class2);
4301         r600_dpm_print_cap_info(rps->caps);
4302         printk("\tuvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
4303         for (i = 0; i < ps->performance_level_count; i++) {
4304                 pl = &ps->performance_levels[i];
4305                 if (rdev->family >= CHIP_TAHITI)
4306                         printk("\t\tpower level %d    sclk: %u mclk: %u vddc: %u vddci: %u pcie gen: %u\n",
4307                                i, pl->sclk, pl->mclk, pl->vddc, pl->vddci, pl->pcie_gen + 1);
4308                 else
4309                         printk("\t\tpower level %d    sclk: %u mclk: %u vddc: %u vddci: %u\n",
4310                                i, pl->sclk, pl->mclk, pl->vddc, pl->vddci);
4311         }
4312         r600_dpm_print_ps_status(rdev, rps);
4313 }
4314
4315 void ni_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev,
4316                                                     struct seq_file *m)
4317 {
4318         struct radeon_ps *rps = rdev->pm.dpm.current_ps;
4319         struct ni_ps *ps = ni_get_ps(rps);
4320         struct rv7xx_pl *pl;
4321         u32 current_index =
4322                 (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_INDEX_MASK) >>
4323                 CURRENT_STATE_INDEX_SHIFT;
4324
4325         if (current_index >= ps->performance_level_count) {
4326                 seq_printf(m, "invalid dpm profile %d\n", current_index);
4327         } else {
4328                 pl = &ps->performance_levels[current_index];
4329                 seq_printf(m, "uvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
4330                 seq_printf(m, "power level %d    sclk: %u mclk: %u vddc: %u vddci: %u\n",
4331                            current_index, pl->sclk, pl->mclk, pl->vddc, pl->vddci);
4332         }
4333 }
4334
4335 u32 ni_dpm_get_sclk(struct radeon_device *rdev, bool low)
4336 {
4337         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4338         struct ni_ps *requested_state = ni_get_ps(&eg_pi->requested_rps);
4339
4340         if (low)
4341                 return requested_state->performance_levels[0].sclk;
4342         else
4343                 return requested_state->performance_levels[requested_state->performance_level_count - 1].sclk;
4344 }
4345
4346 u32 ni_dpm_get_mclk(struct radeon_device *rdev, bool low)
4347 {
4348         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4349         struct ni_ps *requested_state = ni_get_ps(&eg_pi->requested_rps);
4350
4351         if (low)
4352                 return requested_state->performance_levels[0].mclk;
4353         else
4354                 return requested_state->performance_levels[requested_state->performance_level_count - 1].mclk;
4355 }
4356