drivers/gpu/drm/radeon/evergreen.c

   1 /*
   2  * Copyright 2010 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/platform_device.h>
  26 #include <linux/slab.h>
  27 #include <drm/drmP.h>
  28 #include "radeon.h"
  29 #include "radeon_asic.h"
  30 #include <drm/radeon_drm.h>
  31 #include "evergreend.h"
  32 #include "atom.h"
  33 #include "avivod.h"
  34 #include "evergreen_reg.h"
  35 #include "evergreen_blit_shaders.h"
  36 #include "radeon_ucode.h"
  37
  38 static const u32 crtc_offsets[6] =
  39 {
  40         EVERGREEN_CRTC0_REGISTER_OFFSET,
  41         EVERGREEN_CRTC1_REGISTER_OFFSET,
  42         EVERGREEN_CRTC2_REGISTER_OFFSET,
  43         EVERGREEN_CRTC3_REGISTER_OFFSET,
  44         EVERGREEN_CRTC4_REGISTER_OFFSET,
  45         EVERGREEN_CRTC5_REGISTER_OFFSET
  46 };
  47
  48 #include "clearstate_evergreen.h"
  49
  50 static u32 sumo_rlc_save_restore_register_list[] =
  51 {
  52         0x98fc,
  53         0x9830,
  54         0x9834,
  55         0x9838,
  56         0x9870,
  57         0x9874,
  58         0x8a14,
  59         0x8b24,
  60         0x8bcc,
  61         0x8b10,
  62         0x8d00,
  63         0x8d04,
  64         0x8c00,
  65         0x8c04,
  66         0x8c08,
  67         0x8c0c,
  68         0x8d8c,
  69         0x8c20,
  70         0x8c24,
  71         0x8c28,
  72         0x8c18,
  73         0x8c1c,
  74         0x8cf0,
  75         0x8e2c,
  76         0x8e38,
  77         0x8c30,
  78         0x9508,
  79         0x9688,
  80         0x9608,
  81         0x960c,
  82         0x9610,
  83         0x9614,
  84         0x88c4,
  85         0x88d4,
  86         0xa008,
  87         0x900c,
  88         0x9100,
  89         0x913c,
  90         0x98f8,
  91         0x98f4,
  92         0x9b7c,
  93         0x3f8c,
  94         0x8950,
  95         0x8954,
  96         0x8a18,
  97         0x8b28,
  98         0x9144,
  99         0x9148,
 100         0x914c,
 101         0x3f90,
 102         0x3f94,
 103         0x915c,
 104         0x9160,
 105         0x9178,
 106         0x917c,
 107         0x9180,
 108         0x918c,
 109         0x9190,
 110         0x9194,
 111         0x9198,
 112         0x919c,
 113         0x91a8,
 114         0x91ac,
 115         0x91b0,
 116         0x91b4,
 117         0x91b8,
 118         0x91c4,
 119         0x91c8,
 120         0x91cc,
 121         0x91d0,
 122         0x91d4,
 123         0x91e0,
 124         0x91e4,
 125         0x91ec,
 126         0x91f0,
 127         0x91f4,
 128         0x9200,
 129         0x9204,
 130         0x929c,
 131         0x9150,
 132         0x802c,
 133 };
 134 static u32 sumo_rlc_save_restore_register_list_size = ARRAY_SIZE(sumo_rlc_save_restore_register_list);
 135
 136 static void evergreen_gpu_init(struct radeon_device *rdev);
 137 void evergreen_fini(struct radeon_device *rdev);
 138 void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
 139 void evergreen_program_aspm(struct radeon_device *rdev);
 140 extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
 141                                      int ring, u32 cp_int_cntl);
 142
 143 static const u32 evergreen_golden_registers[] =
 144 {
 145         0x3f90, 0xffff0000, 0xff000000,
 146         0x9148, 0xffff0000, 0xff000000,
 147         0x3f94, 0xffff0000, 0xff000000,
 148         0x914c, 0xffff0000, 0xff000000,
 149         0x9b7c, 0xffffffff, 0x00000000,
 150         0x8a14, 0xffffffff, 0x00000007,
 151         0x8b10, 0xffffffff, 0x00000000,
 152         0x960c, 0xffffffff, 0x54763210,
 153         0x88c4, 0xffffffff, 0x000000c2,
 154         0x88d4, 0xffffffff, 0x00000010,
 155         0x8974, 0xffffffff, 0x00000000,
 156         0xc78, 0x00000080, 0x00000080,
 157         0x5eb4, 0xffffffff, 0x00000002,
 158         0x5e78, 0xffffffff, 0x001000f0,
 159         0x6104, 0x01000300, 0x00000000,
 160         0x5bc0, 0x00300000, 0x00000000,
 161         0x7030, 0xffffffff, 0x00000011,
 162         0x7c30, 0xffffffff, 0x00000011,
 163         0x10830, 0xffffffff, 0x00000011,
 164         0x11430, 0xffffffff, 0x00000011,
 165         0x12030, 0xffffffff, 0x00000011,
 166         0x12c30, 0xffffffff, 0x00000011,
 167         0xd02c, 0xffffffff, 0x08421000,
 168         0x240c, 0xffffffff, 0x00000380,
 169         0x8b24, 0xffffffff, 0x00ff0fff,
 170         0x28a4c, 0x06000000, 0x06000000,
 171         0x10c, 0x00000001, 0x00000001,
 172         0x8d00, 0xffffffff, 0x100e4848,
 173         0x8d04, 0xffffffff, 0x00164745,
 174         0x8c00, 0xffffffff, 0xe4000003,
 175         0x8c04, 0xffffffff, 0x40600060,
 176         0x8c08, 0xffffffff, 0x001c001c,
 177         0x8cf0, 0xffffffff, 0x08e00620,
 178         0x8c20, 0xffffffff, 0x00800080,
 179         0x8c24, 0xffffffff, 0x00800080,
 180         0x8c18, 0xffffffff, 0x20202078,
 181         0x8c1c, 0xffffffff, 0x00001010,
 182         0x28350, 0xffffffff, 0x00000000,
 183         0xa008, 0xffffffff, 0x00010000,
 184         0x5cc, 0xffffffff, 0x00000001,
 185         0x9508, 0xffffffff, 0x00000002,
 186         0x913c, 0x0000000f, 0x0000000a
 187 };
 188
 189 static const u32 evergreen_golden_registers2[] =
 190 {
 191         0x2f4c, 0xffffffff, 0x00000000,
 192         0x54f4, 0xffffffff, 0x00000000,
 193         0x54f0, 0xffffffff, 0x00000000,
 194         0x5498, 0xffffffff, 0x00000000,
 195         0x549c, 0xffffffff, 0x00000000,
 196         0x5494, 0xffffffff, 0x00000000,
 197         0x53cc, 0xffffffff, 0x00000000,
 198         0x53c8, 0xffffffff, 0x00000000,
 199         0x53c4, 0xffffffff, 0x00000000,
 200         0x53c0, 0xffffffff, 0x00000000,
 201         0x53bc, 0xffffffff, 0x00000000,
 202         0x53b8, 0xffffffff, 0x00000000,
 203         0x53b4, 0xffffffff, 0x00000000,
 204         0x53b0, 0xffffffff, 0x00000000
 205 };
 206
 207 static const u32 cypress_mgcg_init[] =
 208 {
 209         0x802c, 0xffffffff, 0xc0000000,
 210         0x5448, 0xffffffff, 0x00000100,
 211         0x55e4, 0xffffffff, 0x00000100,
 212         0x160c, 0xffffffff, 0x00000100,
 213         0x5644, 0xffffffff, 0x00000100,
 214         0xc164, 0xffffffff, 0x00000100,
 215         0x8a18, 0xffffffff, 0x00000100,
 216         0x897c, 0xffffffff, 0x06000100,
 217         0x8b28, 0xffffffff, 0x00000100,
 218         0x9144, 0xffffffff, 0x00000100,
 219         0x9a60, 0xffffffff, 0x00000100,
 220         0x9868, 0xffffffff, 0x00000100,
 221         0x8d58, 0xffffffff, 0x00000100,
 222         0x9510, 0xffffffff, 0x00000100,
 223         0x949c, 0xffffffff, 0x00000100,
 224         0x9654, 0xffffffff, 0x00000100,
 225         0x9030, 0xffffffff, 0x00000100,
 226         0x9034, 0xffffffff, 0x00000100,
 227         0x9038, 0xffffffff, 0x00000100,
 228         0x903c, 0xffffffff, 0x00000100,
 229         0x9040, 0xffffffff, 0x00000100,
 230         0xa200, 0xffffffff, 0x00000100,
 231         0xa204, 0xffffffff, 0x00000100,
 232         0xa208, 0xffffffff, 0x00000100,
 233         0xa20c, 0xffffffff, 0x00000100,
 234         0x971c, 0xffffffff, 0x00000100,
 235         0x977c, 0xffffffff, 0x00000100,
 236         0x3f80, 0xffffffff, 0x00000100,
 237         0xa210, 0xffffffff, 0x00000100,
 238         0xa214, 0xffffffff, 0x00000100,
 239         0x4d8, 0xffffffff, 0x00000100,
 240         0x9784, 0xffffffff, 0x00000100,
 241         0x9698, 0xffffffff, 0x00000100,
 242         0x4d4, 0xffffffff, 0x00000200,
 243         0x30cc, 0xffffffff, 0x00000100,
 244         0xd0c0, 0xffffffff, 0xff000100,
 245         0x802c, 0xffffffff, 0x40000000,
 246         0x915c, 0xffffffff, 0x00010000,
 247         0x9160, 0xffffffff, 0x00030002,
 248         0x9178, 0xffffffff, 0x00070000,
 249         0x917c, 0xffffffff, 0x00030002,
 250         0x9180, 0xffffffff, 0x00050004,
 251         0x918c, 0xffffffff, 0x00010006,
 252         0x9190, 0xffffffff, 0x00090008,
 253         0x9194, 0xffffffff, 0x00070000,
 254         0x9198, 0xffffffff, 0x00030002,
 255         0x919c, 0xffffffff, 0x00050004,
 256         0x91a8, 0xffffffff, 0x00010006,
 257         0x91ac, 0xffffffff, 0x00090008,
 258         0x91b0, 0xffffffff, 0x00070000,
 259         0x91b4, 0xffffffff, 0x00030002,
 260         0x91b8, 0xffffffff, 0x00050004,
 261         0x91c4, 0xffffffff, 0x00010006,
 262         0x91c8, 0xffffffff, 0x00090008,
 263         0x91cc, 0xffffffff, 0x00070000,
 264         0x91d0, 0xffffffff, 0x00030002,
 265         0x91d4, 0xffffffff, 0x00050004,
 266         0x91e0, 0xffffffff, 0x00010006,
 267         0x91e4, 0xffffffff, 0x00090008,
 268         0x91e8, 0xffffffff, 0x00000000,
 269         0x91ec, 0xffffffff, 0x00070000,
 270         0x91f0, 0xffffffff, 0x00030002,
 271         0x91f4, 0xffffffff, 0x00050004,
 272         0x9200, 0xffffffff, 0x00010006,
 273         0x9204, 0xffffffff, 0x00090008,
 274         0x9208, 0xffffffff, 0x00070000,
 275         0x920c, 0xffffffff, 0x00030002,
 276         0x9210, 0xffffffff, 0x00050004,
 277         0x921c, 0xffffffff, 0x00010006,
 278         0x9220, 0xffffffff, 0x00090008,
 279         0x9224, 0xffffffff, 0x00070000,
 280         0x9228, 0xffffffff, 0x00030002,
 281         0x922c, 0xffffffff, 0x00050004,
 282         0x9238, 0xffffffff, 0x00010006,
 283         0x923c, 0xffffffff, 0x00090008,
 284         0x9240, 0xffffffff, 0x00070000,
 285         0x9244, 0xffffffff, 0x00030002,
 286         0x9248, 0xffffffff, 0x00050004,
 287         0x9254, 0xffffffff, 0x00010006,
 288         0x9258, 0xffffffff, 0x00090008,
 289         0x925c, 0xffffffff, 0x00070000,
 290         0x9260, 0xffffffff, 0x00030002,
 291         0x9264, 0xffffffff, 0x00050004,
 292         0x9270, 0xffffffff, 0x00010006,
 293         0x9274, 0xffffffff, 0x00090008,
 294         0x9278, 0xffffffff, 0x00070000,
 295         0x927c, 0xffffffff, 0x00030002,
 296         0x9280, 0xffffffff, 0x00050004,
 297         0x928c, 0xffffffff, 0x00010006,
 298         0x9290, 0xffffffff, 0x00090008,
 299         0x9294, 0xffffffff, 0x00000000,
 300         0x929c, 0xffffffff, 0x00000001,
 301         0x802c, 0xffffffff, 0x40010000,
 302         0x915c, 0xffffffff, 0x00010000,
 303         0x9160, 0xffffffff, 0x00030002,
 304         0x9178, 0xffffffff, 0x00070000,
 305         0x917c, 0xffffffff, 0x00030002,
 306         0x9180, 0xffffffff, 0x00050004,
 307         0x918c, 0xffffffff, 0x00010006,
 308         0x9190, 0xffffffff, 0x00090008,
 309         0x9194, 0xffffffff, 0x00070000,
 310         0x9198, 0xffffffff, 0x00030002,
 311         0x919c, 0xffffffff, 0x00050004,
 312         0x91a8, 0xffffffff, 0x00010006,
 313         0x91ac, 0xffffffff, 0x00090008,
 314         0x91b0, 0xffffffff, 0x00070000,
 315         0x91b4, 0xffffffff, 0x00030002,
 316         0x91b8, 0xffffffff, 0x00050004,
 317         0x91c4, 0xffffffff, 0x00010006,
 318         0x91c8, 0xffffffff, 0x00090008,
 319         0x91cc, 0xffffffff, 0x00070000,
 320         0x91d0, 0xffffffff, 0x00030002,
 321         0x91d4, 0xffffffff, 0x00050004,
 322         0x91e0, 0xffffffff, 0x00010006,
 323         0x91e4, 0xffffffff, 0x00090008,
 324         0x91e8, 0xffffffff, 0x00000000,
 325         0x91ec, 0xffffffff, 0x00070000,
 326         0x91f0, 0xffffffff, 0x00030002,
 327         0x91f4, 0xffffffff, 0x00050004,
 328         0x9200, 0xffffffff, 0x00010006,
 329         0x9204, 0xffffffff, 0x00090008,
 330         0x9208, 0xffffffff, 0x00070000,
 331         0x920c, 0xffffffff, 0x00030002,
 332         0x9210, 0xffffffff, 0x00050004,
 333         0x921c, 0xffffffff, 0x00010006,
 334         0x9220, 0xffffffff, 0x00090008,
 335         0x9224, 0xffffffff, 0x00070000,
 336         0x9228, 0xffffffff, 0x00030002,
 337         0x922c, 0xffffffff, 0x00050004,
 338         0x9238, 0xffffffff, 0x00010006,
 339         0x923c, 0xffffffff, 0x00090008,
 340         0x9240, 0xffffffff, 0x00070000,
 341         0x9244, 0xffffffff, 0x00030002,
 342         0x9248, 0xffffffff, 0x00050004,
 343         0x9254, 0xffffffff, 0x00010006,
 344         0x9258, 0xffffffff, 0x00090008,
 345         0x925c, 0xffffffff, 0x00070000,
 346         0x9260, 0xffffffff, 0x00030002,
 347         0x9264, 0xffffffff, 0x00050004,
 348         0x9270, 0xffffffff, 0x00010006,
 349         0x9274, 0xffffffff, 0x00090008,
 350         0x9278, 0xffffffff, 0x00070000,
 351         0x927c, 0xffffffff, 0x00030002,
 352         0x9280, 0xffffffff, 0x00050004,
 353         0x928c, 0xffffffff, 0x00010006,
 354         0x9290, 0xffffffff, 0x00090008,
 355         0x9294, 0xffffffff, 0x00000000,
 356         0x929c, 0xffffffff, 0x00000001,
 357         0x802c, 0xffffffff, 0xc0000000
 358 };
 359
 360 static const u32 redwood_mgcg_init[] =
 361 {
 362         0x802c, 0xffffffff, 0xc0000000,
 363         0x5448, 0xffffffff, 0x00000100,
 364         0x55e4, 0xffffffff, 0x00000100,
 365         0x160c, 0xffffffff, 0x00000100,
 366         0x5644, 0xffffffff, 0x00000100,
 367         0xc164, 0xffffffff, 0x00000100,
 368         0x8a18, 0xffffffff, 0x00000100,
 369         0x897c, 0xffffffff, 0x06000100,
 370         0x8b28, 0xffffffff, 0x00000100,
 371         0x9144, 0xffffffff, 0x00000100,
 372         0x9a60, 0xffffffff, 0x00000100,
 373         0x9868, 0xffffffff, 0x00000100,
 374         0x8d58, 0xffffffff, 0x00000100,
 375         0x9510, 0xffffffff, 0x00000100,
 376         0x949c, 0xffffffff, 0x00000100,
 377         0x9654, 0xffffffff, 0x00000100,
 378         0x9030, 0xffffffff, 0x00000100,
 379         0x9034, 0xffffffff, 0x00000100,
 380         0x9038, 0xffffffff, 0x00000100,
 381         0x903c, 0xffffffff, 0x00000100,
 382         0x9040, 0xffffffff, 0x00000100,
 383         0xa200, 0xffffffff, 0x00000100,
 384         0xa204, 0xffffffff, 0x00000100,
 385         0xa208, 0xffffffff, 0x00000100,
 386         0xa20c, 0xffffffff, 0x00000100,
 387         0x971c, 0xffffffff, 0x00000100,
 388         0x977c, 0xffffffff, 0x00000100,
 389         0x3f80, 0xffffffff, 0x00000100,
 390         0xa210, 0xffffffff, 0x00000100,
 391         0xa214, 0xffffffff, 0x00000100,
 392         0x4d8, 0xffffffff, 0x00000100,
 393         0x9784, 0xffffffff, 0x00000100,
 394         0x9698, 0xffffffff, 0x00000100,
 395         0x4d4, 0xffffffff, 0x00000200,
 396         0x30cc, 0xffffffff, 0x00000100,
 397         0xd0c0, 0xffffffff, 0xff000100,
 398         0x802c, 0xffffffff, 0x40000000,
 399         0x915c, 0xffffffff, 0x00010000,
 400         0x9160, 0xffffffff, 0x00030002,
 401         0x9178, 0xffffffff, 0x00070000,
 402         0x917c, 0xffffffff, 0x00030002,
 403         0x9180, 0xffffffff, 0x00050004,
 404         0x918c, 0xffffffff, 0x00010006,
 405         0x9190, 0xffffffff, 0x00090008,
 406         0x9194, 0xffffffff, 0x00070000,
 407         0x9198, 0xffffffff, 0x00030002,
 408         0x919c, 0xffffffff, 0x00050004,
 409         0x91a8, 0xffffffff, 0x00010006,
 410         0x91ac, 0xffffffff, 0x00090008,
 411         0x91b0, 0xffffffff, 0x00070000,
 412         0x91b4, 0xffffffff, 0x00030002,
 413         0x91b8, 0xffffffff, 0x00050004,
 414         0x91c4, 0xffffffff, 0x00010006,
 415         0x91c8, 0xffffffff, 0x00090008,
 416         0x91cc, 0xffffffff, 0x00070000,
 417         0x91d0, 0xffffffff, 0x00030002,
 418         0x91d4, 0xffffffff, 0x00050004,
 419         0x91e0, 0xffffffff, 0x00010006,
 420         0x91e4, 0xffffffff, 0x00090008,
 421         0x91e8, 0xffffffff, 0x00000000,
 422         0x91ec, 0xffffffff, 0x00070000,
 423         0x91f0, 0xffffffff, 0x00030002,
 424         0x91f4, 0xffffffff, 0x00050004,
 425         0x9200, 0xffffffff, 0x00010006,
 426         0x9204, 0xffffffff, 0x00090008,
 427         0x9294, 0xffffffff, 0x00000000,
 428         0x929c, 0xffffffff, 0x00000001,
 429         0x802c, 0xffffffff, 0xc0000000
 430 };
 431
 432 static const u32 cedar_golden_registers[] =
 433 {
 434         0x3f90, 0xffff0000, 0xff000000,
 435         0x9148, 0xffff0000, 0xff000000,
 436         0x3f94, 0xffff0000, 0xff000000,
 437         0x914c, 0xffff0000, 0xff000000,
 438         0x9b7c, 0xffffffff, 0x00000000,
 439         0x8a14, 0xffffffff, 0x00000007,
 440         0x8b10, 0xffffffff, 0x00000000,
 441         0x960c, 0xffffffff, 0x54763210,
 442         0x88c4, 0xffffffff, 0x000000c2,
 443         0x88d4, 0xffffffff, 0x00000000,
 444         0x8974, 0xffffffff, 0x00000000,
 445         0xc78, 0x00000080, 0x00000080,
 446         0x5eb4, 0xffffffff, 0x00000002,
 447         0x5e78, 0xffffffff, 0x001000f0,
 448         0x6104, 0x01000300, 0x00000000,
 449         0x5bc0, 0x00300000, 0x00000000,
 450         0x7030, 0xffffffff, 0x00000011,
 451         0x7c30, 0xffffffff, 0x00000011,
 452         0x10830, 0xffffffff, 0x00000011,
 453         0x11430, 0xffffffff, 0x00000011,
 454         0xd02c, 0xffffffff, 0x08421000,
 455         0x240c, 0xffffffff, 0x00000380,
 456         0x8b24, 0xffffffff, 0x00ff0fff,
 457         0x28a4c, 0x06000000, 0x06000000,
 458         0x10c, 0x00000001, 0x00000001,
 459         0x8d00, 0xffffffff, 0x100e4848,
 460         0x8d04, 0xffffffff, 0x00164745,
 461         0x8c00, 0xffffffff, 0xe4000003,
 462         0x8c04, 0xffffffff, 0x40600060,
 463         0x8c08, 0xffffffff, 0x001c001c,
 464         0x8cf0, 0xffffffff, 0x08e00410,
 465         0x8c20, 0xffffffff, 0x00800080,
 466         0x8c24, 0xffffffff, 0x00800080,
 467         0x8c18, 0xffffffff, 0x20202078,
 468         0x8c1c, 0xffffffff, 0x00001010,
 469         0x28350, 0xffffffff, 0x00000000,
 470         0xa008, 0xffffffff, 0x00010000,
 471         0x5cc, 0xffffffff, 0x00000001,
 472         0x9508, 0xffffffff, 0x00000002
 473 };
 474
 475 static const u32 cedar_mgcg_init[] =
 476 {
 477         0x802c, 0xffffffff, 0xc0000000,
 478         0x5448, 0xffffffff, 0x00000100,
 479         0x55e4, 0xffffffff, 0x00000100,
 480         0x160c, 0xffffffff, 0x00000100,
 481         0x5644, 0xffffffff, 0x00000100,
 482         0xc164, 0xffffffff, 0x00000100,
 483         0x8a18, 0xffffffff, 0x00000100,
 484         0x897c, 0xffffffff, 0x06000100,
 485         0x8b28, 0xffffffff, 0x00000100,
 486         0x9144, 0xffffffff, 0x00000100,
 487         0x9a60, 0xffffffff, 0x00000100,
 488         0x9868, 0xffffffff, 0x00000100,
 489         0x8d58, 0xffffffff, 0x00000100,
 490         0x9510, 0xffffffff, 0x00000100,
 491         0x949c, 0xffffffff, 0x00000100,
 492         0x9654, 0xffffffff, 0x00000100,
 493         0x9030, 0xffffffff, 0x00000100,
 494         0x9034, 0xffffffff, 0x00000100,
 495         0x9038, 0xffffffff, 0x00000100,
 496         0x903c, 0xffffffff, 0x00000100,
 497         0x9040, 0xffffffff, 0x00000100,
 498         0xa200, 0xffffffff, 0x00000100,
 499         0xa204, 0xffffffff, 0x00000100,
 500         0xa208, 0xffffffff, 0x00000100,
 501         0xa20c, 0xffffffff, 0x00000100,
 502         0x971c, 0xffffffff, 0x00000100,
 503         0x977c, 0xffffffff, 0x00000100,
 504         0x3f80, 0xffffffff, 0x00000100,
 505         0xa210, 0xffffffff, 0x00000100,
 506         0xa214, 0xffffffff, 0x00000100,
 507         0x4d8, 0xffffffff, 0x00000100,
 508         0x9784, 0xffffffff, 0x00000100,
 509         0x9698, 0xffffffff, 0x00000100,
 510         0x4d4, 0xffffffff, 0x00000200,
 511         0x30cc, 0xffffffff, 0x00000100,
 512         0xd0c0, 0xffffffff, 0xff000100,
 513         0x802c, 0xffffffff, 0x40000000,
 514         0x915c, 0xffffffff, 0x00010000,
 515         0x9178, 0xffffffff, 0x00050000,
 516         0x917c, 0xffffffff, 0x00030002,
 517         0x918c, 0xffffffff, 0x00010004,
 518         0x9190, 0xffffffff, 0x00070006,
 519         0x9194, 0xffffffff, 0x00050000,
 520         0x9198, 0xffffffff, 0x00030002,
 521         0x91a8, 0xffffffff, 0x00010004,
 522         0x91ac, 0xffffffff, 0x00070006,
 523         0x91e8, 0xffffffff, 0x00000000,
 524         0x9294, 0xffffffff, 0x00000000,
 525         0x929c, 0xffffffff, 0x00000001,
 526         0x802c, 0xffffffff, 0xc0000000
 527 };
 528
 529 static const u32 juniper_mgcg_init[] =
 530 {
 531         0x802c, 0xffffffff, 0xc0000000,
 532         0x5448, 0xffffffff, 0x00000100,
 533         0x55e4, 0xffffffff, 0x00000100,
 534         0x160c, 0xffffffff, 0x00000100,
 535         0x5644, 0xffffffff, 0x00000100,
 536         0xc164, 0xffffffff, 0x00000100,
 537         0x8a18, 0xffffffff, 0x00000100,
 538         0x897c, 0xffffffff, 0x06000100,
 539         0x8b28, 0xffffffff, 0x00000100,
 540         0x9144, 0xffffffff, 0x00000100,
 541         0x9a60, 0xffffffff, 0x00000100,
 542         0x9868, 0xffffffff, 0x00000100,
 543         0x8d58, 0xffffffff, 0x00000100,
 544         0x9510, 0xffffffff, 0x00000100,
 545         0x949c, 0xffffffff, 0x00000100,
 546         0x9654, 0xffffffff, 0x00000100,
 547         0x9030, 0xffffffff, 0x00000100,
 548         0x9034, 0xffffffff, 0x00000100,
 549         0x9038, 0xffffffff, 0x00000100,
 550         0x903c, 0xffffffff, 0x00000100,
 551         0x9040, 0xffffffff, 0x00000100,
 552         0xa200, 0xffffffff, 0x00000100,
 553         0xa204, 0xffffffff, 0x00000100,
 554         0xa208, 0xffffffff, 0x00000100,
 555         0xa20c, 0xffffffff, 0x00000100,
 556         0x971c, 0xffffffff, 0x00000100,
 557         0xd0c0, 0xffffffff, 0xff000100,
 558         0x802c, 0xffffffff, 0x40000000,
 559         0x915c, 0xffffffff, 0x00010000,
 560         0x9160, 0xffffffff, 0x00030002,
 561         0x9178, 0xffffffff, 0x00070000,
 562         0x917c, 0xffffffff, 0x00030002,
 563         0x9180, 0xffffffff, 0x00050004,
 564         0x918c, 0xffffffff, 0x00010006,
 565         0x9190, 0xffffffff, 0x00090008,
 566         0x9194, 0xffffffff, 0x00070000,
 567         0x9198, 0xffffffff, 0x00030002,
 568         0x919c, 0xffffffff, 0x00050004,
 569         0x91a8, 0xffffffff, 0x00010006,
 570         0x91ac, 0xffffffff, 0x00090008,
 571         0x91b0, 0xffffffff, 0x00070000,
 572         0x91b4, 0xffffffff, 0x00030002,
 573         0x91b8, 0xffffffff, 0x00050004,
 574         0x91c4, 0xffffffff, 0x00010006,
 575         0x91c8, 0xffffffff, 0x00090008,
 576         0x91cc, 0xffffffff, 0x00070000,
 577         0x91d0, 0xffffffff, 0x00030002,
 578         0x91d4, 0xffffffff, 0x00050004,
 579         0x91e0, 0xffffffff, 0x00010006,
 580         0x91e4, 0xffffffff, 0x00090008,
 581         0x91e8, 0xffffffff, 0x00000000,
 582         0x91ec, 0xffffffff, 0x00070000,
 583         0x91f0, 0xffffffff, 0x00030002,
 584         0x91f4, 0xffffffff, 0x00050004,
 585         0x9200, 0xffffffff, 0x00010006,
 586         0x9204, 0xffffffff, 0x00090008,
 587         0x9208, 0xffffffff, 0x00070000,
 588         0x920c, 0xffffffff, 0x00030002,
 589         0x9210, 0xffffffff, 0x00050004,
 590         0x921c, 0xffffffff, 0x00010006,
 591         0x9220, 0xffffffff, 0x00090008,
 592         0x9224, 0xffffffff, 0x00070000,
 593         0x9228, 0xffffffff, 0x00030002,
 594         0x922c, 0xffffffff, 0x00050004,
 595         0x9238, 0xffffffff, 0x00010006,
 596         0x923c, 0xffffffff, 0x00090008,
 597         0x9240, 0xffffffff, 0x00070000,
 598         0x9244, 0xffffffff, 0x00030002,
 599         0x9248, 0xffffffff, 0x00050004,
 600         0x9254, 0xffffffff, 0x00010006,
 601         0x9258, 0xffffffff, 0x00090008,
 602         0x925c, 0xffffffff, 0x00070000,
 603         0x9260, 0xffffffff, 0x00030002,
 604         0x9264, 0xffffffff, 0x00050004,
 605         0x9270, 0xffffffff, 0x00010006,
 606         0x9274, 0xffffffff, 0x00090008,
 607         0x9278, 0xffffffff, 0x00070000,
 608         0x927c, 0xffffffff, 0x00030002,
 609         0x9280, 0xffffffff, 0x00050004,
 610         0x928c, 0xffffffff, 0x00010006,
 611         0x9290, 0xffffffff, 0x00090008,
 612         0x9294, 0xffffffff, 0x00000000,
 613         0x929c, 0xffffffff, 0x00000001,
 614         0x802c, 0xffffffff, 0xc0000000,
 615         0x977c, 0xffffffff, 0x00000100,
 616         0x3f80, 0xffffffff, 0x00000100,
 617         0xa210, 0xffffffff, 0x00000100,
 618         0xa214, 0xffffffff, 0x00000100,
 619         0x4d8, 0xffffffff, 0x00000100,
 620         0x9784, 0xffffffff, 0x00000100,
 621         0x9698, 0xffffffff, 0x00000100,
 622         0x4d4, 0xffffffff, 0x00000200,
 623         0x30cc, 0xffffffff, 0x00000100,
 624         0x802c, 0xffffffff, 0xc0000000
 625 };
 626
 627 static const u32 supersumo_golden_registers[] =
 628 {
 629         0x5eb4, 0xffffffff, 0x00000002,
 630         0x5cc, 0xffffffff, 0x00000001,
 631         0x7030, 0xffffffff, 0x00000011,
 632         0x7c30, 0xffffffff, 0x00000011,
 633         0x6104, 0x01000300, 0x00000000,
 634         0x5bc0, 0x00300000, 0x00000000,
 635         0x8c04, 0xffffffff, 0x40600060,
 636         0x8c08, 0xffffffff, 0x001c001c,
 637         0x8c20, 0xffffffff, 0x00800080,
 638         0x8c24, 0xffffffff, 0x00800080,
 639         0x8c18, 0xffffffff, 0x20202078,
 640         0x8c1c, 0xffffffff, 0x00001010,
 641         0x918c, 0xffffffff, 0x00010006,
 642         0x91a8, 0xffffffff, 0x00010006,
 643         0x91c4, 0xffffffff, 0x00010006,
 644         0x91e0, 0xffffffff, 0x00010006,
 645         0x9200, 0xffffffff, 0x00010006,
 646         0x9150, 0xffffffff, 0x6e944040,
 647         0x917c, 0xffffffff, 0x00030002,
 648         0x9180, 0xffffffff, 0x00050004,
 649         0x9198, 0xffffffff, 0x00030002,
 650         0x919c, 0xffffffff, 0x00050004,
 651         0x91b4, 0xffffffff, 0x00030002,
 652         0x91b8, 0xffffffff, 0x00050004,
 653         0x91d0, 0xffffffff, 0x00030002,
 654         0x91d4, 0xffffffff, 0x00050004,
 655         0x91f0, 0xffffffff, 0x00030002,
 656         0x91f4, 0xffffffff, 0x00050004,
 657         0x915c, 0xffffffff, 0x00010000,
 658         0x9160, 0xffffffff, 0x00030002,
 659         0x3f90, 0xffff0000, 0xff000000,
 660         0x9178, 0xffffffff, 0x00070000,
 661         0x9194, 0xffffffff, 0x00070000,
 662         0x91b0, 0xffffffff, 0x00070000,
 663         0x91cc, 0xffffffff, 0x00070000,
 664         0x91ec, 0xffffffff, 0x00070000,
 665         0x9148, 0xffff0000, 0xff000000,
 666         0x9190, 0xffffffff, 0x00090008,
 667         0x91ac, 0xffffffff, 0x00090008,
 668         0x91c8, 0xffffffff, 0x00090008,
 669         0x91e4, 0xffffffff, 0x00090008,
 670         0x9204, 0xffffffff, 0x00090008,
 671         0x3f94, 0xffff0000, 0xff000000,
 672         0x914c, 0xffff0000, 0xff000000,
 673         0x929c, 0xffffffff, 0x00000001,
 674         0x8a18, 0xffffffff, 0x00000100,
 675         0x8b28, 0xffffffff, 0x00000100,
 676         0x9144, 0xffffffff, 0x00000100,
 677         0x5644, 0xffffffff, 0x00000100,
 678         0x9b7c, 0xffffffff, 0x00000000,
 679         0x8030, 0xffffffff, 0x0000100a,
 680         0x8a14, 0xffffffff, 0x00000007,
 681         0x8b24, 0xffffffff, 0x00ff0fff,
 682         0x8b10, 0xffffffff, 0x00000000,
 683         0x28a4c, 0x06000000, 0x06000000,
 684         0x4d8, 0xffffffff, 0x00000100,
 685         0x913c, 0xffff000f, 0x0100000a,
 686         0x960c, 0xffffffff, 0x54763210,
 687         0x88c4, 0xffffffff, 0x000000c2,
 688         0x88d4, 0xffffffff, 0x00000010,
 689         0x8974, 0xffffffff, 0x00000000,
 690         0xc78, 0x00000080, 0x00000080,
 691         0x5e78, 0xffffffff, 0x001000f0,
 692         0xd02c, 0xffffffff, 0x08421000,
 693         0xa008, 0xffffffff, 0x00010000,
 694         0x8d00, 0xffffffff, 0x100e4848,
 695         0x8d04, 0xffffffff, 0x00164745,
 696         0x8c00, 0xffffffff, 0xe4000003,
 697         0x8cf0, 0x1fffffff, 0x08e00620,
 698         0x28350, 0xffffffff, 0x00000000,
 699         0x9508, 0xffffffff, 0x00000002
 700 };
 701
 702 static const u32 sumo_golden_registers[] =
 703 {
 704         0x900c, 0x00ffffff, 0x0017071f,
 705         0x8c18, 0xffffffff, 0x10101060,
 706         0x8c1c, 0xffffffff, 0x00001010,
 707         0x8c30, 0x0000000f, 0x00000005,
 708         0x9688, 0x0000000f, 0x00000007
 709 };
 710
 711 static const u32 wrestler_golden_registers[] =
 712 {
 713         0x5eb4, 0xffffffff, 0x00000002,
 714         0x5cc, 0xffffffff, 0x00000001,
 715         0x7030, 0xffffffff, 0x00000011,
 716         0x7c30, 0xffffffff, 0x00000011,
 717         0x6104, 0x01000300, 0x00000000,
 718         0x5bc0, 0x00300000, 0x00000000,
 719         0x918c, 0xffffffff, 0x00010006,
 720         0x91a8, 0xffffffff, 0x00010006,
 721         0x9150, 0xffffffff, 0x6e944040,
 722         0x917c, 0xffffffff, 0x00030002,
 723         0x9198, 0xffffffff, 0x00030002,
 724         0x915c, 0xffffffff, 0x00010000,
 725         0x3f90, 0xffff0000, 0xff000000,
 726         0x9178, 0xffffffff, 0x00070000,
 727         0x9194, 0xffffffff, 0x00070000,
 728         0x9148, 0xffff0000, 0xff000000,
 729         0x9190, 0xffffffff, 0x00090008,
 730         0x91ac, 0xffffffff, 0x00090008,
 731         0x3f94, 0xffff0000, 0xff000000,
 732         0x914c, 0xffff0000, 0xff000000,
 733         0x929c, 0xffffffff, 0x00000001,
 734         0x8a18, 0xffffffff, 0x00000100,
 735         0x8b28, 0xffffffff, 0x00000100,
 736         0x9144, 0xffffffff, 0x00000100,
 737         0x9b7c, 0xffffffff, 0x00000000,
 738         0x8030, 0xffffffff, 0x0000100a,
 739         0x8a14, 0xffffffff, 0x00000001,
 740         0x8b24, 0xffffffff, 0x00ff0fff,
 741         0x8b10, 0xffffffff, 0x00000000,
 742         0x28a4c, 0x06000000, 0x06000000,
 743         0x4d8, 0xffffffff, 0x00000100,
 744         0x913c, 0xffff000f, 0x0100000a,
 745         0x960c, 0xffffffff, 0x54763210,
 746         0x88c4, 0xffffffff, 0x000000c2,
 747         0x88d4, 0xffffffff, 0x00000010,
 748         0x8974, 0xffffffff, 0x00000000,
 749         0xc78, 0x00000080, 0x00000080,
 750         0x5e78, 0xffffffff, 0x001000f0,
 751         0xd02c, 0xffffffff, 0x08421000,
 752         0xa008, 0xffffffff, 0x00010000,
 753         0x8d00, 0xffffffff, 0x100e4848,
 754         0x8d04, 0xffffffff, 0x00164745,
 755         0x8c00, 0xffffffff, 0xe4000003,
 756         0x8cf0, 0x1fffffff, 0x08e00410,
 757         0x28350, 0xffffffff, 0x00000000,
 758         0x9508, 0xffffffff, 0x00000002,
 759         0x900c, 0xffffffff, 0x0017071f,
 760         0x8c18, 0xffffffff, 0x10101060,
 761         0x8c1c, 0xffffffff, 0x00001010
 762 };
 763
 764 static const u32 barts_golden_registers[] =
 765 {
 766         0x5eb4, 0xffffffff, 0x00000002,
 767         0x5e78, 0x8f311ff1, 0x001000f0,
 768         0x3f90, 0xffff0000, 0xff000000,
 769         0x9148, 0xffff0000, 0xff000000,
 770         0x3f94, 0xffff0000, 0xff000000,
 771         0x914c, 0xffff0000, 0xff000000,
 772         0xc78, 0x00000080, 0x00000080,
 773         0xbd4, 0x70073777, 0x00010001,
 774         0xd02c, 0xbfffff1f, 0x08421000,
 775         0xd0b8, 0x03773777, 0x02011003,
 776         0x5bc0, 0x00200000, 0x50100000,
 777         0x98f8, 0x33773777, 0x02011003,
 778         0x98fc, 0xffffffff, 0x76543210,
 779         0x7030, 0x31000311, 0x00000011,
 780         0x2f48, 0x00000007, 0x02011003,
 781         0x6b28, 0x00000010, 0x00000012,
 782         0x7728, 0x00000010, 0x00000012,
 783         0x10328, 0x00000010, 0x00000012,
 784         0x10f28, 0x00000010, 0x00000012,
 785         0x11b28, 0x00000010, 0x00000012,
 786         0x12728, 0x00000010, 0x00000012,
 787         0x240c, 0x000007ff, 0x00000380,
 788         0x8a14, 0xf000001f, 0x00000007,
 789         0x8b24, 0x3fff3fff, 0x00ff0fff,
 790         0x8b10, 0x0000ff0f, 0x00000000,
 791         0x28a4c, 0x07ffffff, 0x06000000,
 792         0x10c, 0x00000001, 0x00010003,
 793         0xa02c, 0xffffffff, 0x0000009b,
 794         0x913c, 0x0000000f, 0x0100000a,
 795         0x8d00, 0xffff7f7f, 0x100e4848,
 796         0x8d04, 0x00ffffff, 0x00164745,
 797         0x8c00, 0xfffc0003, 0xe4000003,
 798         0x8c04, 0xf8ff00ff, 0x40600060,
 799         0x8c08, 0x00ff00ff, 0x001c001c,
 800         0x8cf0, 0x1fff1fff, 0x08e00620,
 801         0x8c20, 0x0fff0fff, 0x00800080,
 802         0x8c24, 0x0fff0fff, 0x00800080,
 803         0x8c18, 0xffffffff, 0x20202078,
 804         0x8c1c, 0x0000ffff, 0x00001010,
 805         0x28350, 0x00000f01, 0x00000000,
 806         0x9508, 0x3700001f, 0x00000002,
 807         0x960c, 0xffffffff, 0x54763210,
 808         0x88c4, 0x001f3ae3, 0x000000c2,
 809         0x88d4, 0x0000001f, 0x00000010,
 810         0x8974, 0xffffffff, 0x00000000
 811 };
 812
 813 static const u32 turks_golden_registers[] =
 814 {
 815         0x5eb4, 0xffffffff, 0x00000002,
 816         0x5e78, 0x8f311ff1, 0x001000f0,
 817         0x8c8, 0x00003000, 0x00001070,
 818         0x8cc, 0x000fffff, 0x00040035,
 819         0x3f90, 0xffff0000, 0xfff00000,
 820         0x9148, 0xffff0000, 0xfff00000,
 821         0x3f94, 0xffff0000, 0xfff00000,
 822         0x914c, 0xffff0000, 0xfff00000,
 823         0xc78, 0x00000080, 0x00000080,
 824         0xbd4, 0x00073007, 0x00010002,
 825         0xd02c, 0xbfffff1f, 0x08421000,
 826         0xd0b8, 0x03773777, 0x02010002,
 827         0x5bc0, 0x00200000, 0x50100000,
 828         0x98f8, 0x33773777, 0x00010002,
 829         0x98fc, 0xffffffff, 0x33221100,
 830         0x7030, 0x31000311, 0x00000011,
 831         0x2f48, 0x33773777, 0x00010002,
 832         0x6b28, 0x00000010, 0x00000012,
 833         0x7728, 0x00000010, 0x00000012,
 834         0x10328, 0x00000010, 0x00000012,
 835         0x10f28, 0x00000010, 0x00000012,
 836         0x11b28, 0x00000010, 0x00000012,
 837         0x12728, 0x00000010, 0x00000012,
 838         0x240c, 0x000007ff, 0x00000380,
 839         0x8a14, 0xf000001f, 0x00000007,
 840         0x8b24, 0x3fff3fff, 0x00ff0fff,
 841         0x8b10, 0x0000ff0f, 0x00000000,
 842         0x28a4c, 0x07ffffff, 0x06000000,
 843         0x10c, 0x00000001, 0x00010003,
 844         0xa02c, 0xffffffff, 0x0000009b,
 845         0x913c, 0x0000000f, 0x0100000a,
 846         0x8d00, 0xffff7f7f, 0x100e4848,
 847         0x8d04, 0x00ffffff, 0x00164745,
 848         0x8c00, 0xfffc0003, 0xe4000003,
 849         0x8c04, 0xf8ff00ff, 0x40600060,
 850         0x8c08, 0x00ff00ff, 0x001c001c,
 851         0x8cf0, 0x1fff1fff, 0x08e00410,
 852         0x8c20, 0x0fff0fff, 0x00800080,
 853         0x8c24, 0x0fff0fff, 0x00800080,
 854         0x8c18, 0xffffffff, 0x20202078,
 855         0x8c1c, 0x0000ffff, 0x00001010,
 856         0x28350, 0x00000f01, 0x00000000,
 857         0x9508, 0x3700001f, 0x00000002,
 858         0x960c, 0xffffffff, 0x54763210,
 859         0x88c4, 0x001f3ae3, 0x000000c2,
 860         0x88d4, 0x0000001f, 0x00000010,
 861         0x8974, 0xffffffff, 0x00000000
 862 };
 863
 864 static const u32 caicos_golden_registers[] =
 865 {
 866         0x5eb4, 0xffffffff, 0x00000002,
 867         0x5e78, 0x8f311ff1, 0x001000f0,
 868         0x8c8, 0x00003420, 0x00001450,
 869         0x8cc, 0x000fffff, 0x00040035,
 870         0x3f90, 0xffff0000, 0xfffc0000,
 871         0x9148, 0xffff0000, 0xfffc0000,
 872         0x3f94, 0xffff0000, 0xfffc0000,
 873         0x914c, 0xffff0000, 0xfffc0000,
 874         0xc78, 0x00000080, 0x00000080,
 875         0xbd4, 0x00073007, 0x00010001,
 876         0xd02c, 0xbfffff1f, 0x08421000,
 877         0xd0b8, 0x03773777, 0x02010001,
 878         0x5bc0, 0x00200000, 0x50100000,
 879         0x98f8, 0x33773777, 0x02010001,
 880         0x98fc, 0xffffffff, 0x33221100,
 881         0x7030, 0x31000311, 0x00000011,
 882         0x2f48, 0x33773777, 0x02010001,
 883         0x6b28, 0x00000010, 0x00000012,
 884         0x7728, 0x00000010, 0x00000012,
 885         0x10328, 0x00000010, 0x00000012,
 886         0x10f28, 0x00000010, 0x00000012,
 887         0x11b28, 0x00000010, 0x00000012,
 888         0x12728, 0x00000010, 0x00000012,
 889         0x240c, 0x000007ff, 0x00000380,
 890         0x8a14, 0xf000001f, 0x00000001,
 891         0x8b24, 0x3fff3fff, 0x00ff0fff,
 892         0x8b10, 0x0000ff0f, 0x00000000,
 893         0x28a4c, 0x07ffffff, 0x06000000,
 894         0x10c, 0x00000001, 0x00010003,
 895         0xa02c, 0xffffffff, 0x0000009b,
 896         0x913c, 0x0000000f, 0x0100000a,
 897         0x8d00, 0xffff7f7f, 0x100e4848,
 898         0x8d04, 0x00ffffff, 0x00164745,
 899         0x8c00, 0xfffc0003, 0xe4000003,
 900         0x8c04, 0xf8ff00ff, 0x40600060,
 901         0x8c08, 0x00ff00ff, 0x001c001c,
 902         0x8cf0, 0x1fff1fff, 0x08e00410,
 903         0x8c20, 0x0fff0fff, 0x00800080,
 904         0x8c24, 0x0fff0fff, 0x00800080,
 905         0x8c18, 0xffffffff, 0x20202078,
 906         0x8c1c, 0x0000ffff, 0x00001010,
 907         0x28350, 0x00000f01, 0x00000000,
 908         0x9508, 0x3700001f, 0x00000002,
 909         0x960c, 0xffffffff, 0x54763210,
 910         0x88c4, 0x001f3ae3, 0x000000c2,
 911         0x88d4, 0x0000001f, 0x00000010,
 912         0x8974, 0xffffffff, 0x00000000
 913 };
 914
 915 static void evergreen_init_golden_registers(struct radeon_device *rdev)
 916 {
 917         switch (rdev->family) {
 918         case CHIP_CYPRESS:
 919         case CHIP_HEMLOCK:
 920                 radeon_program_register_sequence(rdev,
 921                                                  evergreen_golden_registers,
 922                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
 923                 radeon_program_register_sequence(rdev,
 924                                                  evergreen_golden_registers2,
 925                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
 926                 radeon_program_register_sequence(rdev,
 927                                                  cypress_mgcg_init,
 928                                                  (const u32)ARRAY_SIZE(cypress_mgcg_init));
 929                 break;
 930         case CHIP_JUNIPER:
 931                 radeon_program_register_sequence(rdev,
 932                                                  evergreen_golden_registers,
 933                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
 934                 radeon_program_register_sequence(rdev,
 935                                                  evergreen_golden_registers2,
 936                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
 937                 radeon_program_register_sequence(rdev,
 938                                                  juniper_mgcg_init,
 939                                                  (const u32)ARRAY_SIZE(juniper_mgcg_init));
 940                 break;
 941         case CHIP_REDWOOD:
 942                 radeon_program_register_sequence(rdev,
 943                                                  evergreen_golden_registers,
 944                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
 945                 radeon_program_register_sequence(rdev,
 946                                                  evergreen_golden_registers2,
 947                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
 948                 radeon_program_register_sequence(rdev,
 949                                                  redwood_mgcg_init,
 950                                                  (const u32)ARRAY_SIZE(redwood_mgcg_init));
 951                 break;
 952         case CHIP_CEDAR:
 953                 radeon_program_register_sequence(rdev,
 954                                                  cedar_golden_registers,
 955                                                  (const u32)ARRAY_SIZE(cedar_golden_registers));
 956                 radeon_program_register_sequence(rdev,
 957                                                  evergreen_golden_registers2,
 958                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
 959                 radeon_program_register_sequence(rdev,
 960                                                  cedar_mgcg_init,
 961                                                  (const u32)ARRAY_SIZE(cedar_mgcg_init));
 962                 break;
 963         case CHIP_PALM:
 964                 radeon_program_register_sequence(rdev,
 965                                                  wrestler_golden_registers,
 966                                                  (const u32)ARRAY_SIZE(wrestler_golden_registers));
 967                 break;
 968         case CHIP_SUMO:
 969                 radeon_program_register_sequence(rdev,
 970                                                  supersumo_golden_registers,
 971                                                  (const u32)ARRAY_SIZE(supersumo_golden_registers));
 972                 break;
 973         case CHIP_SUMO2:
 974                 radeon_program_register_sequence(rdev,
 975                                                  supersumo_golden_registers,
 976                                                  (const u32)ARRAY_SIZE(supersumo_golden_registers));
 977                 radeon_program_register_sequence(rdev,
 978                                                  sumo_golden_registers,
 979                                                  (const u32)ARRAY_SIZE(sumo_golden_registers));
 980                 break;
 981         case CHIP_BARTS:
 982                 radeon_program_register_sequence(rdev,
 983                                                  barts_golden_registers,
 984                                                  (const u32)ARRAY_SIZE(barts_golden_registers));
 985                 break;
 986         case CHIP_TURKS:
 987                 radeon_program_register_sequence(rdev,
 988                                                  turks_golden_registers,
 989                                                  (const u32)ARRAY_SIZE(turks_golden_registers));
 990                 break;
 991         case CHIP_CAICOS:
 992                 radeon_program_register_sequence(rdev,
 993                                                  caicos_golden_registers,
 994                                                  (const u32)ARRAY_SIZE(caicos_golden_registers));
 995                 break;
 996         default:
 997                 break;
 998         }
 999 }
1000
1001 void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1002                              unsigned *bankh, unsigned *mtaspect,
1003                              unsigned *tile_split)
1004 {
1005         *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1006         *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1007         *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1008         *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1009         switch (*bankw) {
1010         default:
1011         case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1012         case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1013         case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1014         case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1015         }
1016         switch (*bankh) {
1017         default:
1018         case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1019         case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1020         case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1021         case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1022         }
1023         switch (*mtaspect) {
1024         default:
1025         case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1026         case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1027         case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1028         case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1029         }
1030 }
1031
1032 static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1033                               u32 cntl_reg, u32 status_reg)
1034 {
1035         int r, i;
1036         struct atom_clock_dividers dividers;
1037
1038         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1039                                            clock, false, &dividers);
1040         if (r)
1041                 return r;
1042
1043         WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1044
1045         for (i = 0; i < 100; i++) {
1046                 if (RREG32(status_reg) & DCLK_STATUS)
1047                         break;
1048                 mdelay(10);
1049         }
1050         if (i == 100)
1051                 return -ETIMEDOUT;
1052
1053         return 0;
1054 }
1055
1056 int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1057 {
1058         int r = 0;
1059         u32 cg_scratch = RREG32(CG_SCRATCH1);
1060
1061         r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1062         if (r)
1063                 goto done;
1064         cg_scratch &= 0xffff0000;
1065         cg_scratch |= vclk / 100; /* Mhz */
1066
1067         r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1068         if (r)
1069                 goto done;
1070         cg_scratch &= 0x0000ffff;
1071         cg_scratch |= (dclk / 100) << 16; /* Mhz */
1072
1073 done:
1074         WREG32(CG_SCRATCH1, cg_scratch);
1075
1076         return r;
1077 }
1078
1079 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1080 {
1081         /* start off with something large */
1082         unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1083         int r;
1084
1085         /* bypass vclk and dclk with bclk */
1086         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1087                 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1088                 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1089
1090         /* put PLL in bypass mode */
1091         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1092
1093         if (!vclk || !dclk) {
1094                 /* keep the Bypass mode, put PLL to sleep */
1095                 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1096                 return 0;
1097         }
1098
1099         r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1100                                           16384, 0x03FFFFFF, 0, 128, 5,
1101                                           &fb_div, &vclk_div, &dclk_div);
1102         if (r)
1103                 return r;
1104
1105         /* set VCO_MODE to 1 */
1106         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1107
1108         /* toggle UPLL_SLEEP to 1 then back to 0 */
1109         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1110         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1111
1112         /* deassert UPLL_RESET */
1113         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1114
1115         mdelay(1);
1116
1117         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1118         if (r)
1119                 return r;
1120
1121         /* assert UPLL_RESET again */
1122         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1123
1124         /* disable spread spectrum. */
1125         WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1126
1127         /* set feedback divider */
1128         WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1129
1130         /* set ref divider to 0 */
1131         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1132
1133         if (fb_div < 307200)
1134                 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1135         else
1136                 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1137
1138         /* set PDIV_A and PDIV_B */
1139         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1140                 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1141                 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1142
1143         /* give the PLL some time to settle */
1144         mdelay(15);
1145
1146         /* deassert PLL_RESET */
1147         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1148
1149         mdelay(15);
1150
1151         /* switch from bypass mode to normal mode */
1152         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1153
1154         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1155         if (r)
1156                 return r;
1157
1158         /* switch VCLK and DCLK selection */
1159         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1160                 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1161                 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1162
1163         mdelay(100);
1164
1165         return 0;
1166 }
1167
1168 void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1169 {
1170         u16 ctl, v;
1171         int err;
1172
1173         err = pcie_capability_read_word(rdev->pdev, PCI_EXP_DEVCTL, &ctl);
1174         if (err)
1175                 return;
1176
1177         v = (ctl & PCI_EXP_DEVCTL_READRQ) >> 12;
1178
1179         /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1180          * to avoid hangs or perfomance issues
1181          */
1182         if ((v == 0) || (v == 6) || (v == 7)) {
1183                 ctl &= ~PCI_EXP_DEVCTL_READRQ;
1184                 ctl |= (2 << 12);
1185                 pcie_capability_write_word(rdev->pdev, PCI_EXP_DEVCTL, ctl);
1186         }
1187 }
1188
1189 static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1190 {
1191         if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1192                 return true;
1193         else
1194                 return false;
1195 }
1196
1197 static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1198 {
1199         u32 pos1, pos2;
1200
1201         pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1202         pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1203
1204         if (pos1 != pos2)
1205                 return true;
1206         else
1207                 return false;
1208 }
1209
1210 /**
1211  * dce4_wait_for_vblank - vblank wait asic callback.
1212  *
1213  * @rdev: radeon_device pointer
1214  * @crtc: crtc to wait for vblank on
1215  *
1216  * Wait for vblank on the requested crtc (evergreen+).
1217  */
1218 void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1219 {
1220         unsigned i = 0;
1221
1222         if (crtc >= rdev->num_crtc)
1223                 return;
1224
1225         if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1226                 return;
1227
1228         /* depending on when we hit vblank, we may be close to active; if so,
1229          * wait for another frame.
1230          */
1231         while (dce4_is_in_vblank(rdev, crtc)) {
1232                 if (i++ % 100 == 0) {
1233                         if (!dce4_is_counter_moving(rdev, crtc))
1234                                 break;
1235                 }
1236         }
1237
1238         while (!dce4_is_in_vblank(rdev, crtc)) {
1239                 if (i++ % 100 == 0) {
1240                         if (!dce4_is_counter_moving(rdev, crtc))
1241                                 break;
1242                 }
1243         }
1244 }
1245
1246 /**
1247  * radeon_irq_kms_pflip_irq_get - pre-pageflip callback.
1248  *
1249  * @rdev: radeon_device pointer
1250  * @crtc: crtc to prepare for pageflip on
1251  *
1252  * Pre-pageflip callback (evergreen+).
1253  * Enables the pageflip irq (vblank irq).
1254  */
1255 void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc)
1256 {
1257         /* enable the pflip int */
1258         radeon_irq_kms_pflip_irq_get(rdev, crtc);
1259 }
1260
1261 /**
1262  * evergreen_post_page_flip - pos-pageflip callback.
1263  *
1264  * @rdev: radeon_device pointer
1265  * @crtc: crtc to cleanup pageflip on
1266  *
1267  * Post-pageflip callback (evergreen+).
1268  * Disables the pageflip irq (vblank irq).
1269  */
1270 void evergreen_post_page_flip(struct radeon_device *rdev, int crtc)
1271 {
1272         /* disable the pflip int */
1273         radeon_irq_kms_pflip_irq_put(rdev, crtc);
1274 }
1275
1276 /**
1277  * evergreen_page_flip - pageflip callback.
1278  *
1279  * @rdev: radeon_device pointer
1280  * @crtc_id: crtc to cleanup pageflip on
1281  * @crtc_base: new address of the crtc (GPU MC address)
1282  *
1283  * Does the actual pageflip (evergreen+).
1284  * During vblank we take the crtc lock and wait for the update_pending
1285  * bit to go high, when it does, we release the lock, and allow the
1286  * double buffered update to take place.
1287  * Returns the current update pending status.
1288  */
1289 u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
1290 {
1291         struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1292         u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
1293         int i;
1294
1295         /* Lock the graphics update lock */
1296         tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
1297         WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1298
1299         /* update the scanout addresses */
1300         WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1301                upper_32_bits(crtc_base));
1302         WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1303                (u32)crtc_base);
1304
1305         WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1306                upper_32_bits(crtc_base));
1307         WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1308                (u32)crtc_base);
1309
1310         /* Wait for update_pending to go high. */
1311         for (i = 0; i < rdev->usec_timeout; i++) {
1312                 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
1313                         break;
1314                 udelay(1);
1315         }
1316         DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
1317
1318         /* Unlock the lock, so double-buffering can take place inside vblank */
1319         tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
1320         WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1321
1322         /* Return current update_pending status: */
1323         return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING;
1324 }
1325
1326 /* get temperature in millidegrees */
1327 int evergreen_get_temp(struct radeon_device *rdev)
1328 {
1329         u32 temp, toffset;
1330         int actual_temp = 0;
1331
1332         if (rdev->family == CHIP_JUNIPER) {
1333                 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1334                         TOFFSET_SHIFT;
1335                 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1336                         TS0_ADC_DOUT_SHIFT;
1337
1338                 if (toffset & 0x100)
1339                         actual_temp = temp / 2 - (0x200 - toffset);
1340                 else
1341                         actual_temp = temp / 2 + toffset;
1342
1343                 actual_temp = actual_temp * 1000;
1344
1345         } else {
1346                 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1347                         ASIC_T_SHIFT;
1348
1349                 if (temp & 0x400)
1350                         actual_temp = -256;
1351                 else if (temp & 0x200)
1352                         actual_temp = 255;
1353                 else if (temp & 0x100) {
1354                         actual_temp = temp & 0x1ff;
1355                         actual_temp |= ~0x1ff;
1356                 } else
1357                         actual_temp = temp & 0xff;
1358
1359                 actual_temp = (actual_temp * 1000) / 2;
1360         }
1361
1362         return actual_temp;
1363 }
1364
1365 int sumo_get_temp(struct radeon_device *rdev)
1366 {
1367         u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1368         int actual_temp = temp - 49;
1369
1370         return actual_temp * 1000;
1371 }
1372
1373 /**
1374  * sumo_pm_init_profile - Initialize power profiles callback.
1375  *
1376  * @rdev: radeon_device pointer
1377  *
1378  * Initialize the power states used in profile mode
1379  * (sumo, trinity, SI).
1380  * Used for profile mode only.
1381  */
1382 void sumo_pm_init_profile(struct radeon_device *rdev)
1383 {
1384         int idx;
1385
1386         /* default */
1387         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1388         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1389         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1390         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1391
1392         /* low,mid sh/mh */
1393         if (rdev->flags & RADEON_IS_MOBILITY)
1394                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1395         else
1396                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1397
1398         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1399         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1400         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1401         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1402
1403         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1404         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1405         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1406         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1407
1408         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1409         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1410         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1411         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1412
1413         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1414         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1415         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1416         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1417
1418         /* high sh/mh */
1419         idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1420         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1421         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1422         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1423         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1424                 rdev->pm.power_state[idx].num_clock_modes - 1;
1425
1426         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1427         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1428         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1429         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1430                 rdev->pm.power_state[idx].num_clock_modes - 1;
1431 }
1432
1433 /**
1434  * btc_pm_init_profile - Initialize power profiles callback.
1435  *
1436  * @rdev: radeon_device pointer
1437  *
1438  * Initialize the power states used in profile mode
1439  * (BTC, cayman).
1440  * Used for profile mode only.
1441  */
1442 void btc_pm_init_profile(struct radeon_device *rdev)
1443 {
1444         int idx;
1445
1446         /* default */
1447         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1448         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1449         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1450         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1451         /* starting with BTC, there is one state that is used for both
1452          * MH and SH.  Difference is that we always use the high clock index for
1453          * mclk.
1454          */
1455         if (rdev->flags & RADEON_IS_MOBILITY)
1456                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1457         else
1458                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1459         /* low sh */
1460         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1461         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1462         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1463         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1464         /* mid sh */
1465         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1466         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1467         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1468         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1469         /* high sh */
1470         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1471         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1472         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1473         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1474         /* low mh */
1475         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1476         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1477         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1478         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1479         /* mid mh */
1480         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1481         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1482         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1483         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1484         /* high mh */
1485         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1486         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1487         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1488         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1489 }
1490
1491 /**
1492  * evergreen_pm_misc - set additional pm hw parameters callback.
1493  *
1494  * @rdev: radeon_device pointer
1495  *
1496  * Set non-clock parameters associated with a power state
1497  * (voltage, etc.) (evergreen+).
1498  */
1499 void evergreen_pm_misc(struct radeon_device *rdev)
1500 {
1501         int req_ps_idx = rdev->pm.requested_power_state_index;
1502         int req_cm_idx = rdev->pm.requested_clock_mode_index;
1503         struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1504         struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1505
1506         if (voltage->type == VOLTAGE_SW) {
1507                 /* 0xff01 is a flag rather then an actual voltage */
1508                 if (voltage->voltage == 0xff01)
1509                         return;
1510                 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1511                         radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1512                         rdev->pm.current_vddc = voltage->voltage;
1513                         DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1514                 }
1515
1516                 /* starting with BTC, there is one state that is used for both
1517                  * MH and SH.  Difference is that we always use the high clock index for
1518                  * mclk and vddci.
1519                  */
1520                 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1521                     (rdev->family >= CHIP_BARTS) &&
1522                     rdev->pm.active_crtc_count &&
1523                     ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1524                      (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1525                         voltage = &rdev->pm.power_state[req_ps_idx].
1526                                 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1527
1528                 /* 0xff01 is a flag rather then an actual voltage */
1529                 if (voltage->vddci == 0xff01)
1530                         return;
1531                 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1532                         radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1533                         rdev->pm.current_vddci = voltage->vddci;
1534                         DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1535                 }
1536         }
1537 }
1538
1539 /**
1540  * evergreen_pm_prepare - pre-power state change callback.
1541  *
1542  * @rdev: radeon_device pointer
1543  *
1544  * Prepare for a power state change (evergreen+).
1545  */
1546 void evergreen_pm_prepare(struct radeon_device *rdev)
1547 {
1548         struct drm_device *ddev = rdev->ddev;
1549         struct drm_crtc *crtc;
1550         struct radeon_crtc *radeon_crtc;
1551         u32 tmp;
1552
1553         /* disable any active CRTCs */
1554         list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1555                 radeon_crtc = to_radeon_crtc(crtc);
1556                 if (radeon_crtc->enabled) {
1557                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1558                         tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1559                         WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1560                 }
1561         }
1562 }
1563
1564 /**
1565  * evergreen_pm_finish - post-power state change callback.
1566  *
1567  * @rdev: radeon_device pointer
1568  *
1569  * Clean up after a power state change (evergreen+).
1570  */
1571 void evergreen_pm_finish(struct radeon_device *rdev)
1572 {
1573         struct drm_device *ddev = rdev->ddev;
1574         struct drm_crtc *crtc;
1575         struct radeon_crtc *radeon_crtc;
1576         u32 tmp;
1577
1578         /* enable any active CRTCs */
1579         list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1580                 radeon_crtc = to_radeon_crtc(crtc);
1581                 if (radeon_crtc->enabled) {
1582                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1583                         tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1584                         WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1585                 }
1586         }
1587 }
1588
1589 /**
1590  * evergreen_hpd_sense - hpd sense callback.
1591  *
1592  * @rdev: radeon_device pointer
1593  * @hpd: hpd (hotplug detect) pin
1594  *
1595  * Checks if a digital monitor is connected (evergreen+).
1596  * Returns true if connected, false if not connected.
1597  */
1598 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1599 {
1600         bool connected = false;
1601
1602         switch (hpd) {
1603         case RADEON_HPD_1:
1604                 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1605                         connected = true;
1606                 break;
1607         case RADEON_HPD_2:
1608                 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1609                         connected = true;
1610                 break;
1611         case RADEON_HPD_3:
1612                 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1613                         connected = true;
1614                 break;
1615         case RADEON_HPD_4:
1616                 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1617                         connected = true;
1618                 break;
1619         case RADEON_HPD_5:
1620                 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1621                         connected = true;
1622                 break;
1623         case RADEON_HPD_6:
1624                 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1625                         connected = true;
1626                         break;
1627         default:
1628                 break;
1629         }
1630
1631         return connected;
1632 }
1633
1634 /**
1635  * evergreen_hpd_set_polarity - hpd set polarity callback.
1636  *
1637  * @rdev: radeon_device pointer
1638  * @hpd: hpd (hotplug detect) pin
1639  *
1640  * Set the polarity of the hpd pin (evergreen+).
1641  */
1642 void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1643                                 enum radeon_hpd_id hpd)
1644 {
1645         u32 tmp;
1646         bool connected = evergreen_hpd_sense(rdev, hpd);
1647
1648         switch (hpd) {
1649         case RADEON_HPD_1:
1650                 tmp = RREG32(DC_HPD1_INT_CONTROL);
1651                 if (connected)
1652                         tmp &= ~DC_HPDx_INT_POLARITY;
1653                 else
1654                         tmp |= DC_HPDx_INT_POLARITY;
1655                 WREG32(DC_HPD1_INT_CONTROL, tmp);
1656                 break;
1657         case RADEON_HPD_2:
1658                 tmp = RREG32(DC_HPD2_INT_CONTROL);
1659                 if (connected)
1660                         tmp &= ~DC_HPDx_INT_POLARITY;
1661                 else
1662                         tmp |= DC_HPDx_INT_POLARITY;
1663                 WREG32(DC_HPD2_INT_CONTROL, tmp);
1664                 break;
1665         case RADEON_HPD_3:
1666                 tmp = RREG32(DC_HPD3_INT_CONTROL);
1667                 if (connected)
1668                         tmp &= ~DC_HPDx_INT_POLARITY;
1669                 else
1670                         tmp |= DC_HPDx_INT_POLARITY;
1671                 WREG32(DC_HPD3_INT_CONTROL, tmp);
1672                 break;
1673         case RADEON_HPD_4:
1674                 tmp = RREG32(DC_HPD4_INT_CONTROL);
1675                 if (connected)
1676                         tmp &= ~DC_HPDx_INT_POLARITY;
1677                 else
1678                         tmp |= DC_HPDx_INT_POLARITY;
1679                 WREG32(DC_HPD4_INT_CONTROL, tmp);
1680                 break;
1681         case RADEON_HPD_5:
1682                 tmp = RREG32(DC_HPD5_INT_CONTROL);
1683                 if (connected)
1684                         tmp &= ~DC_HPDx_INT_POLARITY;
1685                 else
1686                         tmp |= DC_HPDx_INT_POLARITY;
1687                 WREG32(DC_HPD5_INT_CONTROL, tmp);
1688                         break;
1689         case RADEON_HPD_6:
1690                 tmp = RREG32(DC_HPD6_INT_CONTROL);
1691                 if (connected)
1692                         tmp &= ~DC_HPDx_INT_POLARITY;
1693                 else
1694                         tmp |= DC_HPDx_INT_POLARITY;
1695                 WREG32(DC_HPD6_INT_CONTROL, tmp);
1696                 break;
1697         default:
1698                 break;
1699         }
1700 }
1701
1702 /**
1703  * evergreen_hpd_init - hpd setup callback.
1704  *
1705  * @rdev: radeon_device pointer
1706  *
1707  * Setup the hpd pins used by the card (evergreen+).
1708  * Enable the pin, set the polarity, and enable the hpd interrupts.
1709  */
1710 void evergreen_hpd_init(struct radeon_device *rdev)
1711 {
1712         struct drm_device *dev = rdev->ddev;
1713         struct drm_connector *connector;
1714         unsigned enabled = 0;
1715         u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1716                 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1717
1718         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1719                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1720
1721                 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1722                     connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1723                         /* don't try to enable hpd on eDP or LVDS avoid breaking the
1724                          * aux dp channel on imac and help (but not completely fix)
1725                          * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1726                          * also avoid interrupt storms during dpms.
1727                          */
1728                         continue;
1729                 }
1730                 switch (radeon_connector->hpd.hpd) {
1731                 case RADEON_HPD_1:
1732                         WREG32(DC_HPD1_CONTROL, tmp);
1733                         break;
1734                 case RADEON_HPD_2:
1735                         WREG32(DC_HPD2_CONTROL, tmp);
1736                         break;
1737                 case RADEON_HPD_3:
1738                         WREG32(DC_HPD3_CONTROL, tmp);
1739                         break;
1740                 case RADEON_HPD_4:
1741                         WREG32(DC_HPD4_CONTROL, tmp);
1742                         break;
1743                 case RADEON_HPD_5:
1744                         WREG32(DC_HPD5_CONTROL, tmp);
1745                         break;
1746                 case RADEON_HPD_6:
1747                         WREG32(DC_HPD6_CONTROL, tmp);
1748                         break;
1749                 default:
1750                         break;
1751                 }
1752                 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
1753                 enabled |= 1 << radeon_connector->hpd.hpd;
1754         }
1755         radeon_irq_kms_enable_hpd(rdev, enabled);
1756 }
1757
1758 /**
1759  * evergreen_hpd_fini - hpd tear down callback.
1760  *
1761  * @rdev: radeon_device pointer
1762  *
1763  * Tear down the hpd pins used by the card (evergreen+).
1764  * Disable the hpd interrupts.
1765  */
1766 void evergreen_hpd_fini(struct radeon_device *rdev)
1767 {
1768         struct drm_device *dev = rdev->ddev;
1769         struct drm_connector *connector;
1770         unsigned disabled = 0;
1771
1772         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1773                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1774                 switch (radeon_connector->hpd.hpd) {
1775                 case RADEON_HPD_1:
1776                         WREG32(DC_HPD1_CONTROL, 0);
1777                         break;
1778                 case RADEON_HPD_2:
1779                         WREG32(DC_HPD2_CONTROL, 0);
1780                         break;
1781                 case RADEON_HPD_3:
1782                         WREG32(DC_HPD3_CONTROL, 0);
1783                         break;
1784                 case RADEON_HPD_4:
1785                         WREG32(DC_HPD4_CONTROL, 0);
1786                         break;
1787                 case RADEON_HPD_5:
1788                         WREG32(DC_HPD5_CONTROL, 0);
1789                         break;
1790                 case RADEON_HPD_6:
1791                         WREG32(DC_HPD6_CONTROL, 0);
1792                         break;
1793                 default:
1794                         break;
1795                 }
1796                 disabled |= 1 << radeon_connector->hpd.hpd;
1797         }
1798         radeon_irq_kms_disable_hpd(rdev, disabled);
1799 }
1800
1801 /* watermark setup */
1802
1803 static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1804                                         struct radeon_crtc *radeon_crtc,
1805                                         struct drm_display_mode *mode,
1806                                         struct drm_display_mode *other_mode)
1807 {
1808         u32 tmp;
1809         /*
1810          * Line Buffer Setup
1811          * There are 3 line buffers, each one shared by 2 display controllers.
1812          * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1813          * the display controllers.  The paritioning is done via one of four
1814          * preset allocations specified in bits 2:0:
1815          * first display controller
1816          *  0 - first half of lb (3840 * 2)
1817          *  1 - first 3/4 of lb (5760 * 2)
1818          *  2 - whole lb (7680 * 2), other crtc must be disabled
1819          *  3 - first 1/4 of lb (1920 * 2)
1820          * second display controller
1821          *  4 - second half of lb (3840 * 2)
1822          *  5 - second 3/4 of lb (5760 * 2)
1823          *  6 - whole lb (7680 * 2), other crtc must be disabled
1824          *  7 - last 1/4 of lb (1920 * 2)
1825          */
1826         /* this can get tricky if we have two large displays on a paired group
1827          * of crtcs.  Ideally for multiple large displays we'd assign them to
1828          * non-linked crtcs for maximum line buffer allocation.
1829          */
1830         if (radeon_crtc->base.enabled && mode) {
1831                 if (other_mode)
1832                         tmp = 0; /* 1/2 */
1833                 else
1834                         tmp = 2; /* whole */
1835         } else
1836                 tmp = 0;
1837
1838         /* second controller of the pair uses second half of the lb */
1839         if (radeon_crtc->crtc_id % 2)
1840                 tmp += 4;
1841         WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1842
1843         if (radeon_crtc->base.enabled && mode) {
1844                 switch (tmp) {
1845                 case 0:
1846                 case 4:
1847                 default:
1848                         if (ASIC_IS_DCE5(rdev))
1849                                 return 4096 * 2;
1850                         else
1851                                 return 3840 * 2;
1852                 case 1:
1853                 case 5:
1854                         if (ASIC_IS_DCE5(rdev))
1855                                 return 6144 * 2;
1856                         else
1857                                 return 5760 * 2;
1858                 case 2:
1859                 case 6:
1860                         if (ASIC_IS_DCE5(rdev))
1861                                 return 8192 * 2;
1862                         else
1863                                 return 7680 * 2;
1864                 case 3:
1865                 case 7:
1866                         if (ASIC_IS_DCE5(rdev))
1867                                 return 2048 * 2;
1868                         else
1869                                 return 1920 * 2;
1870                 }
1871         }
1872
1873         /* controller not enabled, so no lb used */
1874         return 0;
1875 }
1876
1877 u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1878 {
1879         u32 tmp = RREG32(MC_SHARED_CHMAP);
1880
1881         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1882         case 0:
1883         default:
1884                 return 1;
1885         case 1:
1886                 return 2;
1887         case 2:
1888                 return 4;
1889         case 3:
1890                 return 8;
1891         }
1892 }
1893
1894 struct evergreen_wm_params {
1895         u32 dram_channels; /* number of dram channels */
1896         u32 yclk;          /* bandwidth per dram data pin in kHz */
1897         u32 sclk;          /* engine clock in kHz */
1898         u32 disp_clk;      /* display clock in kHz */
1899         u32 src_width;     /* viewport width */
1900         u32 active_time;   /* active display time in ns */
1901         u32 blank_time;    /* blank time in ns */
1902         bool interlaced;    /* mode is interlaced */
1903         fixed20_12 vsc;    /* vertical scale ratio */
1904         u32 num_heads;     /* number of active crtcs */
1905         u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1906         u32 lb_size;       /* line buffer allocated to pipe */
1907         u32 vtaps;         /* vertical scaler taps */
1908 };
1909
1910 static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1911 {
1912         /* Calculate DRAM Bandwidth and the part allocated to display. */
1913         fixed20_12 dram_efficiency; /* 0.7 */
1914         fixed20_12 yclk, dram_channels, bandwidth;
1915         fixed20_12 a;
1916
1917         a.full = dfixed_const(1000);
1918         yclk.full = dfixed_const(wm->yclk);
1919         yclk.full = dfixed_div(yclk, a);
1920         dram_channels.full = dfixed_const(wm->dram_channels * 4);
1921         a.full = dfixed_const(10);
1922         dram_efficiency.full = dfixed_const(7);
1923         dram_efficiency.full = dfixed_div(dram_efficiency, a);
1924         bandwidth.full = dfixed_mul(dram_channels, yclk);
1925         bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1926
1927         return dfixed_trunc(bandwidth);
1928 }
1929
1930 static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1931 {
1932         /* Calculate DRAM Bandwidth and the part allocated to display. */
1933         fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1934         fixed20_12 yclk, dram_channels, bandwidth;
1935         fixed20_12 a;
1936
1937         a.full = dfixed_const(1000);
1938         yclk.full = dfixed_const(wm->yclk);
1939         yclk.full = dfixed_div(yclk, a);
1940         dram_channels.full = dfixed_const(wm->dram_channels * 4);
1941         a.full = dfixed_const(10);
1942         disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1943         disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1944         bandwidth.full = dfixed_mul(dram_channels, yclk);
1945         bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1946
1947         return dfixed_trunc(bandwidth);
1948 }
1949
1950 static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
1951 {
1952         /* Calculate the display Data return Bandwidth */
1953         fixed20_12 return_efficiency; /* 0.8 */
1954         fixed20_12 sclk, bandwidth;
1955         fixed20_12 a;
1956
1957         a.full = dfixed_const(1000);
1958         sclk.full = dfixed_const(wm->sclk);
1959         sclk.full = dfixed_div(sclk, a);
1960         a.full = dfixed_const(10);
1961         return_efficiency.full = dfixed_const(8);
1962         return_efficiency.full = dfixed_div(return_efficiency, a);
1963         a.full = dfixed_const(32);
1964         bandwidth.full = dfixed_mul(a, sclk);
1965         bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1966
1967         return dfixed_trunc(bandwidth);
1968 }
1969
1970 static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
1971 {
1972         /* Calculate the DMIF Request Bandwidth */
1973         fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1974         fixed20_12 disp_clk, bandwidth;
1975         fixed20_12 a;
1976
1977         a.full = dfixed_const(1000);
1978         disp_clk.full = dfixed_const(wm->disp_clk);
1979         disp_clk.full = dfixed_div(disp_clk, a);
1980         a.full = dfixed_const(10);
1981         disp_clk_request_efficiency.full = dfixed_const(8);
1982         disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1983         a.full = dfixed_const(32);
1984         bandwidth.full = dfixed_mul(a, disp_clk);
1985         bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
1986
1987         return dfixed_trunc(bandwidth);
1988 }
1989
1990 static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
1991 {
1992         /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1993         u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
1994         u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
1995         u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
1996
1997         return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1998 }
1999
2000 static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2001 {
2002         /* Calculate the display mode Average Bandwidth
2003          * DisplayMode should contain the source and destination dimensions,
2004          * timing, etc.
2005          */
2006         fixed20_12 bpp;
2007         fixed20_12 line_time;
2008         fixed20_12 src_width;
2009         fixed20_12 bandwidth;
2010         fixed20_12 a;
2011
2012         a.full = dfixed_const(1000);
2013         line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2014         line_time.full = dfixed_div(line_time, a);
2015         bpp.full = dfixed_const(wm->bytes_per_pixel);
2016         src_width.full = dfixed_const(wm->src_width);
2017         bandwidth.full = dfixed_mul(src_width, bpp);
2018         bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2019         bandwidth.full = dfixed_div(bandwidth, line_time);
2020
2021         return dfixed_trunc(bandwidth);
2022 }
2023
2024 static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2025 {
2026         /* First calcualte the latency in ns */
2027         u32 mc_latency = 2000; /* 2000 ns. */
2028         u32 available_bandwidth = evergreen_available_bandwidth(wm);
2029         u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2030         u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2031         u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2032         u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2033                 (wm->num_heads * cursor_line_pair_return_time);
2034         u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2035         u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2036         fixed20_12 a, b, c;
2037
2038         if (wm->num_heads == 0)
2039                 return 0;
2040
2041         a.full = dfixed_const(2);
2042         b.full = dfixed_const(1);
2043         if ((wm->vsc.full > a.full) ||
2044             ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2045             (wm->vtaps >= 5) ||
2046             ((wm->vsc.full >= a.full) && wm->interlaced))
2047                 max_src_lines_per_dst_line = 4;
2048         else
2049                 max_src_lines_per_dst_line = 2;
2050
2051         a.full = dfixed_const(available_bandwidth);
2052         b.full = dfixed_const(wm->num_heads);
2053         a.full = dfixed_div(a, b);
2054
2055         b.full = dfixed_const(1000);
2056         c.full = dfixed_const(wm->disp_clk);
2057         b.full = dfixed_div(c, b);
2058         c.full = dfixed_const(wm->bytes_per_pixel);
2059         b.full = dfixed_mul(b, c);
2060
2061         lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
2062
2063         a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2064         b.full = dfixed_const(1000);
2065         c.full = dfixed_const(lb_fill_bw);
2066         b.full = dfixed_div(c, b);
2067         a.full = dfixed_div(a, b);
2068         line_fill_time = dfixed_trunc(a);
2069
2070         if (line_fill_time < wm->active_time)
2071                 return latency;
2072         else
2073                 return latency + (line_fill_time - wm->active_time);
2074
2075 }
2076
2077 static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2078 {
2079         if (evergreen_average_bandwidth(wm) <=
2080             (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2081                 return true;
2082         else
2083                 return false;
2084 };
2085
2086 static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2087 {
2088         if (evergreen_average_bandwidth(wm) <=
2089             (evergreen_available_bandwidth(wm) / wm->num_heads))
2090                 return true;
2091         else
2092                 return false;
2093 };
2094
2095 static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2096 {
2097         u32 lb_partitions = wm->lb_size / wm->src_width;
2098         u32 line_time = wm->active_time + wm->blank_time;
2099         u32 latency_tolerant_lines;
2100         u32 latency_hiding;
2101         fixed20_12 a;
2102
2103         a.full = dfixed_const(1);
2104         if (wm->vsc.full > a.full)
2105                 latency_tolerant_lines = 1;
2106         else {
2107                 if (lb_partitions <= (wm->vtaps + 1))
2108                         latency_tolerant_lines = 1;
2109                 else
2110                         latency_tolerant_lines = 2;
2111         }
2112
2113         latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2114
2115         if (evergreen_latency_watermark(wm) <= latency_hiding)
2116                 return true;
2117         else
2118                 return false;
2119 }
2120
2121 static void evergreen_program_watermarks(struct radeon_device *rdev,
2122                                          struct radeon_crtc *radeon_crtc,
2123                                          u32 lb_size, u32 num_heads)
2124 {
2125         struct drm_display_mode *mode = &radeon_crtc->base.mode;
2126         struct evergreen_wm_params wm_low, wm_high;
2127         u32 dram_channels;
2128         u32 pixel_period;
2129         u32 line_time = 0;
2130         u32 latency_watermark_a = 0, latency_watermark_b = 0;
2131         u32 priority_a_mark = 0, priority_b_mark = 0;
2132         u32 priority_a_cnt = PRIORITY_OFF;
2133         u32 priority_b_cnt = PRIORITY_OFF;
2134         u32 pipe_offset = radeon_crtc->crtc_id * 16;
2135         u32 tmp, arb_control3;
2136         fixed20_12 a, b, c;
2137
2138         if (radeon_crtc->base.enabled && num_heads && mode) {
2139                 pixel_period = 1000000 / (u32)mode->clock;
2140                 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2141                 priority_a_cnt = 0;
2142                 priority_b_cnt = 0;
2143                 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2144
2145                 /* watermark for high clocks */
2146                 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2147                         wm_high.yclk =
2148                                 radeon_dpm_get_mclk(rdev, false) * 10;
2149                         wm_high.sclk =
2150                                 radeon_dpm_get_sclk(rdev, false) * 10;
2151                 } else {
2152                         wm_high.yclk = rdev->pm.current_mclk * 10;
2153                         wm_high.sclk = rdev->pm.current_sclk * 10;
2154                 }
2155
2156                 wm_high.disp_clk = mode->clock;
2157                 wm_high.src_width = mode->crtc_hdisplay;
2158                 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2159                 wm_high.blank_time = line_time - wm_high.active_time;
2160                 wm_high.interlaced = false;
2161                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2162                         wm_high.interlaced = true;
2163                 wm_high.vsc = radeon_crtc->vsc;
2164                 wm_high.vtaps = 1;
2165                 if (radeon_crtc->rmx_type != RMX_OFF)
2166                         wm_high.vtaps = 2;
2167                 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2168                 wm_high.lb_size = lb_size;
2169                 wm_high.dram_channels = dram_channels;
2170                 wm_high.num_heads = num_heads;
2171
2172                 /* watermark for low clocks */
2173                 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2174                         wm_low.yclk =
2175                                 radeon_dpm_get_mclk(rdev, true) * 10;
2176                         wm_low.sclk =
2177                                 radeon_dpm_get_sclk(rdev, true) * 10;
2178                 } else {
2179                         wm_low.yclk = rdev->pm.current_mclk * 10;
2180                         wm_low.sclk = rdev->pm.current_sclk * 10;
2181                 }
2182
2183                 wm_low.disp_clk = mode->clock;
2184                 wm_low.src_width = mode->crtc_hdisplay;
2185                 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2186                 wm_low.blank_time = line_time - wm_low.active_time;
2187                 wm_low.interlaced = false;
2188                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2189                         wm_low.interlaced = true;
2190                 wm_low.vsc = radeon_crtc->vsc;
2191                 wm_low.vtaps = 1;
2192                 if (radeon_crtc->rmx_type != RMX_OFF)
2193                         wm_low.vtaps = 2;
2194                 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2195                 wm_low.lb_size = lb_size;
2196                 wm_low.dram_channels = dram_channels;
2197                 wm_low.num_heads = num_heads;
2198
2199                 /* set for high clocks */
2200                 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2201                 /* set for low clocks */
2202                 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2203
2204                 /* possibly force display priority to high */
2205                 /* should really do this at mode validation time... */
2206                 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2207                     !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2208                     !evergreen_check_latency_hiding(&wm_high) ||
2209                     (rdev->disp_priority == 2)) {
2210                         DRM_DEBUG_KMS("force priority a to high\n");
2211                         priority_a_cnt |= PRIORITY_ALWAYS_ON;
2212                 }
2213                 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2214                     !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2215                     !evergreen_check_latency_hiding(&wm_low) ||
2216                     (rdev->disp_priority == 2)) {
2217                         DRM_DEBUG_KMS("force priority b to high\n");
2218                         priority_b_cnt |= PRIORITY_ALWAYS_ON;
2219                 }
2220
2221                 a.full = dfixed_const(1000);
2222                 b.full = dfixed_const(mode->clock);
2223                 b.full = dfixed_div(b, a);
2224                 c.full = dfixed_const(latency_watermark_a);
2225                 c.full = dfixed_mul(c, b);
2226                 c.full = dfixed_mul(c, radeon_crtc->hsc);
2227                 c.full = dfixed_div(c, a);
2228                 a.full = dfixed_const(16);
2229                 c.full = dfixed_div(c, a);
2230                 priority_a_mark = dfixed_trunc(c);
2231                 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2232
2233                 a.full = dfixed_const(1000);
2234                 b.full = dfixed_const(mode->clock);
2235                 b.full = dfixed_div(b, a);
2236                 c.full = dfixed_const(latency_watermark_b);
2237                 c.full = dfixed_mul(c, b);
2238                 c.full = dfixed_mul(c, radeon_crtc->hsc);
2239                 c.full = dfixed_div(c, a);
2240                 a.full = dfixed_const(16);
2241                 c.full = dfixed_div(c, a);
2242                 priority_b_mark = dfixed_trunc(c);
2243                 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2244         }
2245
2246         /* select wm A */
2247         arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2248         tmp = arb_control3;
2249         tmp &= ~LATENCY_WATERMARK_MASK(3);
2250         tmp |= LATENCY_WATERMARK_MASK(1);
2251         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2252         WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2253                (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2254                 LATENCY_HIGH_WATERMARK(line_time)));
2255         /* select wm B */
2256         tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2257         tmp &= ~LATENCY_WATERMARK_MASK(3);
2258         tmp |= LATENCY_WATERMARK_MASK(2);
2259         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2260         WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2261                (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2262                 LATENCY_HIGH_WATERMARK(line_time)));
2263         /* restore original selection */
2264         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2265
2266         /* write the priority marks */
2267         WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2268         WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2269
2270         /* save values for DPM */
2271         radeon_crtc->line_time = line_time;
2272         radeon_crtc->wm_high = latency_watermark_a;
2273         radeon_crtc->wm_low = latency_watermark_b;
2274 }
2275
2276 /**
2277  * evergreen_bandwidth_update - update display watermarks callback.
2278  *
2279  * @rdev: radeon_device pointer
2280  *
2281  * Update the display watermarks based on the requested mode(s)
2282  * (evergreen+).
2283  */
2284 void evergreen_bandwidth_update(struct radeon_device *rdev)
2285 {
2286         struct drm_display_mode *mode0 = NULL;
2287         struct drm_display_mode *mode1 = NULL;
2288         u32 num_heads = 0, lb_size;
2289         int i;
2290
2291         radeon_update_display_priority(rdev);
2292
2293         for (i = 0; i < rdev->num_crtc; i++) {
2294                 if (rdev->mode_info.crtcs[i]->base.enabled)
2295                         num_heads++;
2296         }
2297         for (i = 0; i < rdev->num_crtc; i += 2) {
2298                 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2299                 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2300                 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2301                 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2302                 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2303                 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2304         }
2305 }
2306
2307 /**
2308  * evergreen_mc_wait_for_idle - wait for MC idle callback.
2309  *
2310  * @rdev: radeon_device pointer
2311  *
2312  * Wait for the MC (memory controller) to be idle.
2313  * (evergreen+).
2314  * Returns 0 if the MC is idle, -1 if not.
2315  */
2316 int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2317 {
2318         unsigned i;
2319         u32 tmp;
2320
2321         for (i = 0; i < rdev->usec_timeout; i++) {
2322                 /* read MC_STATUS */
2323                 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2324                 if (!tmp)
2325                         return 0;
2326                 udelay(1);
2327         }
2328         return -1;
2329 }
2330
2331 /*
2332  * GART
2333  */
2334 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2335 {
2336         unsigned i;
2337         u32 tmp;
2338
2339         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2340
2341         WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2342         for (i = 0; i < rdev->usec_timeout; i++) {
2343                 /* read MC_STATUS */
2344                 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2345                 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2346                 if (tmp == 2) {
2347                         printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
2348                         return;
2349                 }
2350                 if (tmp) {
2351                         return;
2352                 }
2353                 udelay(1);
2354         }
2355 }
2356
2357 static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2358 {
2359         u32 tmp;
2360         int r;
2361
2362         if (rdev->gart.robj == NULL) {
2363                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2364                 return -EINVAL;
2365         }
2366         r = radeon_gart_table_vram_pin(rdev);
2367         if (r)
2368                 return r;
2369         radeon_gart_restore(rdev);
2370         /* Setup L2 cache */
2371         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2372                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2373                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2374         WREG32(VM_L2_CNTL2, 0);
2375         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2376         /* Setup TLB control */
2377         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2378                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2379                 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2380                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2381         if (rdev->flags & RADEON_IS_IGP) {
2382                 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2383                 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2384                 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2385         } else {
2386                 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2387                 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2388                 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2389                 if ((rdev->family == CHIP_JUNIPER) ||
2390                     (rdev->family == CHIP_CYPRESS) ||
2391                     (rdev->family == CHIP_HEMLOCK) ||
2392                     (rdev->family == CHIP_BARTS))
2393                         WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2394         }
2395         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2396         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2397         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2398         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2399         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2400         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2401         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2402         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2403                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2404         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2405                         (u32)(rdev->dummy_page.addr >> 12));
2406         WREG32(VM_CONTEXT1_CNTL, 0);
2407
2408         evergreen_pcie_gart_tlb_flush(rdev);
2409         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2410                  (unsigned)(rdev->mc.gtt_size >> 20),
2411                  (unsigned long long)rdev->gart.table_addr);
2412         rdev->gart.ready = true;
2413         return 0;
2414 }
2415
2416 static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2417 {
2418         u32 tmp;
2419
2420         /* Disable all tables */
2421         WREG32(VM_CONTEXT0_CNTL, 0);
2422         WREG32(VM_CONTEXT1_CNTL, 0);
2423
2424         /* Setup L2 cache */
2425         WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2426                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2427         WREG32(VM_L2_CNTL2, 0);
2428         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2429         /* Setup TLB control */
2430         tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2431         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2432         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2433         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2434         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2435         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2436         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2437         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2438         radeon_gart_table_vram_unpin(rdev);
2439 }
2440
2441 static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2442 {
2443         evergreen_pcie_gart_disable(rdev);
2444         radeon_gart_table_vram_free(rdev);
2445         radeon_gart_fini(rdev);
2446 }
2447
2448
2449 static void evergreen_agp_enable(struct radeon_device *rdev)
2450 {
2451         u32 tmp;
2452
2453         /* Setup L2 cache */
2454         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2455                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2456                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2457         WREG32(VM_L2_CNTL2, 0);
2458         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2459         /* Setup TLB control */
2460         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2461                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2462                 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2463                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2464         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2465         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2466         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2467         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2468         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2469         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2470         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2471         WREG32(VM_CONTEXT0_CNTL, 0);
2472         WREG32(VM_CONTEXT1_CNTL, 0);
2473 }
2474
2475 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2476 {
2477         u32 crtc_enabled, tmp, frame_count, blackout;
2478         int i, j;
2479
2480         if (!ASIC_IS_NODCE(rdev)) {
2481                 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2482                 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2483
2484                 /* disable VGA render */
2485                 WREG32(VGA_RENDER_CONTROL, 0);
2486         }
2487         /* blank the display controllers */
2488         for (i = 0; i < rdev->num_crtc; i++) {
2489                 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2490                 if (crtc_enabled) {
2491                         save->crtc_enabled[i] = true;
2492                         if (ASIC_IS_DCE6(rdev)) {
2493                                 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2494                                 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2495                                         radeon_wait_for_vblank(rdev, i);
2496                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2497                                         tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2498                                         WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2499                                 }
2500                         } else {
2501                                 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2502                                 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2503                                         radeon_wait_for_vblank(rdev, i);
2504                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2505                                         tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2506                                         WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2507                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2508                                 }
2509                         }
2510                         /* wait for the next frame */
2511                         frame_count = radeon_get_vblank_counter(rdev, i);
2512                         for (j = 0; j < rdev->usec_timeout; j++) {
2513                                 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2514                                         break;
2515                                 udelay(1);
2516                         }
2517
2518                         /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2519                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2520                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2521                         tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2522                         WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2523                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2524                         save->crtc_enabled[i] = false;
2525                         /* ***** */
2526                 } else {
2527                         save->crtc_enabled[i] = false;
2528                 }
2529         }
2530
2531         radeon_mc_wait_for_idle(rdev);
2532
2533         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2534         if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2535                 /* Block CPU access */
2536                 WREG32(BIF_FB_EN, 0);
2537                 /* blackout the MC */
2538                 blackout &= ~BLACKOUT_MODE_MASK;
2539                 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2540         }
2541         /* wait for the MC to settle */
2542         udelay(100);
2543
2544         /* lock double buffered regs */
2545         for (i = 0; i < rdev->num_crtc; i++) {
2546                 if (save->crtc_enabled[i]) {
2547                         tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2548                         if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2549                                 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2550                                 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2551                         }
2552                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2553                         if (!(tmp & 1)) {
2554                                 tmp |= 1;
2555                                 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2556                         }
2557                 }
2558         }
2559 }
2560
2561 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2562 {
2563         u32 tmp, frame_count;
2564         int i, j;
2565
2566         /* update crtc base addresses */
2567         for (i = 0; i < rdev->num_crtc; i++) {
2568                 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2569                        upper_32_bits(rdev->mc.vram_start));
2570                 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2571                        upper_32_bits(rdev->mc.vram_start));
2572                 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2573                        (u32)rdev->mc.vram_start);
2574                 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2575                        (u32)rdev->mc.vram_start);
2576         }
2577
2578         if (!ASIC_IS_NODCE(rdev)) {
2579                 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2580                 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2581         }
2582
2583         /* unlock regs and wait for update */
2584         for (i = 0; i < rdev->num_crtc; i++) {
2585                 if (save->crtc_enabled[i]) {
2586                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2587                         if ((tmp & 0x3) != 0) {
2588                                 tmp &= ~0x3;
2589                                 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2590                         }
2591                         tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2592                         if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2593                                 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2594                                 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2595                         }
2596                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2597                         if (tmp & 1) {
2598                                 tmp &= ~1;
2599                                 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2600                         }
2601                         for (j = 0; j < rdev->usec_timeout; j++) {
2602                                 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2603                                 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2604                                         break;
2605                                 udelay(1);
2606                         }
2607                 }
2608         }
2609
2610         /* unblackout the MC */
2611         tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2612         tmp &= ~BLACKOUT_MODE_MASK;
2613         WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2614         /* allow CPU access */
2615         WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2616
2617         for (i = 0; i < rdev->num_crtc; i++) {
2618                 if (save->crtc_enabled[i]) {
2619                         if (ASIC_IS_DCE6(rdev)) {
2620                                 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2621                                 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2622                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2623                                 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2624                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2625                         } else {
2626                                 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2627                                 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2628                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2629                                 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2630                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2631                         }
2632                         /* wait for the next frame */
2633                         frame_count = radeon_get_vblank_counter(rdev, i);
2634                         for (j = 0; j < rdev->usec_timeout; j++) {
2635                                 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2636                                         break;
2637                                 udelay(1);
2638                         }
2639                 }
2640         }
2641         if (!ASIC_IS_NODCE(rdev)) {
2642                 /* Unlock vga access */
2643                 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2644                 mdelay(1);
2645                 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2646         }
2647 }
2648
2649 void evergreen_mc_program(struct radeon_device *rdev)
2650 {
2651         struct evergreen_mc_save save;
2652         u32 tmp;
2653         int i, j;
2654
2655         /* Initialize HDP */
2656         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2657                 WREG32((0x2c14 + j), 0x00000000);
2658                 WREG32((0x2c18 + j), 0x00000000);
2659                 WREG32((0x2c1c + j), 0x00000000);
2660                 WREG32((0x2c20 + j), 0x00000000);
2661                 WREG32((0x2c24 + j), 0x00000000);
2662         }
2663         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2664
2665         evergreen_mc_stop(rdev, &save);
2666         if (evergreen_mc_wait_for_idle(rdev)) {
2667                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2668         }
2669         /* Lockout access through VGA aperture*/
2670         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2671         /* Update configuration */
2672         if (rdev->flags & RADEON_IS_AGP) {
2673                 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2674                         /* VRAM before AGP */
2675                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2676                                 rdev->mc.vram_start >> 12);
2677                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2678                                 rdev->mc.gtt_end >> 12);
2679                 } else {
2680                         /* VRAM after AGP */
2681                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2682                                 rdev->mc.gtt_start >> 12);
2683                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2684                                 rdev->mc.vram_end >> 12);
2685                 }
2686         } else {
2687                 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2688                         rdev->mc.vram_start >> 12);
2689                 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2690                         rdev->mc.vram_end >> 12);
2691         }
2692         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2693         /* llano/ontario only */
2694         if ((rdev->family == CHIP_PALM) ||
2695             (rdev->family == CHIP_SUMO) ||
2696             (rdev->family == CHIP_SUMO2)) {
2697                 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2698                 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2699                 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2700                 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2701         }
2702         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2703         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2704         WREG32(MC_VM_FB_LOCATION, tmp);
2705         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2706         WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2707         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2708         if (rdev->flags & RADEON_IS_AGP) {
2709                 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2710                 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2711                 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2712         } else {
2713                 WREG32(MC_VM_AGP_BASE, 0);
2714                 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2715                 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2716         }
2717         if (evergreen_mc_wait_for_idle(rdev)) {
2718                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2719         }
2720         evergreen_mc_resume(rdev, &save);
2721         /* we need to own VRAM, so turn off the VGA renderer here
2722          * to stop it overwriting our objects */
2723         rv515_vga_render_disable(rdev);
2724 }
2725
2726 /*
2727  * CP.
2728  */
2729 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2730 {
2731         struct radeon_ring *ring = &rdev->ring[ib->ring];
2732         u32 next_rptr;
2733
2734         /* set to DX10/11 mode */
2735         radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2736         radeon_ring_write(ring, 1);
2737
2738         if (ring->rptr_save_reg) {
2739                 next_rptr = ring->wptr + 3 + 4;
2740                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2741                 radeon_ring_write(ring, ((ring->rptr_save_reg -
2742                                           PACKET3_SET_CONFIG_REG_START) >> 2));
2743                 radeon_ring_write(ring, next_rptr);
2744         } else if (rdev->wb.enabled) {
2745                 next_rptr = ring->wptr + 5 + 4;
2746                 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2747                 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2748                 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2749                 radeon_ring_write(ring, next_rptr);
2750                 radeon_ring_write(ring, 0);
2751         }
2752
2753         radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2754         radeon_ring_write(ring,
2755 #ifdef __BIG_ENDIAN
2756                           (2 << 0) |
2757 #endif
2758                           (ib->gpu_addr & 0xFFFFFFFC));
2759         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2760         radeon_ring_write(ring, ib->length_dw);
2761 }
2762
2763
2764 static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2765 {
2766         const __be32 *fw_data;
2767         int i;
2768
2769         if (!rdev->me_fw || !rdev->pfp_fw)
2770                 return -EINVAL;
2771
2772         r700_cp_stop(rdev);
2773         WREG32(CP_RB_CNTL,
2774 #ifdef __BIG_ENDIAN
2775                BUF_SWAP_32BIT |
2776 #endif
2777                RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2778
2779         fw_data = (const __be32 *)rdev->pfp_fw->data;
2780         WREG32(CP_PFP_UCODE_ADDR, 0);
2781         for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2782                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2783         WREG32(CP_PFP_UCODE_ADDR, 0);
2784
2785         fw_data = (const __be32 *)rdev->me_fw->data;
2786         WREG32(CP_ME_RAM_WADDR, 0);
2787         for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2788                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2789
2790         WREG32(CP_PFP_UCODE_ADDR, 0);
2791         WREG32(CP_ME_RAM_WADDR, 0);
2792         WREG32(CP_ME_RAM_RADDR, 0);
2793         return 0;
2794 }
2795
2796 static int evergreen_cp_start(struct radeon_device *rdev)
2797 {
2798         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2799         int r, i;
2800         uint32_t cp_me;
2801
2802         r = radeon_ring_lock(rdev, ring, 7);
2803         if (r) {
2804                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2805                 return r;
2806         }
2807         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2808         radeon_ring_write(ring, 0x1);
2809         radeon_ring_write(ring, 0x0);
2810         radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
2811         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2812         radeon_ring_write(ring, 0);
2813         radeon_ring_write(ring, 0);
2814         radeon_ring_unlock_commit(rdev, ring);
2815
2816         cp_me = 0xff;
2817         WREG32(CP_ME_CNTL, cp_me);
2818
2819         r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
2820         if (r) {
2821                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2822                 return r;
2823         }
2824
2825         /* setup clear context state */
2826         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2827         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2828
2829         for (i = 0; i < evergreen_default_size; i++)
2830                 radeon_ring_write(ring, evergreen_default_state[i]);
2831
2832         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2833         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2834
2835         /* set clear context state */
2836         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2837         radeon_ring_write(ring, 0);
2838
2839         /* SQ_VTX_BASE_VTX_LOC */
2840         radeon_ring_write(ring, 0xc0026f00);
2841         radeon_ring_write(ring, 0x00000000);
2842         radeon_ring_write(ring, 0x00000000);
2843         radeon_ring_write(ring, 0x00000000);
2844
2845         /* Clear consts */
2846         radeon_ring_write(ring, 0xc0036f00);
2847         radeon_ring_write(ring, 0x00000bc4);
2848         radeon_ring_write(ring, 0xffffffff);
2849         radeon_ring_write(ring, 0xffffffff);
2850         radeon_ring_write(ring, 0xffffffff);
2851
2852         radeon_ring_write(ring, 0xc0026900);
2853         radeon_ring_write(ring, 0x00000316);
2854         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2855         radeon_ring_write(ring, 0x00000010); /*  */
2856
2857         radeon_ring_unlock_commit(rdev, ring);
2858
2859         return 0;
2860 }
2861
2862 static int evergreen_cp_resume(struct radeon_device *rdev)
2863 {
2864         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2865         u32 tmp;
2866         u32 rb_bufsz;
2867         int r;
2868
2869         /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
2870         WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
2871                                  SOFT_RESET_PA |
2872                                  SOFT_RESET_SH |
2873                                  SOFT_RESET_VGT |
2874                                  SOFT_RESET_SPI |
2875                                  SOFT_RESET_SX));
2876         RREG32(GRBM_SOFT_RESET);
2877         mdelay(15);
2878         WREG32(GRBM_SOFT_RESET, 0);
2879         RREG32(GRBM_SOFT_RESET);
2880
2881         /* Set ring buffer size */
2882         rb_bufsz = drm_order(ring->ring_size / 8);
2883         tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2884 #ifdef __BIG_ENDIAN
2885         tmp |= BUF_SWAP_32BIT;
2886 #endif
2887         WREG32(CP_RB_CNTL, tmp);
2888         WREG32(CP_SEM_WAIT_TIMER, 0x0);
2889         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
2890
2891         /* Set the write pointer delay */
2892         WREG32(CP_RB_WPTR_DELAY, 0);
2893
2894         /* Initialize the ring buffer's read and write pointers */
2895         WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2896         WREG32(CP_RB_RPTR_WR, 0);
2897         ring->wptr = 0;
2898         WREG32(CP_RB_WPTR, ring->wptr);
2899
2900         /* set the wb address whether it's enabled or not */
2901         WREG32(CP_RB_RPTR_ADDR,
2902                ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
2903         WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2904         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2905
2906         if (rdev->wb.enabled)
2907                 WREG32(SCRATCH_UMSK, 0xff);
2908         else {
2909                 tmp |= RB_NO_UPDATE;
2910                 WREG32(SCRATCH_UMSK, 0);
2911         }
2912
2913         mdelay(1);
2914         WREG32(CP_RB_CNTL, tmp);
2915
2916         WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
2917         WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
2918
2919         ring->rptr = RREG32(CP_RB_RPTR);
2920
2921         evergreen_cp_start(rdev);
2922         ring->ready = true;
2923         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
2924         if (r) {
2925                 ring->ready = false;
2926                 return r;
2927         }
2928         return 0;
2929 }
2930
2931 /*
2932  * Core functions
2933  */
2934 static void evergreen_gpu_init(struct radeon_device *rdev)
2935 {
2936         u32 gb_addr_config;
2937         u32 mc_shared_chmap, mc_arb_ramcfg;
2938         u32 sx_debug_1;
2939         u32 smx_dc_ctl0;
2940         u32 sq_config;
2941         u32 sq_lds_resource_mgmt;
2942         u32 sq_gpr_resource_mgmt_1;
2943         u32 sq_gpr_resource_mgmt_2;
2944         u32 sq_gpr_resource_mgmt_3;
2945         u32 sq_thread_resource_mgmt;
2946         u32 sq_thread_resource_mgmt_2;
2947         u32 sq_stack_resource_mgmt_1;
2948         u32 sq_stack_resource_mgmt_2;
2949         u32 sq_stack_resource_mgmt_3;
2950         u32 vgt_cache_invalidation;
2951         u32 hdp_host_path_cntl, tmp;
2952         u32 disabled_rb_mask;
2953         int i, j, num_shader_engines, ps_thread_count;
2954
2955         switch (rdev->family) {
2956         case CHIP_CYPRESS:
2957         case CHIP_HEMLOCK:
2958                 rdev->config.evergreen.num_ses = 2;
2959                 rdev->config.evergreen.max_pipes = 4;
2960                 rdev->config.evergreen.max_tile_pipes = 8;
2961                 rdev->config.evergreen.max_simds = 10;
2962                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
2963                 rdev->config.evergreen.max_gprs = 256;
2964                 rdev->config.evergreen.max_threads = 248;
2965                 rdev->config.evergreen.max_gs_threads = 32;
2966                 rdev->config.evergreen.max_stack_entries = 512;
2967                 rdev->config.evergreen.sx_num_of_sets = 4;
2968                 rdev->config.evergreen.sx_max_export_size = 256;
2969                 rdev->config.evergreen.sx_max_export_pos_size = 64;
2970                 rdev->config.evergreen.sx_max_export_smx_size = 192;
2971                 rdev->config.evergreen.max_hw_contexts = 8;
2972                 rdev->config.evergreen.sq_num_cf_insts = 2;
2973
2974                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2975                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2976                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2977                 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
2978                 break;
2979         case CHIP_JUNIPER:
2980                 rdev->config.evergreen.num_ses = 1;
2981                 rdev->config.evergreen.max_pipes = 4;
2982                 rdev->config.evergreen.max_tile_pipes = 4;
2983                 rdev->config.evergreen.max_simds = 10;
2984                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
2985                 rdev->config.evergreen.max_gprs = 256;
2986                 rdev->config.evergreen.max_threads = 248;
2987                 rdev->config.evergreen.max_gs_threads = 32;
2988                 rdev->config.evergreen.max_stack_entries = 512;
2989                 rdev->config.evergreen.sx_num_of_sets = 4;
2990                 rdev->config.evergreen.sx_max_export_size = 256;
2991                 rdev->config.evergreen.sx_max_export_pos_size = 64;
2992                 rdev->config.evergreen.sx_max_export_smx_size = 192;
2993                 rdev->config.evergreen.max_hw_contexts = 8;
2994                 rdev->config.evergreen.sq_num_cf_insts = 2;
2995
2996                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2997                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2998                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2999                 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3000                 break;
3001         case CHIP_REDWOOD:
3002                 rdev->config.evergreen.num_ses = 1;
3003                 rdev->config.evergreen.max_pipes = 4;
3004                 rdev->config.evergreen.max_tile_pipes = 4;
3005                 rdev->config.evergreen.max_simds = 5;
3006                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3007                 rdev->config.evergreen.max_gprs = 256;
3008                 rdev->config.evergreen.max_threads = 248;
3009                 rdev->config.evergreen.max_gs_threads = 32;
3010                 rdev->config.evergreen.max_stack_entries = 256;
3011                 rdev->config.evergreen.sx_num_of_sets = 4;
3012                 rdev->config.evergreen.sx_max_export_size = 256;
3013                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3014                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3015                 rdev->config.evergreen.max_hw_contexts = 8;
3016                 rdev->config.evergreen.sq_num_cf_insts = 2;
3017
3018                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3019                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3020                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3021                 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3022                 break;
3023         case CHIP_CEDAR:
3024         default:
3025                 rdev->config.evergreen.num_ses = 1;
3026                 rdev->config.evergreen.max_pipes = 2;
3027                 rdev->config.evergreen.max_tile_pipes = 2;
3028                 rdev->config.evergreen.max_simds = 2;
3029                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3030                 rdev->config.evergreen.max_gprs = 256;
3031                 rdev->config.evergreen.max_threads = 192;
3032                 rdev->config.evergreen.max_gs_threads = 16;
3033                 rdev->config.evergreen.max_stack_entries = 256;
3034                 rdev->config.evergreen.sx_num_of_sets = 4;
3035                 rdev->config.evergreen.sx_max_export_size = 128;
3036                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3037                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3038                 rdev->config.evergreen.max_hw_contexts = 4;
3039                 rdev->config.evergreen.sq_num_cf_insts = 1;
3040
3041                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3042                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3043                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3044                 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3045                 break;
3046         case CHIP_PALM:
3047                 rdev->config.evergreen.num_ses = 1;
3048                 rdev->config.evergreen.max_pipes = 2;
3049                 rdev->config.evergreen.max_tile_pipes = 2;
3050                 rdev->config.evergreen.max_simds = 2;
3051                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3052                 rdev->config.evergreen.max_gprs = 256;
3053                 rdev->config.evergreen.max_threads = 192;
3054                 rdev->config.evergreen.max_gs_threads = 16;
3055                 rdev->config.evergreen.max_stack_entries = 256;
3056                 rdev->config.evergreen.sx_num_of_sets = 4;
3057                 rdev->config.evergreen.sx_max_export_size = 128;
3058                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3059                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3060                 rdev->config.evergreen.max_hw_contexts = 4;
3061                 rdev->config.evergreen.sq_num_cf_insts = 1;
3062
3063                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3064                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3065                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3066                 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3067                 break;
3068         case CHIP_SUMO:
3069                 rdev->config.evergreen.num_ses = 1;
3070                 rdev->config.evergreen.max_pipes = 4;
3071                 rdev->config.evergreen.max_tile_pipes = 4;
3072                 if (rdev->pdev->device == 0x9648)
3073                         rdev->config.evergreen.max_simds = 3;
3074                 else if ((rdev->pdev->device == 0x9647) ||
3075                          (rdev->pdev->device == 0x964a))
3076                         rdev->config.evergreen.max_simds = 4;
3077                 else
3078                         rdev->config.evergreen.max_simds = 5;
3079                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3080                 rdev->config.evergreen.max_gprs = 256;
3081                 rdev->config.evergreen.max_threads = 248;
3082                 rdev->config.evergreen.max_gs_threads = 32;
3083                 rdev->config.evergreen.max_stack_entries = 256;
3084                 rdev->config.evergreen.sx_num_of_sets = 4;
3085                 rdev->config.evergreen.sx_max_export_size = 256;
3086                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3087                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3088                 rdev->config.evergreen.max_hw_contexts = 8;
3089                 rdev->config.evergreen.sq_num_cf_insts = 2;
3090
3091                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3092                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3093                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3094                 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3095                 break;
3096         case CHIP_SUMO2:
3097                 rdev->config.evergreen.num_ses = 1;
3098                 rdev->config.evergreen.max_pipes = 4;
3099                 rdev->config.evergreen.max_tile_pipes = 4;
3100                 rdev->config.evergreen.max_simds = 2;
3101                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3102                 rdev->config.evergreen.max_gprs = 256;
3103                 rdev->config.evergreen.max_threads = 248;
3104                 rdev->config.evergreen.max_gs_threads = 32;
3105                 rdev->config.evergreen.max_stack_entries = 512;
3106                 rdev->config.evergreen.sx_num_of_sets = 4;
3107                 rdev->config.evergreen.sx_max_export_size = 256;
3108                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3109                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3110                 rdev->config.evergreen.max_hw_contexts = 8;
3111                 rdev->config.evergreen.sq_num_cf_insts = 2;
3112
3113                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3114                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3115                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3116                 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3117                 break;
3118         case CHIP_BARTS:
3119                 rdev->config.evergreen.num_ses = 2;
3120                 rdev->config.evergreen.max_pipes = 4;
3121                 rdev->config.evergreen.max_tile_pipes = 8;
3122                 rdev->config.evergreen.max_simds = 7;
3123                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3124                 rdev->config.evergreen.max_gprs = 256;
3125                 rdev->config.evergreen.max_threads = 248;
3126                 rdev->config.evergreen.max_gs_threads = 32;
3127                 rdev->config.evergreen.max_stack_entries = 512;
3128                 rdev->config.evergreen.sx_num_of_sets = 4;
3129                 rdev->config.evergreen.sx_max_export_size = 256;
3130                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3131                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3132                 rdev->config.evergreen.max_hw_contexts = 8;
3133                 rdev->config.evergreen.sq_num_cf_insts = 2;
3134
3135                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3136                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3137                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3138                 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3139                 break;
3140         case CHIP_TURKS:
3141                 rdev->config.evergreen.num_ses = 1;
3142                 rdev->config.evergreen.max_pipes = 4;
3143                 rdev->config.evergreen.max_tile_pipes = 4;
3144                 rdev->config.evergreen.max_simds = 6;
3145                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3146                 rdev->config.evergreen.max_gprs = 256;
3147                 rdev->config.evergreen.max_threads = 248;
3148                 rdev->config.evergreen.max_gs_threads = 32;
3149                 rdev->config.evergreen.max_stack_entries = 256;
3150                 rdev->config.evergreen.sx_num_of_sets = 4;
3151                 rdev->config.evergreen.sx_max_export_size = 256;
3152                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3153                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3154                 rdev->config.evergreen.max_hw_contexts = 8;
3155                 rdev->config.evergreen.sq_num_cf_insts = 2;
3156
3157                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3158                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3159                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3160                 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3161                 break;
3162         case CHIP_CAICOS:
3163                 rdev->config.evergreen.num_ses = 1;
3164                 rdev->config.evergreen.max_pipes = 2;
3165                 rdev->config.evergreen.max_tile_pipes = 2;
3166                 rdev->config.evergreen.max_simds = 2;
3167                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3168                 rdev->config.evergreen.max_gprs = 256;
3169                 rdev->config.evergreen.max_threads = 192;
3170                 rdev->config.evergreen.max_gs_threads = 16;
3171                 rdev->config.evergreen.max_stack_entries = 256;
3172                 rdev->config.evergreen.sx_num_of_sets = 4;
3173                 rdev->config.evergreen.sx_max_export_size = 128;
3174                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3175                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3176                 rdev->config.evergreen.max_hw_contexts = 4;
3177                 rdev->config.evergreen.sq_num_cf_insts = 1;
3178
3179                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3180                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3181                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3182                 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3183                 break;
3184         }
3185
3186         /* Initialize HDP */
3187         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3188                 WREG32((0x2c14 + j), 0x00000000);
3189                 WREG32((0x2c18 + j), 0x00000000);
3190                 WREG32((0x2c1c + j), 0x00000000);
3191                 WREG32((0x2c20 + j), 0x00000000);
3192                 WREG32((0x2c24 + j), 0x00000000);
3193         }
3194
3195         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3196
3197         evergreen_fix_pci_max_read_req_size(rdev);
3198
3199         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3200         if ((rdev->family == CHIP_PALM) ||
3201             (rdev->family == CHIP_SUMO) ||
3202             (rdev->family == CHIP_SUMO2))
3203                 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3204         else
3205                 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3206
3207         /* setup tiling info dword.  gb_addr_config is not adequate since it does
3208          * not have bank info, so create a custom tiling dword.
3209          * bits 3:0   num_pipes
3210          * bits 7:4   num_banks
3211          * bits 11:8  group_size
3212          * bits 15:12 row_size
3213          */
3214         rdev->config.evergreen.tile_config = 0;
3215         switch (rdev->config.evergreen.max_tile_pipes) {
3216         case 1:
3217         default:
3218                 rdev->config.evergreen.tile_config |= (0 << 0);
3219                 break;
3220         case 2:
3221                 rdev->config.evergreen.tile_config |= (1 << 0);
3222                 break;
3223         case 4:
3224                 rdev->config.evergreen.tile_config |= (2 << 0);
3225                 break;
3226         case 8:
3227                 rdev->config.evergreen.tile_config |= (3 << 0);
3228                 break;
3229         }
3230         /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3231         if (rdev->flags & RADEON_IS_IGP)
3232                 rdev->config.evergreen.tile_config |= 1 << 4;
3233         else {
3234                 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3235                 case 0: /* four banks */
3236                         rdev->config.evergreen.tile_config |= 0 << 4;
3237                         break;
3238                 case 1: /* eight banks */
3239                         rdev->config.evergreen.tile_config |= 1 << 4;
3240                         break;
3241                 case 2: /* sixteen banks */
3242                 default:
3243                         rdev->config.evergreen.tile_config |= 2 << 4;
3244                         break;
3245                 }
3246         }
3247         rdev->config.evergreen.tile_config |= 0 << 8;
3248         rdev->config.evergreen.tile_config |=
3249                 ((gb_addr_config & 0x30000000) >> 28) << 12;
3250
3251         num_shader_engines = (gb_addr_config & NUM_SHADER_ENGINES(3) >> 12) + 1;
3252
3253         if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3254                 u32 efuse_straps_4;
3255                 u32 efuse_straps_3;
3256
3257                 efuse_straps_4 = RREG32_RCU(0x204);
3258                 efuse_straps_3 = RREG32_RCU(0x203);
3259                 tmp = (((efuse_straps_4 & 0xf) << 4) |
3260                       ((efuse_straps_3 & 0xf0000000) >> 28));
3261         } else {
3262                 tmp = 0;
3263                 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3264                         u32 rb_disable_bitmap;
3265
3266                         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3267                         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3268                         rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3269                         tmp <<= 4;
3270                         tmp |= rb_disable_bitmap;
3271                 }
3272         }
3273         /* enabled rb are just the one not disabled :) */
3274         disabled_rb_mask = tmp;
3275         tmp = 0;
3276         for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3277                 tmp |= (1 << i);
3278         /* if all the backends are disabled, fix it up here */
3279         if ((disabled_rb_mask & tmp) == tmp) {
3280                 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3281                         disabled_rb_mask &= ~(1 << i);
3282         }
3283
3284         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3285         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3286
3287         WREG32(GB_ADDR_CONFIG, gb_addr_config);
3288         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3289         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3290         WREG32(DMA_TILING_CONFIG, gb_addr_config);
3291         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3292         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3293         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3294
3295         if ((rdev->config.evergreen.max_backends == 1) &&
3296             (rdev->flags & RADEON_IS_IGP)) {
3297                 if ((disabled_rb_mask & 3) == 1) {
3298                         /* RB0 disabled, RB1 enabled */
3299                         tmp = 0x11111111;
3300                 } else {
3301                         /* RB1 disabled, RB0 enabled */
3302                         tmp = 0x00000000;
3303                 }
3304         } else {
3305                 tmp = gb_addr_config & NUM_PIPES_MASK;
3306                 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3307                                                 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3308         }
3309         WREG32(GB_BACKEND_MAP, tmp);
3310
3311         WREG32(CGTS_SYS_TCC_DISABLE, 0);
3312         WREG32(CGTS_TCC_DISABLE, 0);
3313         WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3314         WREG32(CGTS_USER_TCC_DISABLE, 0);
3315
3316         /* set HW defaults for 3D engine */
3317         WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3318                                      ROQ_IB2_START(0x2b)));
3319
3320         WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3321
3322         WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3323                              SYNC_GRADIENT |
3324                              SYNC_WALKER |
3325                              SYNC_ALIGNER));
3326
3327         sx_debug_1 = RREG32(SX_DEBUG_1);
3328         sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3329         WREG32(SX_DEBUG_1, sx_debug_1);
3330
3331
3332         smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3333         smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3334         smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3335         WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3336
3337         if (rdev->family <= CHIP_SUMO2)
3338                 WREG32(SMX_SAR_CTL0, 0x00010000);
3339
3340         WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3341                                         POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3342                                         SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3343
3344         WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3345                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3346                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3347
3348         WREG32(VGT_NUM_INSTANCES, 1);
3349         WREG32(SPI_CONFIG_CNTL, 0);
3350         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3351         WREG32(CP_PERFMON_CNTL, 0);
3352
3353         WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3354                                   FETCH_FIFO_HIWATER(0x4) |
3355                                   DONE_FIFO_HIWATER(0xe0) |
3356                                   ALU_UPDATE_FIFO_HIWATER(0x8)));
3357
3358         sq_config = RREG32(SQ_CONFIG);
3359         sq_config &= ~(PS_PRIO(3) |
3360                        VS_PRIO(3) |
3361                        GS_PRIO(3) |
3362                        ES_PRIO(3));
3363         sq_config |= (VC_ENABLE |
3364                       EXPORT_SRC_C |
3365                       PS_PRIO(0) |
3366                       VS_PRIO(1) |
3367                       GS_PRIO(2) |
3368                       ES_PRIO(3));
3369
3370         switch (rdev->family) {
3371         case CHIP_CEDAR:
3372         case CHIP_PALM:
3373         case CHIP_SUMO:
3374         case CHIP_SUMO2:
3375         case CHIP_CAICOS:
3376                 /* no vertex cache */
3377                 sq_config &= ~VC_ENABLE;
3378                 break;
3379         default:
3380                 break;
3381         }
3382
3383         sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3384
3385         sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3386         sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3387         sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3388         sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3389         sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3390         sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3391         sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3392
3393         switch (rdev->family) {
3394         case CHIP_CEDAR:
3395         case CHIP_PALM:
3396         case CHIP_SUMO:
3397         case CHIP_SUMO2:
3398                 ps_thread_count = 96;
3399                 break;
3400         default:
3401                 ps_thread_count = 128;
3402                 break;
3403         }
3404
3405         sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3406         sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3407         sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3408         sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3409         sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3410         sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3411
3412         sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3413         sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3414         sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3415         sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3416         sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3417         sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3418
3419         WREG32(SQ_CONFIG, sq_config);
3420         WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3421         WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3422         WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3423         WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3424         WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3425         WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3426         WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3427         WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3428         WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3429         WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3430
3431         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3432                                           FORCE_EOV_MAX_REZ_CNT(255)));
3433
3434         switch (rdev->family) {
3435         case CHIP_CEDAR:
3436         case CHIP_PALM:
3437         case CHIP_SUMO:
3438         case CHIP_SUMO2:
3439         case CHIP_CAICOS:
3440                 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3441                 break;
3442         default:
3443                 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3444                 break;
3445         }
3446         vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3447         WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3448
3449         WREG32(VGT_GS_VERTEX_REUSE, 16);
3450         WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3451         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3452
3453         WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3454         WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3455
3456         WREG32(CB_PERF_CTR0_SEL_0, 0);
3457         WREG32(CB_PERF_CTR0_SEL_1, 0);
3458         WREG32(CB_PERF_CTR1_SEL_0, 0);
3459         WREG32(CB_PERF_CTR1_SEL_1, 0);
3460         WREG32(CB_PERF_CTR2_SEL_0, 0);
3461         WREG32(CB_PERF_CTR2_SEL_1, 0);
3462         WREG32(CB_PERF_CTR3_SEL_0, 0);
3463         WREG32(CB_PERF_CTR3_SEL_1, 0);
3464
3465         /* clear render buffer base addresses */
3466         WREG32(CB_COLOR0_BASE, 0);
3467         WREG32(CB_COLOR1_BASE, 0);
3468         WREG32(CB_COLOR2_BASE, 0);
3469         WREG32(CB_COLOR3_BASE, 0);
3470         WREG32(CB_COLOR4_BASE, 0);
3471         WREG32(CB_COLOR5_BASE, 0);
3472         WREG32(CB_COLOR6_BASE, 0);
3473         WREG32(CB_COLOR7_BASE, 0);
3474         WREG32(CB_COLOR8_BASE, 0);
3475         WREG32(CB_COLOR9_BASE, 0);
3476         WREG32(CB_COLOR10_BASE, 0);
3477         WREG32(CB_COLOR11_BASE, 0);
3478
3479         /* set the shader const cache sizes to 0 */
3480         for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3481                 WREG32(i, 0);
3482         for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3483                 WREG32(i, 0);
3484
3485         tmp = RREG32(HDP_MISC_CNTL);
3486         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3487         WREG32(HDP_MISC_CNTL, tmp);
3488
3489         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3490         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3491
3492         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3493
3494         udelay(50);
3495
3496 }
3497
3498 int evergreen_mc_init(struct radeon_device *rdev)
3499 {
3500         u32 tmp;
3501         int chansize, numchan;
3502
3503         /* Get VRAM informations */
3504         rdev->mc.vram_is_ddr = true;
3505         if ((rdev->family == CHIP_PALM) ||
3506             (rdev->family == CHIP_SUMO) ||
3507             (rdev->family == CHIP_SUMO2))
3508                 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3509         else
3510                 tmp = RREG32(MC_ARB_RAMCFG);
3511         if (tmp & CHANSIZE_OVERRIDE) {
3512                 chansize = 16;
3513         } else if (tmp & CHANSIZE_MASK) {
3514                 chansize = 64;
3515         } else {
3516                 chansize = 32;
3517         }
3518         tmp = RREG32(MC_SHARED_CHMAP);
3519         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3520         case 0:
3521         default:
3522                 numchan = 1;
3523                 break;
3524         case 1:
3525                 numchan = 2;
3526                 break;
3527         case 2:
3528                 numchan = 4;
3529                 break;
3530         case 3:
3531                 numchan = 8;
3532                 break;
3533         }
3534         rdev->mc.vram_width = numchan * chansize;
3535         /* Could aper size report 0 ? */
3536         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3537         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3538         /* Setup GPU memory space */
3539         if ((rdev->family == CHIP_PALM) ||
3540             (rdev->family == CHIP_SUMO) ||
3541             (rdev->family == CHIP_SUMO2)) {
3542                 /* size in bytes on fusion */
3543                 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3544                 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3545         } else {
3546                 /* size in MB on evergreen/cayman/tn */
3547                 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3548                 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3549         }
3550         rdev->mc.visible_vram_size = rdev->mc.aper_size;
3551         r700_vram_gtt_location(rdev, &rdev->mc);
3552         radeon_update_bandwidth_info(rdev);
3553
3554         return 0;
3555 }
3556
3557 void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3558 {
3559         dev_info(rdev->dev, "  GRBM_STATUS               = 0x%08X\n",
3560                 RREG32(GRBM_STATUS));
3561         dev_info(rdev->dev, "  GRBM_STATUS_SE0           = 0x%08X\n",
3562                 RREG32(GRBM_STATUS_SE0));
3563         dev_info(rdev->dev, "  GRBM_STATUS_SE1           = 0x%08X\n",
3564                 RREG32(GRBM_STATUS_SE1));
3565         dev_info(rdev->dev, "  SRBM_STATUS               = 0x%08X\n",
3566                 RREG32(SRBM_STATUS));
3567         dev_info(rdev->dev, "  SRBM_STATUS2              = 0x%08X\n",
3568                 RREG32(SRBM_STATUS2));
3569         dev_info(rdev->dev, "  R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3570                 RREG32(CP_STALLED_STAT1));
3571         dev_info(rdev->dev, "  R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3572                 RREG32(CP_STALLED_STAT2));
3573         dev_info(rdev->dev, "  R_00867C_CP_BUSY_STAT     = 0x%08X\n",
3574                 RREG32(CP_BUSY_STAT));
3575         dev_info(rdev->dev, "  R_008680_CP_STAT          = 0x%08X\n",
3576                 RREG32(CP_STAT));
3577         dev_info(rdev->dev, "  R_00D034_DMA_STATUS_REG   = 0x%08X\n",
3578                 RREG32(DMA_STATUS_REG));
3579         if (rdev->family >= CHIP_CAYMAN) {
3580                 dev_info(rdev->dev, "  R_00D834_DMA_STATUS_REG   = 0x%08X\n",
3581                          RREG32(DMA_STATUS_REG + 0x800));
3582         }
3583 }
3584
3585 bool evergreen_is_display_hung(struct radeon_device *rdev)
3586 {
3587         u32 crtc_hung = 0;
3588         u32 crtc_status[6];
3589         u32 i, j, tmp;
3590
3591         for (i = 0; i < rdev->num_crtc; i++) {
3592                 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3593                         crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3594                         crtc_hung |= (1 << i);
3595                 }
3596         }
3597
3598         for (j = 0; j < 10; j++) {
3599                 for (i = 0; i < rdev->num_crtc; i++) {
3600                         if (crtc_hung & (1 << i)) {
3601                                 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3602                                 if (tmp != crtc_status[i])
3603                                         crtc_hung &= ~(1 << i);
3604                         }
3605                 }
3606                 if (crtc_hung == 0)
3607                         return false;
3608                 udelay(100);
3609         }
3610
3611         return true;
3612 }
3613
3614 static u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3615 {
3616         u32 reset_mask = 0;
3617         u32 tmp;
3618
3619         /* GRBM_STATUS */
3620         tmp = RREG32(GRBM_STATUS);
3621         if (tmp & (PA_BUSY | SC_BUSY |
3622                    SH_BUSY | SX_BUSY |
3623                    TA_BUSY | VGT_BUSY |
3624                    DB_BUSY | CB_BUSY |
3625                    SPI_BUSY | VGT_BUSY_NO_DMA))
3626                 reset_mask |= RADEON_RESET_GFX;
3627
3628         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3629                    CP_BUSY | CP_COHERENCY_BUSY))
3630                 reset_mask |= RADEON_RESET_CP;
3631
3632         if (tmp & GRBM_EE_BUSY)
3633                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3634
3635         /* DMA_STATUS_REG */
3636         tmp = RREG32(DMA_STATUS_REG);
3637         if (!(tmp & DMA_IDLE))
3638                 reset_mask |= RADEON_RESET_DMA;
3639
3640         /* SRBM_STATUS2 */
3641         tmp = RREG32(SRBM_STATUS2);
3642         if (tmp & DMA_BUSY)
3643                 reset_mask |= RADEON_RESET_DMA;
3644
3645         /* SRBM_STATUS */
3646         tmp = RREG32(SRBM_STATUS);
3647         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3648                 reset_mask |= RADEON_RESET_RLC;
3649
3650         if (tmp & IH_BUSY)
3651                 reset_mask |= RADEON_RESET_IH;
3652
3653         if (tmp & SEM_BUSY)
3654                 reset_mask |= RADEON_RESET_SEM;
3655
3656         if (tmp & GRBM_RQ_PENDING)
3657                 reset_mask |= RADEON_RESET_GRBM;
3658
3659         if (tmp & VMC_BUSY)
3660                 reset_mask |= RADEON_RESET_VMC;
3661
3662         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3663                    MCC_BUSY | MCD_BUSY))
3664                 reset_mask |= RADEON_RESET_MC;
3665
3666         if (evergreen_is_display_hung(rdev))
3667                 reset_mask |= RADEON_RESET_DISPLAY;
3668
3669         /* VM_L2_STATUS */
3670         tmp = RREG32(VM_L2_STATUS);
3671         if (tmp & L2_BUSY)
3672                 reset_mask |= RADEON_RESET_VMC;
3673
3674         /* Skip MC reset as it's mostly likely not hung, just busy */
3675         if (reset_mask & RADEON_RESET_MC) {
3676                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3677                 reset_mask &= ~RADEON_RESET_MC;
3678         }
3679
3680         return reset_mask;
3681 }
3682
3683 static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3684 {
3685         struct evergreen_mc_save save;
3686         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3687         u32 tmp;
3688
3689         if (reset_mask == 0)
3690                 return;
3691
3692         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3693
3694         evergreen_print_gpu_status_regs(rdev);
3695
3696         /* Disable CP parsing/prefetching */
3697         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3698
3699         if (reset_mask & RADEON_RESET_DMA) {
3700                 /* Disable DMA */
3701                 tmp = RREG32(DMA_RB_CNTL);
3702                 tmp &= ~DMA_RB_ENABLE;
3703                 WREG32(DMA_RB_CNTL, tmp);
3704         }
3705
3706         udelay(50);
3707
3708         evergreen_mc_stop(rdev, &save);
3709         if (evergreen_mc_wait_for_idle(rdev)) {
3710                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3711         }
3712
3713         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3714                 grbm_soft_reset |= SOFT_RESET_DB |
3715                         SOFT_RESET_CB |
3716                         SOFT_RESET_PA |
3717                         SOFT_RESET_SC |
3718                         SOFT_RESET_SPI |
3719                         SOFT_RESET_SX |
3720                         SOFT_RESET_SH |
3721                         SOFT_RESET_TC |
3722                         SOFT_RESET_TA |
3723                         SOFT_RESET_VC |
3724                         SOFT_RESET_VGT;
3725         }
3726
3727         if (reset_mask & RADEON_RESET_CP) {
3728                 grbm_soft_reset |= SOFT_RESET_CP |
3729                         SOFT_RESET_VGT;
3730
3731                 srbm_soft_reset |= SOFT_RESET_GRBM;
3732         }
3733
3734         if (reset_mask & RADEON_RESET_DMA)
3735                 srbm_soft_reset |= SOFT_RESET_DMA;
3736
3737         if (reset_mask & RADEON_RESET_DISPLAY)
3738                 srbm_soft_reset |= SOFT_RESET_DC;
3739
3740         if (reset_mask & RADEON_RESET_RLC)
3741                 srbm_soft_reset |= SOFT_RESET_RLC;
3742
3743         if (reset_mask & RADEON_RESET_SEM)
3744                 srbm_soft_reset |= SOFT_RESET_SEM;
3745
3746         if (reset_mask & RADEON_RESET_IH)
3747                 srbm_soft_reset |= SOFT_RESET_IH;
3748
3749         if (reset_mask & RADEON_RESET_GRBM)
3750                 srbm_soft_reset |= SOFT_RESET_GRBM;
3751
3752         if (reset_mask & RADEON_RESET_VMC)
3753                 srbm_soft_reset |= SOFT_RESET_VMC;
3754
3755         if (!(rdev->flags & RADEON_IS_IGP)) {
3756                 if (reset_mask & RADEON_RESET_MC)
3757                         srbm_soft_reset |= SOFT_RESET_MC;
3758         }
3759
3760         if (grbm_soft_reset) {
3761                 tmp = RREG32(GRBM_SOFT_RESET);
3762                 tmp |= grbm_soft_reset;
3763                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3764                 WREG32(GRBM_SOFT_RESET, tmp);
3765                 tmp = RREG32(GRBM_SOFT_RESET);
3766
3767                 udelay(50);
3768
3769                 tmp &= ~grbm_soft_reset;
3770                 WREG32(GRBM_SOFT_RESET, tmp);
3771                 tmp = RREG32(GRBM_SOFT_RESET);
3772         }
3773
3774         if (srbm_soft_reset) {
3775                 tmp = RREG32(SRBM_SOFT_RESET);
3776                 tmp |= srbm_soft_reset;
3777                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3778                 WREG32(SRBM_SOFT_RESET, tmp);
3779                 tmp = RREG32(SRBM_SOFT_RESET);
3780
3781                 udelay(50);
3782
3783                 tmp &= ~srbm_soft_reset;
3784                 WREG32(SRBM_SOFT_RESET, tmp);
3785                 tmp = RREG32(SRBM_SOFT_RESET);
3786         }
3787
3788         /* Wait a little for things to settle down */
3789         udelay(50);
3790
3791         evergreen_mc_resume(rdev, &save);
3792         udelay(50);
3793
3794         evergreen_print_gpu_status_regs(rdev);
3795 }
3796
3797 int evergreen_asic_reset(struct radeon_device *rdev)
3798 {
3799         u32 reset_mask;
3800
3801         reset_mask = evergreen_gpu_check_soft_reset(rdev);
3802
3803         if (reset_mask)
3804                 r600_set_bios_scratch_engine_hung(rdev, true);
3805
3806         evergreen_gpu_soft_reset(rdev, reset_mask);
3807
3808         reset_mask = evergreen_gpu_check_soft_reset(rdev);
3809
3810         if (!reset_mask)
3811                 r600_set_bios_scratch_engine_hung(rdev, false);
3812
3813         return 0;
3814 }
3815
3816 /**
3817  * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
3818  *
3819  * @rdev: radeon_device pointer
3820  * @ring: radeon_ring structure holding ring information
3821  *
3822  * Check if the GFX engine is locked up.
3823  * Returns true if the engine appears to be locked up, false if not.
3824  */
3825 bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3826 {
3827         u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3828
3829         if (!(reset_mask & (RADEON_RESET_GFX |
3830                             RADEON_RESET_COMPUTE |
3831                             RADEON_RESET_CP))) {
3832                 radeon_ring_lockup_update(ring);
3833                 return false;
3834         }
3835         /* force CP activities */
3836         radeon_ring_force_activity(rdev, ring);
3837         return radeon_ring_test_lockup(rdev, ring);
3838 }
3839
3840 /**
3841  * evergreen_dma_is_lockup - Check if the DMA engine is locked up
3842  *
3843  * @rdev: radeon_device pointer
3844  * @ring: radeon_ring structure holding ring information
3845  *
3846  * Check if the async DMA engine is locked up.
3847  * Returns true if the engine appears to be locked up, false if not.
3848  */
3849 bool evergreen_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3850 {
3851         u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3852
3853         if (!(reset_mask & RADEON_RESET_DMA)) {
3854                 radeon_ring_lockup_update(ring);
3855                 return false;
3856         }
3857         /* force ring activities */
3858         radeon_ring_force_activity(rdev, ring);
3859         return radeon_ring_test_lockup(rdev, ring);
3860 }
3861
3862 /*
3863  * RLC
3864  */
3865 #define RLC_SAVE_RESTORE_LIST_END_MARKER    0x00000000
3866 #define RLC_CLEAR_STATE_END_MARKER          0x00000001
3867
3868 void sumo_rlc_fini(struct radeon_device *rdev)
3869 {
3870         int r;
3871
3872         /* save restore block */
3873         if (rdev->rlc.save_restore_obj) {
3874                 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3875                 if (unlikely(r != 0))
3876                         dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
3877                 radeon_bo_unpin(rdev->rlc.save_restore_obj);
3878                 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3879
3880                 radeon_bo_unref(&rdev->rlc.save_restore_obj);
3881                 rdev->rlc.save_restore_obj = NULL;
3882         }
3883
3884         /* clear state block */
3885         if (rdev->rlc.clear_state_obj) {
3886                 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
3887                 if (unlikely(r != 0))
3888                         dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
3889                 radeon_bo_unpin(rdev->rlc.clear_state_obj);
3890                 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
3891
3892                 radeon_bo_unref(&rdev->rlc.clear_state_obj);
3893                 rdev->rlc.clear_state_obj = NULL;
3894         }
3895 }
3896
3897 int sumo_rlc_init(struct radeon_device *rdev)
3898 {
3899         u32 *src_ptr;
3900         volatile u32 *dst_ptr;
3901         u32 dws, data, i, j, k, reg_num;
3902         u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index;
3903         u64 reg_list_mc_addr;
3904         struct cs_section_def *cs_data;
3905         int r;
3906
3907         src_ptr = rdev->rlc.reg_list;
3908         dws = rdev->rlc.reg_list_size;
3909         cs_data = rdev->rlc.cs_data;
3910
3911         /* save restore block */
3912         if (rdev->rlc.save_restore_obj == NULL) {
3913                 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
3914                                      RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.save_restore_obj);
3915                 if (r) {
3916                         dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
3917                         return r;
3918                 }
3919         }
3920
3921         r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3922         if (unlikely(r != 0)) {
3923                 sumo_rlc_fini(rdev);
3924                 return r;
3925         }
3926         r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
3927                           &rdev->rlc.save_restore_gpu_addr);
3928         if (r) {
3929                 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3930                 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
3931                 sumo_rlc_fini(rdev);
3932                 return r;
3933         }
3934         r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
3935         if (r) {
3936                 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
3937                 sumo_rlc_fini(rdev);
3938                 return r;
3939         }
3940         /* write the sr buffer */
3941         dst_ptr = rdev->rlc.sr_ptr;
3942         /* format:
3943          * dw0: (reg2 << 16) | reg1
3944          * dw1: reg1 save space
3945          * dw2: reg2 save space
3946          */
3947         for (i = 0; i < dws; i++) {
3948                 data = src_ptr[i] >> 2;
3949                 i++;
3950                 if (i < dws)
3951                         data |= (src_ptr[i] >> 2) << 16;
3952                 j = (((i - 1) * 3) / 2);
3953                 dst_ptr[j] = data;
3954         }
3955         j = ((i * 3) / 2);
3956         dst_ptr[j] = RLC_SAVE_RESTORE_LIST_END_MARKER;
3957
3958         radeon_bo_kunmap(rdev->rlc.save_restore_obj);
3959         radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3960
3961         /* clear state block */
3962         reg_list_num = 0;
3963         dws = 0;
3964         for (i = 0; cs_data[i].section != NULL; i++) {
3965                 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
3966                         reg_list_num++;
3967                         dws += cs_data[i].section[j].reg_count;
3968                 }
3969         }
3970         reg_list_blk_index = (3 * reg_list_num + 2);
3971         dws += reg_list_blk_index;
3972
3973         if (rdev->rlc.clear_state_obj == NULL) {
3974                 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
3975                                      RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.clear_state_obj);
3976                 if (r) {
3977                         dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
3978                         sumo_rlc_fini(rdev);
3979                         return r;
3980                 }
3981         }
3982         r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
3983         if (unlikely(r != 0)) {
3984                 sumo_rlc_fini(rdev);
3985                 return r;
3986         }
3987         r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
3988                           &rdev->rlc.clear_state_gpu_addr);
3989         if (r) {
3990
3991                 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
3992                 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
3993                 sumo_rlc_fini(rdev);
3994                 return r;
3995         }
3996         r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
3997         if (r) {
3998                 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
3999                 sumo_rlc_fini(rdev);
4000                 return r;
4001         }
4002         /* set up the cs buffer */
4003         dst_ptr = rdev->rlc.cs_ptr;
4004         reg_list_hdr_blk_index = 0;
4005         reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4006         data = upper_32_bits(reg_list_mc_addr);
4007         dst_ptr[reg_list_hdr_blk_index] = data;
4008         reg_list_hdr_blk_index++;
4009         for (i = 0; cs_data[i].section != NULL; i++) {
4010                 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4011                         reg_num = cs_data[i].section[j].reg_count;
4012                         data = reg_list_mc_addr & 0xffffffff;
4013                         dst_ptr[reg_list_hdr_blk_index] = data;
4014                         reg_list_hdr_blk_index++;
4015
4016                         data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4017                         dst_ptr[reg_list_hdr_blk_index] = data;
4018                         reg_list_hdr_blk_index++;
4019
4020                         data = 0x08000000 | (reg_num * 4);
4021                         dst_ptr[reg_list_hdr_blk_index] = data;
4022                         reg_list_hdr_blk_index++;
4023
4024                         for (k = 0; k < reg_num; k++) {
4025                                 data = cs_data[i].section[j].extent[k];
4026                                 dst_ptr[reg_list_blk_index + k] = data;
4027                         }
4028                         reg_list_mc_addr += reg_num * 4;
4029                         reg_list_blk_index += reg_num;
4030                 }
4031         }
4032         dst_ptr[reg_list_hdr_blk_index] = RLC_CLEAR_STATE_END_MARKER;
4033
4034         radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4035         radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4036
4037         return 0;
4038 }
4039
4040 static void evergreen_rlc_start(struct radeon_device *rdev)
4041 {
4042         u32 mask = RLC_ENABLE;
4043
4044         if (rdev->flags & RADEON_IS_IGP) {
4045                 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4046         }
4047
4048         WREG32(RLC_CNTL, mask);
4049 }
4050
4051 int evergreen_rlc_resume(struct radeon_device *rdev)
4052 {
4053         u32 i;
4054         const __be32 *fw_data;
4055
4056         if (!rdev->rlc_fw)
4057                 return -EINVAL;
4058
4059         r600_rlc_stop(rdev);
4060
4061         WREG32(RLC_HB_CNTL, 0);
4062
4063         if (rdev->flags & RADEON_IS_IGP) {
4064                 if (rdev->family == CHIP_ARUBA) {
4065                         u32 always_on_bitmap =
4066                                 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4067                         /* find out the number of active simds */
4068                         u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4069                         tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4070                         tmp = hweight32(~tmp);
4071                         if (tmp == rdev->config.cayman.max_simds_per_se) {
4072                                 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4073                                 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4074                                 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4075                                 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4076                                 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4077                         }
4078                 } else {
4079                         WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4080                         WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4081                 }
4082                 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4083                 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4084         } else {
4085                 WREG32(RLC_HB_BASE, 0);
4086                 WREG32(RLC_HB_RPTR, 0);
4087                 WREG32(RLC_HB_WPTR, 0);
4088                 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4089                 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4090         }
4091         WREG32(RLC_MC_CNTL, 0);
4092         WREG32(RLC_UCODE_CNTL, 0);
4093
4094         fw_data = (const __be32 *)rdev->rlc_fw->data;
4095         if (rdev->family >= CHIP_ARUBA) {
4096                 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4097                         WREG32(RLC_UCODE_ADDR, i);
4098                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4099                 }
4100         } else if (rdev->family >= CHIP_CAYMAN) {
4101                 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4102                         WREG32(RLC_UCODE_ADDR, i);
4103                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4104                 }
4105         } else {
4106                 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4107                         WREG32(RLC_UCODE_ADDR, i);
4108                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4109                 }
4110         }
4111         WREG32(RLC_UCODE_ADDR, 0);
4112
4113         evergreen_rlc_start(rdev);
4114
4115         return 0;
4116 }
4117
4118 /* Interrupts */
4119
4120 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4121 {
4122         if (crtc >= rdev->num_crtc)
4123                 return 0;
4124         else
4125                 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4126 }
4127
4128 void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4129 {
4130         u32 tmp;
4131
4132         if (rdev->family >= CHIP_CAYMAN) {
4133                 cayman_cp_int_cntl_setup(rdev, 0,
4134                                          CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4135                 cayman_cp_int_cntl_setup(rdev, 1, 0);
4136                 cayman_cp_int_cntl_setup(rdev, 2, 0);
4137                 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4138                 WREG32(CAYMAN_DMA1_CNTL, tmp);
4139         } else
4140                 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4141         tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4142         WREG32(DMA_CNTL, tmp);
4143         WREG32(GRBM_INT_CNTL, 0);
4144         WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4145         WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4146         if (rdev->num_crtc >= 4) {
4147                 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4148                 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4149         }
4150         if (rdev->num_crtc >= 6) {
4151                 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4152                 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4153         }
4154
4155         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4156         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4157         if (rdev->num_crtc >= 4) {
4158                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4159                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4160         }
4161         if (rdev->num_crtc >= 6) {
4162                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4163                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4164         }
4165
4166         /* only one DAC on DCE6 */
4167         if (!ASIC_IS_DCE6(rdev))
4168                 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4169         WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4170
4171         tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4172         WREG32(DC_HPD1_INT_CONTROL, tmp);
4173         tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4174         WREG32(DC_HPD2_INT_CONTROL, tmp);
4175         tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4176         WREG32(DC_HPD3_INT_CONTROL, tmp);
4177         tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4178         WREG32(DC_HPD4_INT_CONTROL, tmp);
4179         tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4180         WREG32(DC_HPD5_INT_CONTROL, tmp);
4181         tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4182         WREG32(DC_HPD6_INT_CONTROL, tmp);
4183
4184 }
4185
4186 int evergreen_irq_set(struct radeon_device *rdev)
4187 {
4188         u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4189         u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4190         u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4191         u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
4192         u32 grbm_int_cntl = 0;
4193         u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
4194         u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
4195         u32 dma_cntl, dma_cntl1 = 0;
4196         u32 thermal_int = 0;
4197
4198         if (!rdev->irq.installed) {
4199                 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4200                 return -EINVAL;
4201         }
4202         /* don't enable anything if the ih is disabled */
4203         if (!rdev->ih.enabled) {
4204                 r600_disable_interrupts(rdev);
4205                 /* force the active interrupt state to all disabled */
4206                 evergreen_disable_interrupt_state(rdev);
4207                 return 0;
4208         }
4209
4210         hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4211         hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4212         hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
4213         hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
4214         hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
4215         hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
4216         if (rdev->family == CHIP_ARUBA)
4217                 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4218                         ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4219         else
4220                 thermal_int = RREG32(CG_THERMAL_INT) &
4221                         ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4222
4223         afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4224         afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4225         afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4226         afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4227         afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4228         afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4229
4230         dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4231
4232         if (rdev->family >= CHIP_CAYMAN) {
4233                 /* enable CP interrupts on all rings */
4234                 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4235                         DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4236                         cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4237                 }
4238                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4239                         DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4240                         cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4241                 }
4242                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4243                         DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4244                         cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4245                 }
4246         } else {
4247                 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4248                         DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4249                         cp_int_cntl |= RB_INT_ENABLE;
4250                         cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4251                 }
4252         }
4253
4254         if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4255                 DRM_DEBUG("r600_irq_set: sw int dma\n");
4256                 dma_cntl |= TRAP_ENABLE;
4257         }
4258
4259         if (rdev->family >= CHIP_CAYMAN) {
4260                 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4261                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4262                         DRM_DEBUG("r600_irq_set: sw int dma1\n");
4263                         dma_cntl1 |= TRAP_ENABLE;
4264                 }
4265         }
4266
4267         if (rdev->irq.dpm_thermal) {
4268                 DRM_DEBUG("dpm thermal\n");
4269                 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4270         }
4271
4272         if (rdev->irq.crtc_vblank_int[0] ||
4273             atomic_read(&rdev->irq.pflip[0])) {
4274                 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
4275                 crtc1 |= VBLANK_INT_MASK;
4276         }
4277         if (rdev->irq.crtc_vblank_int[1] ||
4278             atomic_read(&rdev->irq.pflip[1])) {
4279                 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
4280                 crtc2 |= VBLANK_INT_MASK;
4281         }
4282         if (rdev->irq.crtc_vblank_int[2] ||
4283             atomic_read(&rdev->irq.pflip[2])) {
4284                 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
4285                 crtc3 |= VBLANK_INT_MASK;
4286         }
4287         if (rdev->irq.crtc_vblank_int[3] ||
4288             atomic_read(&rdev->irq.pflip[3])) {
4289                 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
4290                 crtc4 |= VBLANK_INT_MASK;
4291         }
4292         if (rdev->irq.crtc_vblank_int[4] ||
4293             atomic_read(&rdev->irq.pflip[4])) {
4294                 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
4295                 crtc5 |= VBLANK_INT_MASK;
4296         }
4297         if (rdev->irq.crtc_vblank_int[5] ||
4298             atomic_read(&rdev->irq.pflip[5])) {
4299                 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
4300                 crtc6 |= VBLANK_INT_MASK;
4301         }
4302         if (rdev->irq.hpd[0]) {
4303                 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
4304                 hpd1 |= DC_HPDx_INT_EN;
4305         }
4306         if (rdev->irq.hpd[1]) {
4307                 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
4308                 hpd2 |= DC_HPDx_INT_EN;
4309         }
4310         if (rdev->irq.hpd[2]) {
4311                 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
4312                 hpd3 |= DC_HPDx_INT_EN;
4313         }
4314         if (rdev->irq.hpd[3]) {
4315                 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
4316                 hpd4 |= DC_HPDx_INT_EN;
4317         }
4318         if (rdev->irq.hpd[4]) {
4319                 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
4320                 hpd5 |= DC_HPDx_INT_EN;
4321         }
4322         if (rdev->irq.hpd[5]) {
4323                 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
4324                 hpd6 |= DC_HPDx_INT_EN;
4325         }
4326         if (rdev->irq.afmt[0]) {
4327                 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
4328                 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4329         }
4330         if (rdev->irq.afmt[1]) {
4331                 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
4332                 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4333         }
4334         if (rdev->irq.afmt[2]) {
4335                 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
4336                 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4337         }
4338         if (rdev->irq.afmt[3]) {
4339                 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
4340                 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4341         }
4342         if (rdev->irq.afmt[4]) {
4343                 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
4344                 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4345         }
4346         if (rdev->irq.afmt[5]) {
4347                 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
4348                 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4349         }
4350
4351         if (rdev->family >= CHIP_CAYMAN) {
4352                 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4353                 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4354                 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4355         } else
4356                 WREG32(CP_INT_CNTL, cp_int_cntl);
4357
4358         WREG32(DMA_CNTL, dma_cntl);
4359
4360         if (rdev->family >= CHIP_CAYMAN)
4361                 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4362
4363         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4364
4365         WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4366         WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
4367         if (rdev->num_crtc >= 4) {
4368                 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4369                 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
4370         }
4371         if (rdev->num_crtc >= 6) {
4372                 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4373                 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4374         }
4375
4376         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
4377         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
4378         if (rdev->num_crtc >= 4) {
4379                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
4380                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
4381         }
4382         if (rdev->num_crtc >= 6) {
4383                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
4384                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
4385         }
4386
4387         WREG32(DC_HPD1_INT_CONTROL, hpd1);
4388         WREG32(DC_HPD2_INT_CONTROL, hpd2);
4389         WREG32(DC_HPD3_INT_CONTROL, hpd3);
4390         WREG32(DC_HPD4_INT_CONTROL, hpd4);
4391         WREG32(DC_HPD5_INT_CONTROL, hpd5);
4392         WREG32(DC_HPD6_INT_CONTROL, hpd6);
4393         if (rdev->family == CHIP_ARUBA)
4394                 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4395         else
4396                 WREG32(CG_THERMAL_INT, thermal_int);
4397
4398         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
4399         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
4400         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
4401         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
4402         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
4403         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
4404
4405         return 0;
4406 }
4407
4408 static void evergreen_irq_ack(struct radeon_device *rdev)
4409 {
4410         u32 tmp;
4411
4412         rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4413         rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4414         rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4415         rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4416         rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4417         rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4418         rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4419         rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4420         if (rdev->num_crtc >= 4) {
4421                 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4422                 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4423         }
4424         if (rdev->num_crtc >= 6) {
4425                 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4426                 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4427         }
4428
4429         rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4430         rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4431         rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4432         rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4433         rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4434         rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4435
4436         if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4437                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4438         if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4439                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4440         if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
4441                 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
4442         if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
4443                 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
4444         if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
4445                 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
4446         if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
4447                 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4448
4449         if (rdev->num_crtc >= 4) {
4450                 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4451                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4452                 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4453                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4454                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4455                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4456                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4457                         WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4458                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4459                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4460                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4461                         WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4462         }
4463
4464         if (rdev->num_crtc >= 6) {
4465                 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4466                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4467                 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4468                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4469                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4470                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4471                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4472                         WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4473                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4474                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4475                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4476                         WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4477         }
4478
4479         if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4480                 tmp = RREG32(DC_HPD1_INT_CONTROL);
4481                 tmp |= DC_HPDx_INT_ACK;
4482                 WREG32(DC_HPD1_INT_CONTROL, tmp);
4483         }
4484         if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4485                 tmp = RREG32(DC_HPD2_INT_CONTROL);
4486                 tmp |= DC_HPDx_INT_ACK;
4487                 WREG32(DC_HPD2_INT_CONTROL, tmp);
4488         }
4489         if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4490                 tmp = RREG32(DC_HPD3_INT_CONTROL);
4491                 tmp |= DC_HPDx_INT_ACK;
4492                 WREG32(DC_HPD3_INT_CONTROL, tmp);
4493         }
4494         if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4495                 tmp = RREG32(DC_HPD4_INT_CONTROL);
4496                 tmp |= DC_HPDx_INT_ACK;
4497                 WREG32(DC_HPD4_INT_CONTROL, tmp);
4498         }
4499         if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4500                 tmp = RREG32(DC_HPD5_INT_CONTROL);
4501                 tmp |= DC_HPDx_INT_ACK;
4502                 WREG32(DC_HPD5_INT_CONTROL, tmp);
4503         }
4504         if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4505                 tmp = RREG32(DC_HPD5_INT_CONTROL);
4506                 tmp |= DC_HPDx_INT_ACK;
4507                 WREG32(DC_HPD6_INT_CONTROL, tmp);
4508         }
4509         if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4510                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
4511                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4512                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
4513         }
4514         if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4515                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
4516                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4517                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
4518         }
4519         if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4520                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
4521                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4522                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
4523         }
4524         if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4525                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
4526                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4527                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
4528         }
4529         if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4530                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
4531                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4532                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
4533         }
4534         if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4535                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
4536                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4537                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
4538         }
4539 }
4540
4541 static void evergreen_irq_disable(struct radeon_device *rdev)
4542 {
4543         r600_disable_interrupts(rdev);
4544         /* Wait and acknowledge irq */
4545         mdelay(1);
4546         evergreen_irq_ack(rdev);
4547         evergreen_disable_interrupt_state(rdev);
4548 }
4549
4550 void evergreen_irq_suspend(struct radeon_device *rdev)
4551 {
4552         evergreen_irq_disable(rdev);
4553         r600_rlc_stop(rdev);
4554 }
4555
4556 static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4557 {
4558         u32 wptr, tmp;
4559
4560         if (rdev->wb.enabled)
4561                 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4562         else
4563                 wptr = RREG32(IH_RB_WPTR);
4564
4565         if (wptr & RB_OVERFLOW) {
4566                 /* When a ring buffer overflow happen start parsing interrupt
4567                  * from the last not overwritten vector (wptr + 16). Hopefully
4568                  * this should allow us to catchup.
4569                  */
4570                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
4571                         wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
4572                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4573                 tmp = RREG32(IH_RB_CNTL);
4574                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4575                 WREG32(IH_RB_CNTL, tmp);
4576         }
4577         return (wptr & rdev->ih.ptr_mask);
4578 }
4579
4580 int evergreen_irq_process(struct radeon_device *rdev)
4581 {
4582         u32 wptr;
4583         u32 rptr;
4584         u32 src_id, src_data;
4585         u32 ring_index;
4586         bool queue_hotplug = false;
4587         bool queue_hdmi = false;
4588         bool queue_thermal = false;
4589
4590         if (!rdev->ih.enabled || rdev->shutdown)
4591                 return IRQ_NONE;
4592
4593         wptr = evergreen_get_ih_wptr(rdev);
4594
4595 restart_ih:
4596         /* is somebody else already processing irqs? */
4597         if (atomic_xchg(&rdev->ih.lock, 1))
4598                 return IRQ_NONE;
4599
4600         rptr = rdev->ih.rptr;
4601         DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4602
4603         /* Order reading of wptr vs. reading of IH ring data */
4604         rmb();
4605
4606         /* display interrupts */
4607         evergreen_irq_ack(rdev);
4608
4609         while (rptr != wptr) {
4610                 /* wptr/rptr are in bytes! */
4611                 ring_index = rptr / 4;
4612                 src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4613                 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4614
4615                 switch (src_id) {
4616                 case 1: /* D1 vblank/vline */
4617                         switch (src_data) {
4618                         case 0: /* D1 vblank */
4619                                 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
4620                                         if (rdev->irq.crtc_vblank_int[0]) {
4621                                                 drm_handle_vblank(rdev->ddev, 0);
4622                                                 rdev->pm.vblank_sync = true;
4623                                                 wake_up(&rdev->irq.vblank_queue);
4624                                         }
4625                                         if (atomic_read(&rdev->irq.pflip[0]))
4626                                                 radeon_crtc_handle_flip(rdev, 0);
4627                                         rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
4628                                         DRM_DEBUG("IH: D1 vblank\n");
4629                                 }
4630                                 break;
4631                         case 1: /* D1 vline */
4632                                 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
4633                                         rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
4634                                         DRM_DEBUG("IH: D1 vline\n");
4635                                 }
4636                                 break;
4637                         default:
4638                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4639                                 break;
4640                         }
4641                         break;
4642                 case 2: /* D2 vblank/vline */
4643                         switch (src_data) {
4644                         case 0: /* D2 vblank */
4645                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
4646                                         if (rdev->irq.crtc_vblank_int[1]) {
4647                                                 drm_handle_vblank(rdev->ddev, 1);
4648                                                 rdev->pm.vblank_sync = true;
4649                                                 wake_up(&rdev->irq.vblank_queue);
4650                                         }
4651                                         if (atomic_read(&rdev->irq.pflip[1]))
4652                                                 radeon_crtc_handle_flip(rdev, 1);
4653                                         rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
4654                                         DRM_DEBUG("IH: D2 vblank\n");
4655                                 }
4656                                 break;
4657                         case 1: /* D2 vline */
4658                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
4659                                         rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
4660                                         DRM_DEBUG("IH: D2 vline\n");
4661                                 }
4662                                 break;
4663                         default:
4664                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4665                                 break;
4666                         }
4667                         break;
4668                 case 3: /* D3 vblank/vline */
4669                         switch (src_data) {
4670                         case 0: /* D3 vblank */
4671                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
4672                                         if (rdev->irq.crtc_vblank_int[2]) {
4673                                                 drm_handle_vblank(rdev->ddev, 2);
4674                                                 rdev->pm.vblank_sync = true;
4675                                                 wake_up(&rdev->irq.vblank_queue);
4676                                         }
4677                                         if (atomic_read(&rdev->irq.pflip[2]))
4678                                                 radeon_crtc_handle_flip(rdev, 2);
4679                                         rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
4680                                         DRM_DEBUG("IH: D3 vblank\n");
4681                                 }
4682                                 break;
4683                         case 1: /* D3 vline */
4684                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
4685                                         rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
4686                                         DRM_DEBUG("IH: D3 vline\n");
4687                                 }
4688                                 break;
4689                         default:
4690                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4691                                 break;
4692                         }
4693                         break;
4694                 case 4: /* D4 vblank/vline */
4695                         switch (src_data) {
4696                         case 0: /* D4 vblank */
4697                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
4698                                         if (rdev->irq.crtc_vblank_int[3]) {
4699                                                 drm_handle_vblank(rdev->ddev, 3);
4700                                                 rdev->pm.vblank_sync = true;
4701                                                 wake_up(&rdev->irq.vblank_queue);
4702                                         }
4703                                         if (atomic_read(&rdev->irq.pflip[3]))
4704                                                 radeon_crtc_handle_flip(rdev, 3);
4705                                         rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
4706                                         DRM_DEBUG("IH: D4 vblank\n");
4707                                 }
4708                                 break;
4709                         case 1: /* D4 vline */
4710                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
4711                                         rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
4712                                         DRM_DEBUG("IH: D4 vline\n");
4713                                 }
4714                                 break;
4715                         default:
4716                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4717                                 break;
4718                         }
4719                         break;
4720                 case 5: /* D5 vblank/vline */
4721                         switch (src_data) {
4722                         case 0: /* D5 vblank */
4723                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
4724                                         if (rdev->irq.crtc_vblank_int[4]) {
4725                                                 drm_handle_vblank(rdev->ddev, 4);
4726                                                 rdev->pm.vblank_sync = true;
4727                                                 wake_up(&rdev->irq.vblank_queue);
4728                                         }
4729                                         if (atomic_read(&rdev->irq.pflip[4]))
4730                                                 radeon_crtc_handle_flip(rdev, 4);
4731                                         rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
4732                                         DRM_DEBUG("IH: D5 vblank\n");
4733                                 }
4734                                 break;
4735                         case 1: /* D5 vline */
4736                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
4737                                         rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
4738                                         DRM_DEBUG("IH: D5 vline\n");
4739                                 }
4740                                 break;
4741                         default:
4742                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4743                                 break;
4744                         }
4745                         break;
4746                 case 6: /* D6 vblank/vline */
4747                         switch (src_data) {
4748                         case 0: /* D6 vblank */
4749                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
4750                                         if (rdev->irq.crtc_vblank_int[5]) {
4751                                                 drm_handle_vblank(rdev->ddev, 5);
4752                                                 rdev->pm.vblank_sync = true;
4753                                                 wake_up(&rdev->irq.vblank_queue);
4754                                         }
4755                                         if (atomic_read(&rdev->irq.pflip[5]))
4756                                                 radeon_crtc_handle_flip(rdev, 5);
4757                                         rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
4758                                         DRM_DEBUG("IH: D6 vblank\n");
4759                                 }
4760                                 break;
4761                         case 1: /* D6 vline */
4762                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
4763                                         rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
4764                                         DRM_DEBUG("IH: D6 vline\n");
4765                                 }
4766                                 break;
4767                         default:
4768                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4769                                 break;
4770                         }
4771                         break;
4772                 case 42: /* HPD hotplug */
4773                         switch (src_data) {
4774                         case 0:
4775                                 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4776                                         rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
4777                                         queue_hotplug = true;
4778                                         DRM_DEBUG("IH: HPD1\n");
4779                                 }
4780                                 break;
4781                         case 1:
4782                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4783                                         rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
4784                                         queue_hotplug = true;
4785                                         DRM_DEBUG("IH: HPD2\n");
4786                                 }
4787                                 break;
4788                         case 2:
4789                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4790                                         rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
4791                                         queue_hotplug = true;
4792                                         DRM_DEBUG("IH: HPD3\n");
4793                                 }
4794                                 break;
4795                         case 3:
4796                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4797                                         rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
4798                                         queue_hotplug = true;
4799                                         DRM_DEBUG("IH: HPD4\n");
4800                                 }
4801                                 break;
4802                         case 4:
4803                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4804                                         rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
4805                                         queue_hotplug = true;
4806                                         DRM_DEBUG("IH: HPD5\n");
4807                                 }
4808                                 break;
4809                         case 5:
4810                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4811                                         rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
4812                                         queue_hotplug = true;
4813                                         DRM_DEBUG("IH: HPD6\n");
4814                                 }
4815                                 break;
4816                         default:
4817                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4818                                 break;
4819                         }
4820                         break;
4821                 case 44: /* hdmi */
4822                         switch (src_data) {
4823                         case 0:
4824                                 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4825                                         rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
4826                                         queue_hdmi = true;
4827                                         DRM_DEBUG("IH: HDMI0\n");
4828                                 }
4829                                 break;
4830                         case 1:
4831                                 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4832                                         rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
4833                                         queue_hdmi = true;
4834                                         DRM_DEBUG("IH: HDMI1\n");
4835                                 }
4836                                 break;
4837                         case 2:
4838                                 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4839                                         rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
4840                                         queue_hdmi = true;
4841                                         DRM_DEBUG("IH: HDMI2\n");
4842                                 }
4843                                 break;
4844                         case 3:
4845                                 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4846                                         rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
4847                                         queue_hdmi = true;
4848                                         DRM_DEBUG("IH: HDMI3\n");
4849                                 }
4850                                 break;
4851                         case 4:
4852                                 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4853                                         rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
4854                                         queue_hdmi = true;
4855                                         DRM_DEBUG("IH: HDMI4\n");
4856                                 }
4857                                 break;
4858                         case 5:
4859                                 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4860                                         rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
4861                                         queue_hdmi = true;
4862                                         DRM_DEBUG("IH: HDMI5\n");
4863                                 }
4864                                 break;
4865                         default:
4866                                 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
4867                                 break;
4868                         }
4869                 case 124: /* UVD */
4870                         DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
4871                         radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
4872                         break;
4873                 case 146:
4874                 case 147:
4875                         dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
4876                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
4877                                 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4878                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4879                                 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4880                         /* reset addr and status */
4881                         WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
4882                         break;
4883                 case 176: /* CP_INT in ring buffer */
4884                 case 177: /* CP_INT in IB1 */
4885                 case 178: /* CP_INT in IB2 */
4886                         DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
4887                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4888                         break;
4889                 case 181: /* CP EOP event */
4890                         DRM_DEBUG("IH: CP EOP\n");
4891                         if (rdev->family >= CHIP_CAYMAN) {
4892                                 switch (src_data) {
4893                                 case 0:
4894                                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4895                                         break;
4896                                 case 1:
4897                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4898                                         break;
4899                                 case 2:
4900                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4901                                         break;
4902                                 }
4903                         } else
4904                                 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4905                         break;
4906                 case 224: /* DMA trap event */
4907                         DRM_DEBUG("IH: DMA trap\n");
4908                         radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4909                         break;
4910                 case 230: /* thermal low to high */
4911                         DRM_DEBUG("IH: thermal low to high\n");
4912                         rdev->pm.dpm.thermal.high_to_low = false;
4913                         queue_thermal = true;
4914                         break;
4915                 case 231: /* thermal high to low */
4916                         DRM_DEBUG("IH: thermal high to low\n");
4917                         rdev->pm.dpm.thermal.high_to_low = true;
4918                         queue_thermal = true;
4919                         break;
4920                 case 233: /* GUI IDLE */
4921                         DRM_DEBUG("IH: GUI idle\n");
4922                         break;
4923                 case 244: /* DMA trap event */
4924                         if (rdev->family >= CHIP_CAYMAN) {
4925                                 DRM_DEBUG("IH: DMA1 trap\n");
4926                                 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
4927                         }
4928                         break;
4929                 default:
4930                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4931                         break;
4932                 }
4933
4934                 /* wptr/rptr are in bytes! */
4935                 rptr += 16;
4936                 rptr &= rdev->ih.ptr_mask;
4937         }
4938         if (queue_hotplug)
4939                 schedule_work(&rdev->hotplug_work);
4940         if (queue_hdmi)
4941                 schedule_work(&rdev->audio_work);
4942         if (queue_thermal && rdev->pm.dpm_enabled)
4943                 schedule_work(&rdev->pm.dpm.thermal.work);
4944         rdev->ih.rptr = rptr;
4945         WREG32(IH_RB_RPTR, rdev->ih.rptr);
4946         atomic_set(&rdev->ih.lock, 0);
4947
4948         /* make sure wptr hasn't changed while processing */
4949         wptr = evergreen_get_ih_wptr(rdev);
4950         if (wptr != rptr)
4951                 goto restart_ih;
4952
4953         return IRQ_HANDLED;
4954 }
4955
4956 /**
4957  * evergreen_dma_fence_ring_emit - emit a fence on the DMA ring
4958  *
4959  * @rdev: radeon_device pointer
4960  * @fence: radeon fence object
4961  *
4962  * Add a DMA fence packet to the ring to write
4963  * the fence seq number and DMA trap packet to generate
4964  * an interrupt if needed (evergreen-SI).
4965  */
4966 void evergreen_dma_fence_ring_emit(struct radeon_device *rdev,
4967                                    struct radeon_fence *fence)
4968 {
4969         struct radeon_ring *ring = &rdev->ring[fence->ring];
4970         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
4971         /* write the fence */
4972         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0));
4973         radeon_ring_write(ring, addr & 0xfffffffc);
4974         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff));
4975         radeon_ring_write(ring, fence->seq);
4976         /* generate an interrupt */
4977         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_TRAP, 0, 0));
4978         /* flush HDP */
4979         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0));
4980         radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
4981         radeon_ring_write(ring, 1);
4982 }
4983
4984 /**
4985  * evergreen_dma_ring_ib_execute - schedule an IB on the DMA engine
4986  *
4987  * @rdev: radeon_device pointer
4988  * @ib: IB object to schedule
4989  *
4990  * Schedule an IB in the DMA ring (evergreen).
4991  */
4992 void evergreen_dma_ring_ib_execute(struct radeon_device *rdev,
4993                                    struct radeon_ib *ib)
4994 {
4995         struct radeon_ring *ring = &rdev->ring[ib->ring];
4996
4997         if (rdev->wb.enabled) {
4998                 u32 next_rptr = ring->wptr + 4;
4999                 while ((next_rptr & 7) != 5)
5000                         next_rptr++;
5001                 next_rptr += 3;
5002                 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 1));
5003                 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
5004                 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff);
5005                 radeon_ring_write(ring, next_rptr);
5006         }
5007
5008         /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring.
5009          * Pad as necessary with NOPs.
5010          */
5011         while ((ring->wptr & 7) != 5)
5012                 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5013         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_INDIRECT_BUFFER, 0, 0));
5014         radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0));
5015         radeon_ring_write(ring, (ib->length_dw << 12) | (upper_32_bits(ib->gpu_addr) & 0xFF));
5016
5017 }
5018
5019 /**
5020  * evergreen_copy_dma - copy pages using the DMA engine
5021  *
5022  * @rdev: radeon_device pointer
5023  * @src_offset: src GPU address
5024  * @dst_offset: dst GPU address
5025  * @num_gpu_pages: number of GPU pages to xfer
5026  * @fence: radeon fence object
5027  *
5028  * Copy GPU paging using the DMA engine (evergreen-cayman).
5029  * Used by the radeon ttm implementation to move pages if
5030  * registered as the asic copy callback.
5031  */
5032 int evergreen_copy_dma(struct radeon_device *rdev,
5033                        uint64_t src_offset, uint64_t dst_offset,
5034                        unsigned num_gpu_pages,
5035                        struct radeon_fence **fence)
5036 {
5037         struct radeon_semaphore *sem = NULL;
5038         int ring_index = rdev->asic->copy.dma_ring_index;
5039         struct radeon_ring *ring = &rdev->ring[ring_index];
5040         u32 size_in_dw, cur_size_in_dw;
5041         int i, num_loops;
5042         int r = 0;
5043
5044         r = radeon_semaphore_create(rdev, &sem);
5045         if (r) {
5046                 DRM_ERROR("radeon: moving bo (%d).\n", r);
5047                 return r;
5048         }
5049
5050         size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4;
5051         num_loops = DIV_ROUND_UP(size_in_dw, 0xfffff);
5052         r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11);
5053         if (r) {
5054                 DRM_ERROR("radeon: moving bo (%d).\n", r);
5055                 radeon_semaphore_free(rdev, &sem, NULL);
5056                 return r;
5057         }
5058
5059         if (radeon_fence_need_sync(*fence, ring->idx)) {
5060                 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
5061                                             ring->idx);
5062                 radeon_fence_note_sync(*fence, ring->idx);
5063         } else {
5064                 radeon_semaphore_free(rdev, &sem, NULL);
5065         }
5066
5067         for (i = 0; i < num_loops; i++) {
5068                 cur_size_in_dw = size_in_dw;
5069                 if (cur_size_in_dw > 0xFFFFF)
5070                         cur_size_in_dw = 0xFFFFF;
5071                 size_in_dw -= cur_size_in_dw;
5072                 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 0, cur_size_in_dw));
5073                 radeon_ring_write(ring, dst_offset & 0xfffffffc);
5074                 radeon_ring_write(ring, src_offset & 0xfffffffc);
5075                 radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff);
5076                 radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff);
5077                 src_offset += cur_size_in_dw * 4;
5078                 dst_offset += cur_size_in_dw * 4;
5079         }
5080
5081         r = radeon_fence_emit(rdev, fence, ring->idx);
5082         if (r) {
5083                 radeon_ring_unlock_undo(rdev, ring);
5084                 return r;
5085         }
5086
5087         radeon_ring_unlock_commit(rdev, ring);
5088         radeon_semaphore_free(rdev, &sem, *fence);
5089
5090         return r;
5091 }
5092
5093 static int evergreen_startup(struct radeon_device *rdev)
5094 {
5095         struct radeon_ring *ring;
5096         int r;
5097
5098         /* enable pcie gen2 link */
5099         evergreen_pcie_gen2_enable(rdev);
5100         /* enable aspm */
5101         evergreen_program_aspm(rdev);
5102
5103         if (ASIC_IS_DCE5(rdev)) {
5104                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5105                         r = ni_init_microcode(rdev);
5106                         if (r) {
5107                                 DRM_ERROR("Failed to load firmware!\n");
5108                                 return r;
5109                         }
5110                 }
5111                 r = ni_mc_load_microcode(rdev);
5112                 if (r) {
5113                         DRM_ERROR("Failed to load MC firmware!\n");
5114                         return r;
5115                 }
5116         } else {
5117                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5118                         r = r600_init_microcode(rdev);
5119                         if (r) {
5120                                 DRM_ERROR("Failed to load firmware!\n");
5121                                 return r;
5122                         }
5123                 }
5124         }
5125
5126         r = r600_vram_scratch_init(rdev);
5127         if (r)
5128                 return r;
5129
5130         evergreen_mc_program(rdev);
5131         if (rdev->flags & RADEON_IS_AGP) {
5132                 evergreen_agp_enable(rdev);
5133         } else {
5134                 r = evergreen_pcie_gart_enable(rdev);
5135                 if (r)
5136                         return r;
5137         }
5138         evergreen_gpu_init(rdev);
5139
5140         r = evergreen_blit_init(rdev);
5141         if (r) {
5142                 r600_blit_fini(rdev);
5143                 rdev->asic->copy.copy = NULL;
5144                 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
5145         }
5146
5147         /* allocate rlc buffers */
5148         if (rdev->flags & RADEON_IS_IGP) {
5149                 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5150                 rdev->rlc.reg_list_size = sumo_rlc_save_restore_register_list_size;
5151                 rdev->rlc.cs_data = evergreen_cs_data;
5152                 r = sumo_rlc_init(rdev);
5153                 if (r) {
5154                         DRM_ERROR("Failed to init rlc BOs!\n");
5155                         return r;
5156                 }
5157         }
5158
5159         /* allocate wb buffer */
5160         r = radeon_wb_init(rdev);
5161         if (r)
5162                 return r;
5163
5164         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5165         if (r) {
5166                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5167                 return r;
5168         }
5169
5170         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5171         if (r) {
5172                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5173                 return r;
5174         }
5175
5176         r = rv770_uvd_resume(rdev);
5177         if (!r) {
5178                 r = radeon_fence_driver_start_ring(rdev,
5179                                                    R600_RING_TYPE_UVD_INDEX);
5180                 if (r)
5181                         dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5182         }
5183
5184         if (r)
5185                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5186
5187         /* Enable IRQ */
5188         if (!rdev->irq.installed) {
5189                 r = radeon_irq_kms_init(rdev);
5190                 if (r)
5191                         return r;
5192         }
5193
5194         r = r600_irq_init(rdev);
5195         if (r) {
5196                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5197                 radeon_irq_kms_fini(rdev);
5198                 return r;
5199         }
5200         evergreen_irq_set(rdev);
5201
5202         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5203         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5204                              R600_CP_RB_RPTR, R600_CP_RB_WPTR,
5205                              0, 0xfffff, RADEON_CP_PACKET2);
5206         if (r)
5207                 return r;
5208
5209         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5210         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5211                              DMA_RB_RPTR, DMA_RB_WPTR,
5212                              2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5213         if (r)
5214                 return r;
5215
5216         r = evergreen_cp_load_microcode(rdev);
5217         if (r)
5218                 return r;
5219         r = evergreen_cp_resume(rdev);
5220         if (r)
5221                 return r;
5222         r = r600_dma_resume(rdev);
5223         if (r)
5224                 return r;
5225
5226         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5227         if (ring->ring_size) {
5228                 r = radeon_ring_init(rdev, ring, ring->ring_size,
5229                                      R600_WB_UVD_RPTR_OFFSET,
5230                                      UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
5231                                      0, 0xfffff, RADEON_CP_PACKET2);
5232                 if (!r)
5233                         r = r600_uvd_init(rdev);
5234
5235                 if (r)
5236                         DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
5237         }
5238
5239         r = radeon_ib_pool_init(rdev);
5240         if (r) {
5241                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5242                 return r;
5243         }
5244
5245         r = r600_audio_init(rdev);
5246         if (r) {
5247                 DRM_ERROR("radeon: audio init failed\n");
5248                 return r;
5249         }
5250
5251         return 0;
5252 }
5253
5254 int evergreen_resume(struct radeon_device *rdev)
5255 {
5256         int r;
5257
5258         /* reset the asic, the gfx blocks are often in a bad state
5259          * after the driver is unloaded or after a resume
5260          */
5261         if (radeon_asic_reset(rdev))
5262                 dev_warn(rdev->dev, "GPU reset failed !\n");
5263         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5264          * posting will perform necessary task to bring back GPU into good
5265          * shape.
5266          */
5267         /* post card */
5268         atom_asic_init(rdev->mode_info.atom_context);
5269
5270         /* init golden registers */
5271         evergreen_init_golden_registers(rdev);
5272
5273         rdev->accel_working = true;
5274         r = evergreen_startup(rdev);
5275         if (r) {
5276                 DRM_ERROR("evergreen startup failed on resume\n");
5277                 rdev->accel_working = false;
5278                 return r;
5279         }
5280
5281         return r;
5282
5283 }
5284
5285 int evergreen_suspend(struct radeon_device *rdev)
5286 {
5287         r600_audio_fini(rdev);
5288         radeon_uvd_suspend(rdev);
5289         r700_cp_stop(rdev);
5290         r600_dma_stop(rdev);
5291         r600_uvd_rbc_stop(rdev);
5292         evergreen_irq_suspend(rdev);
5293         radeon_wb_disable(rdev);
5294         evergreen_pcie_gart_disable(rdev);
5295
5296         return 0;
5297 }
5298
5299 /* Plan is to move initialization in that function and use
5300  * helper function so that radeon_device_init pretty much
5301  * do nothing more than calling asic specific function. This
5302  * should also allow to remove a bunch of callback function
5303  * like vram_info.
5304  */
5305 int evergreen_init(struct radeon_device *rdev)
5306 {
5307         int r;
5308
5309         /* Read BIOS */
5310         if (!radeon_get_bios(rdev)) {
5311                 if (ASIC_IS_AVIVO(rdev))
5312                         return -EINVAL;
5313         }
5314         /* Must be an ATOMBIOS */
5315         if (!rdev->is_atom_bios) {
5316                 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5317                 return -EINVAL;
5318         }
5319         r = radeon_atombios_init(rdev);
5320         if (r)
5321                 return r;
5322         /* reset the asic, the gfx blocks are often in a bad state
5323          * after the driver is unloaded or after a resume
5324          */
5325         if (radeon_asic_reset(rdev))
5326                 dev_warn(rdev->dev, "GPU reset failed !\n");
5327         /* Post card if necessary */
5328         if (!radeon_card_posted(rdev)) {
5329                 if (!rdev->bios) {
5330                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5331                         return -EINVAL;
5332                 }
5333                 DRM_INFO("GPU not posted. posting now...\n");
5334                 atom_asic_init(rdev->mode_info.atom_context);
5335         }
5336         /* init golden registers */
5337         evergreen_init_golden_registers(rdev);
5338         /* Initialize scratch registers */
5339         r600_scratch_init(rdev);
5340         /* Initialize surface registers */
5341         radeon_surface_init(rdev);
5342         /* Initialize clocks */
5343         radeon_get_clock_info(rdev->ddev);
5344         /* Fence driver */
5345         r = radeon_fence_driver_init(rdev);
5346         if (r)
5347                 return r;
5348         /* initialize AGP */
5349         if (rdev->flags & RADEON_IS_AGP) {
5350                 r = radeon_agp_init(rdev);
5351                 if (r)
5352                         radeon_agp_disable(rdev);
5353         }
5354         /* initialize memory controller */
5355         r = evergreen_mc_init(rdev);
5356         if (r)
5357                 return r;
5358         /* Memory manager */
5359         r = radeon_bo_init(rdev);
5360         if (r)
5361                 return r;
5362
5363         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5364         r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5365
5366         rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5367         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5368
5369         r = radeon_uvd_init(rdev);
5370         if (!r) {
5371                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
5372                 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
5373                                4096);
5374         }
5375
5376         rdev->ih.ring_obj = NULL;
5377         r600_ih_ring_init(rdev, 64 * 1024);
5378
5379         r = r600_pcie_gart_init(rdev);
5380         if (r)
5381                 return r;
5382
5383         rdev->accel_working = true;
5384         r = evergreen_startup(rdev);
5385         if (r) {
5386                 dev_err(rdev->dev, "disabling GPU acceleration\n");
5387                 r700_cp_fini(rdev);
5388                 r600_dma_fini(rdev);
5389                 r600_irq_fini(rdev);
5390                 if (rdev->flags & RADEON_IS_IGP)
5391                         sumo_rlc_fini(rdev);
5392                 radeon_wb_fini(rdev);
5393                 radeon_ib_pool_fini(rdev);
5394                 radeon_irq_kms_fini(rdev);
5395                 evergreen_pcie_gart_fini(rdev);
5396                 rdev->accel_working = false;
5397         }
5398
5399         /* Don't start up if the MC ucode is missing on BTC parts.
5400          * The default clocks and voltages before the MC ucode
5401          * is loaded are not suffient for advanced operations.
5402          */
5403         if (ASIC_IS_DCE5(rdev)) {
5404                 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5405                         DRM_ERROR("radeon: MC ucode required for NI+.\n");
5406                         return -EINVAL;
5407                 }
5408         }
5409
5410         return 0;
5411 }
5412
5413 void evergreen_fini(struct radeon_device *rdev)
5414 {
5415         r600_audio_fini(rdev);
5416         r600_blit_fini(rdev);
5417         r700_cp_fini(rdev);
5418         r600_dma_fini(rdev);
5419         r600_irq_fini(rdev);
5420         if (rdev->flags & RADEON_IS_IGP)
5421                 sumo_rlc_fini(rdev);
5422         radeon_wb_fini(rdev);
5423         radeon_ib_pool_fini(rdev);
5424         radeon_irq_kms_fini(rdev);
5425         evergreen_pcie_gart_fini(rdev);
5426         radeon_uvd_fini(rdev);
5427         r600_vram_scratch_fini(rdev);
5428         radeon_gem_fini(rdev);
5429         radeon_fence_driver_fini(rdev);
5430         radeon_agp_fini(rdev);
5431         radeon_bo_fini(rdev);
5432         radeon_atombios_fini(rdev);
5433         kfree(rdev->bios);
5434         rdev->bios = NULL;
5435 }
5436
5437 void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5438 {
5439         u32 link_width_cntl, speed_cntl;
5440
5441         if (radeon_pcie_gen2 == 0)
5442                 return;
5443
5444         if (rdev->flags & RADEON_IS_IGP)
5445                 return;
5446
5447         if (!(rdev->flags & RADEON_IS_PCIE))
5448                 return;
5449
5450         /* x2 cards have a special sequence */
5451         if (ASIC_IS_X2(rdev))
5452                 return;
5453
5454         if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5455                 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5456                 return;
5457
5458         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5459         if (speed_cntl & LC_CURRENT_DATA_RATE) {
5460                 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5461                 return;
5462         }
5463
5464         DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5465
5466         if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5467             (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5468
5469                 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5470                 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5471                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5472
5473                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5474                 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5475                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5476
5477                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5478                 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5479                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5480
5481                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5482                 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5483                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5484
5485                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5486                 speed_cntl |= LC_GEN2_EN_STRAP;
5487                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5488
5489         } else {
5490                 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5491                 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5492                 if (1)
5493                         link_width_cntl |= LC_UPCONFIGURE_DIS;
5494                 else
5495                         link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5496                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5497         }
5498 }
5499
5500 void evergreen_program_aspm(struct radeon_device *rdev)
5501 {
5502         u32 data, orig;
5503         u32 pcie_lc_cntl, pcie_lc_cntl_old;
5504         bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5505         /* fusion_platform = true
5506          * if the system is a fusion system
5507          * (APU or DGPU in a fusion system).
5508          * todo: check if the system is a fusion platform.
5509          */
5510         bool fusion_platform = false;
5511
5512         if (!(rdev->flags & RADEON_IS_PCIE))
5513                 return;
5514
5515         switch (rdev->family) {
5516         case CHIP_CYPRESS:
5517         case CHIP_HEMLOCK:
5518         case CHIP_JUNIPER:
5519         case CHIP_REDWOOD:
5520         case CHIP_CEDAR:
5521         case CHIP_SUMO:
5522         case CHIP_SUMO2:
5523         case CHIP_PALM:
5524         case CHIP_ARUBA:
5525                 disable_l0s = true;
5526                 break;
5527         default:
5528                 disable_l0s = false;
5529                 break;
5530         }
5531
5532         if (rdev->flags & RADEON_IS_IGP)
5533                 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5534
5535         data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5536         if (fusion_platform)
5537                 data &= ~MULTI_PIF;
5538         else
5539                 data |= MULTI_PIF;
5540         if (data != orig)
5541                 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5542
5543         data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5544         if (fusion_platform)
5545                 data &= ~MULTI_PIF;
5546         else
5547                 data |= MULTI_PIF;
5548         if (data != orig)
5549                 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5550
5551         pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5552         pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5553         if (!disable_l0s) {
5554                 if (rdev->family >= CHIP_BARTS)
5555                         pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5556                 else
5557                         pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5558         }
5559
5560         if (!disable_l1) {
5561                 if (rdev->family >= CHIP_BARTS)
5562                         pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5563                 else
5564                         pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5565
5566                 if (!disable_plloff_in_l1) {
5567                         data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5568                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5569                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5570                         if (data != orig)
5571                                 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5572
5573                         data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5574                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5575                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5576                         if (data != orig)
5577                                 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5578
5579                         data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5580                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5581                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5582                         if (data != orig)
5583                                 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5584
5585                         data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5586                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5587                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5588                         if (data != orig)
5589                                 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5590
5591                         if (rdev->family >= CHIP_BARTS) {
5592                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5593                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5594                                 data |= PLL_RAMP_UP_TIME_0(4);
5595                                 if (data != orig)
5596                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5597
5598                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5599                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5600                                 data |= PLL_RAMP_UP_TIME_1(4);
5601                                 if (data != orig)
5602                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5603
5604                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5605                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5606                                 data |= PLL_RAMP_UP_TIME_0(4);
5607                                 if (data != orig)
5608                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5609
5610                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5611                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5612                                 data |= PLL_RAMP_UP_TIME_1(4);
5613                                 if (data != orig)
5614                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5615                         }
5616
5617                         data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5618                         data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5619                         data |= LC_DYN_LANES_PWR_STATE(3);
5620                         if (data != orig)
5621                                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5622
5623                         if (rdev->family >= CHIP_BARTS) {
5624                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5625                                 data &= ~LS2_EXIT_TIME_MASK;
5626                                 data |= LS2_EXIT_TIME(1);
5627                                 if (data != orig)
5628                                         WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5629
5630                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5631                                 data &= ~LS2_EXIT_TIME_MASK;
5632                                 data |= LS2_EXIT_TIME(1);
5633                                 if (data != orig)
5634                                         WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5635                         }
5636                 }
5637         }
5638
5639         /* evergreen parts only */
5640         if (rdev->family < CHIP_BARTS)
5641                 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5642
5643         if (pcie_lc_cntl != pcie_lc_cntl_old)
5644                 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5645 }