drivers/gpu/drm/radeon/cik.c

   1 /*
   2  * Copyright 2012 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/platform_device.h>
  26 #include <linux/slab.h>
  27 #include <linux/module.h>
  28 #include "drmP.h"
  29 #include "radeon.h"
  30 #include "radeon_asic.h"
  31 #include "cikd.h"
  32 #include "atom.h"
  33 #include "cik_blit_shaders.h"
  34
  35 /* GFX */
  36 #define CIK_PFP_UCODE_SIZE 2144
  37 #define CIK_ME_UCODE_SIZE 2144
  38 #define CIK_CE_UCODE_SIZE 2144
  39 /* compute */
  40 #define CIK_MEC_UCODE_SIZE 4192
  41 /* interrupts */
  42 #define BONAIRE_RLC_UCODE_SIZE 2048
  43 #define KB_RLC_UCODE_SIZE 2560
  44 #define KV_RLC_UCODE_SIZE 2560
  45 /* gddr controller */
  46 #define CIK_MC_UCODE_SIZE 7866
  47 /* sdma */
  48 #define CIK_SDMA_UCODE_SIZE 1050
  49 #define CIK_SDMA_UCODE_VERSION 64
  50
  51 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
  52 MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
  53 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
  54 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
  55 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
  56 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
  57 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
  58 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
  59 MODULE_FIRMWARE("radeon/KAVERI_me.bin");
  60 MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
  61 MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
  62 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
  63 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
  64 MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
  65 MODULE_FIRMWARE("radeon/KABINI_me.bin");
  66 MODULE_FIRMWARE("radeon/KABINI_ce.bin");
  67 MODULE_FIRMWARE("radeon/KABINI_mec.bin");
  68 MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
  69 MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
  70
  71 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
  72 extern void r600_ih_ring_fini(struct radeon_device *rdev);
  73 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
  74 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
  75 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
  76 extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
  77 extern void si_rlc_fini(struct radeon_device *rdev);
  78 extern int si_rlc_init(struct radeon_device *rdev);
  79 static void cik_rlc_stop(struct radeon_device *rdev);
  80
  81 /*
  82  * Indirect registers accessor
  83  */
  84 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
  85 {
  86         u32 r;
  87
  88         WREG32(PCIE_INDEX, reg);
  89         (void)RREG32(PCIE_INDEX);
  90         r = RREG32(PCIE_DATA);
  91         return r;
  92 }
  93
  94 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  95 {
  96         WREG32(PCIE_INDEX, reg);
  97         (void)RREG32(PCIE_INDEX);
  98         WREG32(PCIE_DATA, v);
  99         (void)RREG32(PCIE_DATA);
 100 }
 101
 102 static const u32 bonaire_golden_spm_registers[] =
 103 {
 104         0x30800, 0xe0ffffff, 0xe0000000
 105 };
 106
 107 static const u32 bonaire_golden_common_registers[] =
 108 {
 109         0xc770, 0xffffffff, 0x00000800,
 110         0xc774, 0xffffffff, 0x00000800,
 111         0xc798, 0xffffffff, 0x00007fbf,
 112         0xc79c, 0xffffffff, 0x00007faf
 113 };
 114
 115 static const u32 bonaire_golden_registers[] =
 116 {
 117         0x3354, 0x00000333, 0x00000333,
 118         0x3350, 0x000c0fc0, 0x00040200,
 119         0x9a10, 0x00010000, 0x00058208,
 120         0x3c000, 0xffff1fff, 0x00140000,
 121         0x3c200, 0xfdfc0fff, 0x00000100,
 122         0x3c234, 0x40000000, 0x40000200,
 123         0x9830, 0xffffffff, 0x00000000,
 124         0x9834, 0xf00fffff, 0x00000400,
 125         0x9838, 0x0002021c, 0x00020200,
 126         0xc78, 0x00000080, 0x00000000,
 127         0x5bb0, 0x000000f0, 0x00000070,
 128         0x5bc0, 0xf0311fff, 0x80300000,
 129         0x98f8, 0x73773777, 0x12010001,
 130         0x350c, 0x00810000, 0x408af000,
 131         0x7030, 0x31000111, 0x00000011,
 132         0x2f48, 0x73773777, 0x12010001,
 133         0x220c, 0x00007fb6, 0x0021a1b1,
 134         0x2210, 0x00007fb6, 0x002021b1,
 135         0x2180, 0x00007fb6, 0x00002191,
 136         0x2218, 0x00007fb6, 0x002121b1,
 137         0x221c, 0x00007fb6, 0x002021b1,
 138         0x21dc, 0x00007fb6, 0x00002191,
 139         0x21e0, 0x00007fb6, 0x00002191,
 140         0x3628, 0x0000003f, 0x0000000a,
 141         0x362c, 0x0000003f, 0x0000000a,
 142         0x2ae4, 0x00073ffe, 0x000022a2,
 143         0x240c, 0x000007ff, 0x00000000,
 144         0x8a14, 0xf000003f, 0x00000007,
 145         0x8bf0, 0x00002001, 0x00000001,
 146         0x8b24, 0xffffffff, 0x00ffffff,
 147         0x30a04, 0x0000ff0f, 0x00000000,
 148         0x28a4c, 0x07ffffff, 0x06000000,
 149         0x4d8, 0x00000fff, 0x00000100,
 150         0x3e78, 0x00000001, 0x00000002,
 151         0x9100, 0x03000000, 0x0362c688,
 152         0x8c00, 0x000000ff, 0x00000001,
 153         0xe40, 0x00001fff, 0x00001fff,
 154         0x9060, 0x0000007f, 0x00000020,
 155         0x9508, 0x00010000, 0x00010000,
 156         0xac14, 0x000003ff, 0x000000f3,
 157         0xac0c, 0xffffffff, 0x00001032
 158 };
 159
 160 static const u32 bonaire_mgcg_cgcg_init[] =
 161 {
 162         0xc420, 0xffffffff, 0xfffffffc,
 163         0x30800, 0xffffffff, 0xe0000000,
 164         0x3c2a0, 0xffffffff, 0x00000100,
 165         0x3c208, 0xffffffff, 0x00000100,
 166         0x3c2c0, 0xffffffff, 0xc0000100,
 167         0x3c2c8, 0xffffffff, 0xc0000100,
 168         0x3c2c4, 0xffffffff, 0xc0000100,
 169         0x55e4, 0xffffffff, 0x00600100,
 170         0x3c280, 0xffffffff, 0x00000100,
 171         0x3c214, 0xffffffff, 0x06000100,
 172         0x3c220, 0xffffffff, 0x00000100,
 173         0x3c218, 0xffffffff, 0x06000100,
 174         0x3c204, 0xffffffff, 0x00000100,
 175         0x3c2e0, 0xffffffff, 0x00000100,
 176         0x3c224, 0xffffffff, 0x00000100,
 177         0x3c200, 0xffffffff, 0x00000100,
 178         0x3c230, 0xffffffff, 0x00000100,
 179         0x3c234, 0xffffffff, 0x00000100,
 180         0x3c250, 0xffffffff, 0x00000100,
 181         0x3c254, 0xffffffff, 0x00000100,
 182         0x3c258, 0xffffffff, 0x00000100,
 183         0x3c25c, 0xffffffff, 0x00000100,
 184         0x3c260, 0xffffffff, 0x00000100,
 185         0x3c27c, 0xffffffff, 0x00000100,
 186         0x3c278, 0xffffffff, 0x00000100,
 187         0x3c210, 0xffffffff, 0x06000100,
 188         0x3c290, 0xffffffff, 0x00000100,
 189         0x3c274, 0xffffffff, 0x00000100,
 190         0x3c2b4, 0xffffffff, 0x00000100,
 191         0x3c2b0, 0xffffffff, 0x00000100,
 192         0x3c270, 0xffffffff, 0x00000100,
 193         0x30800, 0xffffffff, 0xe0000000,
 194         0x3c020, 0xffffffff, 0x00010000,
 195         0x3c024, 0xffffffff, 0x00030002,
 196         0x3c028, 0xffffffff, 0x00040007,
 197         0x3c02c, 0xffffffff, 0x00060005,
 198         0x3c030, 0xffffffff, 0x00090008,
 199         0x3c034, 0xffffffff, 0x00010000,
 200         0x3c038, 0xffffffff, 0x00030002,
 201         0x3c03c, 0xffffffff, 0x00040007,
 202         0x3c040, 0xffffffff, 0x00060005,
 203         0x3c044, 0xffffffff, 0x00090008,
 204         0x3c048, 0xffffffff, 0x00010000,
 205         0x3c04c, 0xffffffff, 0x00030002,
 206         0x3c050, 0xffffffff, 0x00040007,
 207         0x3c054, 0xffffffff, 0x00060005,
 208         0x3c058, 0xffffffff, 0x00090008,
 209         0x3c05c, 0xffffffff, 0x00010000,
 210         0x3c060, 0xffffffff, 0x00030002,
 211         0x3c064, 0xffffffff, 0x00040007,
 212         0x3c068, 0xffffffff, 0x00060005,
 213         0x3c06c, 0xffffffff, 0x00090008,
 214         0x3c070, 0xffffffff, 0x00010000,
 215         0x3c074, 0xffffffff, 0x00030002,
 216         0x3c078, 0xffffffff, 0x00040007,
 217         0x3c07c, 0xffffffff, 0x00060005,
 218         0x3c080, 0xffffffff, 0x00090008,
 219         0x3c084, 0xffffffff, 0x00010000,
 220         0x3c088, 0xffffffff, 0x00030002,
 221         0x3c08c, 0xffffffff, 0x00040007,
 222         0x3c090, 0xffffffff, 0x00060005,
 223         0x3c094, 0xffffffff, 0x00090008,
 224         0x3c098, 0xffffffff, 0x00010000,
 225         0x3c09c, 0xffffffff, 0x00030002,
 226         0x3c0a0, 0xffffffff, 0x00040007,
 227         0x3c0a4, 0xffffffff, 0x00060005,
 228         0x3c0a8, 0xffffffff, 0x00090008,
 229         0x3c000, 0xffffffff, 0x96e00200,
 230         0x8708, 0xffffffff, 0x00900100,
 231         0xc424, 0xffffffff, 0x0020003f,
 232         0x38, 0xffffffff, 0x0140001c,
 233         0x3c, 0x000f0000, 0x000f0000,
 234         0x220, 0xffffffff, 0xC060000C,
 235         0x224, 0xc0000fff, 0x00000100,
 236         0xf90, 0xffffffff, 0x00000100,
 237         0xf98, 0x00000101, 0x00000000,
 238         0x20a8, 0xffffffff, 0x00000104,
 239         0x55e4, 0xff000fff, 0x00000100,
 240         0x30cc, 0xc0000fff, 0x00000104,
 241         0xc1e4, 0x00000001, 0x00000001,
 242         0xd00c, 0xff000ff0, 0x00000100,
 243         0xd80c, 0xff000ff0, 0x00000100
 244 };
 245
 246 static const u32 spectre_golden_spm_registers[] =
 247 {
 248         0x30800, 0xe0ffffff, 0xe0000000
 249 };
 250
 251 static const u32 spectre_golden_common_registers[] =
 252 {
 253         0xc770, 0xffffffff, 0x00000800,
 254         0xc774, 0xffffffff, 0x00000800,
 255         0xc798, 0xffffffff, 0x00007fbf,
 256         0xc79c, 0xffffffff, 0x00007faf
 257 };
 258
 259 static const u32 spectre_golden_registers[] =
 260 {
 261         0x3c000, 0xffff1fff, 0x96940200,
 262         0x3c00c, 0xffff0001, 0xff000000,
 263         0x3c200, 0xfffc0fff, 0x00000100,
 264         0x6ed8, 0x00010101, 0x00010000,
 265         0x9834, 0xf00fffff, 0x00000400,
 266         0x9838, 0xfffffffc, 0x00020200,
 267         0x5bb0, 0x000000f0, 0x00000070,
 268         0x5bc0, 0xf0311fff, 0x80300000,
 269         0x98f8, 0x73773777, 0x12010001,
 270         0x9b7c, 0x00ff0000, 0x00fc0000,
 271         0x2f48, 0x73773777, 0x12010001,
 272         0x8a14, 0xf000003f, 0x00000007,
 273         0x8b24, 0xffffffff, 0x00ffffff,
 274         0x28350, 0x3f3f3fff, 0x00000082,
 275         0x28355, 0x0000003f, 0x00000000,
 276         0x3e78, 0x00000001, 0x00000002,
 277         0x913c, 0xffff03df, 0x00000004,
 278         0xc768, 0x00000008, 0x00000008,
 279         0x8c00, 0x000008ff, 0x00000800,
 280         0x9508, 0x00010000, 0x00010000,
 281         0xac0c, 0xffffffff, 0x54763210,
 282         0x214f8, 0x01ff01ff, 0x00000002,
 283         0x21498, 0x007ff800, 0x00200000,
 284         0x2015c, 0xffffffff, 0x00000f40,
 285         0x30934, 0xffffffff, 0x00000001
 286 };
 287
 288 static const u32 spectre_mgcg_cgcg_init[] =
 289 {
 290         0xc420, 0xffffffff, 0xfffffffc,
 291         0x30800, 0xffffffff, 0xe0000000,
 292         0x3c2a0, 0xffffffff, 0x00000100,
 293         0x3c208, 0xffffffff, 0x00000100,
 294         0x3c2c0, 0xffffffff, 0x00000100,
 295         0x3c2c8, 0xffffffff, 0x00000100,
 296         0x3c2c4, 0xffffffff, 0x00000100,
 297         0x55e4, 0xffffffff, 0x00600100,
 298         0x3c280, 0xffffffff, 0x00000100,
 299         0x3c214, 0xffffffff, 0x06000100,
 300         0x3c220, 0xffffffff, 0x00000100,
 301         0x3c218, 0xffffffff, 0x06000100,
 302         0x3c204, 0xffffffff, 0x00000100,
 303         0x3c2e0, 0xffffffff, 0x00000100,
 304         0x3c224, 0xffffffff, 0x00000100,
 305         0x3c200, 0xffffffff, 0x00000100,
 306         0x3c230, 0xffffffff, 0x00000100,
 307         0x3c234, 0xffffffff, 0x00000100,
 308         0x3c250, 0xffffffff, 0x00000100,
 309         0x3c254, 0xffffffff, 0x00000100,
 310         0x3c258, 0xffffffff, 0x00000100,
 311         0x3c25c, 0xffffffff, 0x00000100,
 312         0x3c260, 0xffffffff, 0x00000100,
 313         0x3c27c, 0xffffffff, 0x00000100,
 314         0x3c278, 0xffffffff, 0x00000100,
 315         0x3c210, 0xffffffff, 0x06000100,
 316         0x3c290, 0xffffffff, 0x00000100,
 317         0x3c274, 0xffffffff, 0x00000100,
 318         0x3c2b4, 0xffffffff, 0x00000100,
 319         0x3c2b0, 0xffffffff, 0x00000100,
 320         0x3c270, 0xffffffff, 0x00000100,
 321         0x30800, 0xffffffff, 0xe0000000,
 322         0x3c020, 0xffffffff, 0x00010000,
 323         0x3c024, 0xffffffff, 0x00030002,
 324         0x3c028, 0xffffffff, 0x00040007,
 325         0x3c02c, 0xffffffff, 0x00060005,
 326         0x3c030, 0xffffffff, 0x00090008,
 327         0x3c034, 0xffffffff, 0x00010000,
 328         0x3c038, 0xffffffff, 0x00030002,
 329         0x3c03c, 0xffffffff, 0x00040007,
 330         0x3c040, 0xffffffff, 0x00060005,
 331         0x3c044, 0xffffffff, 0x00090008,
 332         0x3c048, 0xffffffff, 0x00010000,
 333         0x3c04c, 0xffffffff, 0x00030002,
 334         0x3c050, 0xffffffff, 0x00040007,
 335         0x3c054, 0xffffffff, 0x00060005,
 336         0x3c058, 0xffffffff, 0x00090008,
 337         0x3c05c, 0xffffffff, 0x00010000,
 338         0x3c060, 0xffffffff, 0x00030002,
 339         0x3c064, 0xffffffff, 0x00040007,
 340         0x3c068, 0xffffffff, 0x00060005,
 341         0x3c06c, 0xffffffff, 0x00090008,
 342         0x3c070, 0xffffffff, 0x00010000,
 343         0x3c074, 0xffffffff, 0x00030002,
 344         0x3c078, 0xffffffff, 0x00040007,
 345         0x3c07c, 0xffffffff, 0x00060005,
 346         0x3c080, 0xffffffff, 0x00090008,
 347         0x3c084, 0xffffffff, 0x00010000,
 348         0x3c088, 0xffffffff, 0x00030002,
 349         0x3c08c, 0xffffffff, 0x00040007,
 350         0x3c090, 0xffffffff, 0x00060005,
 351         0x3c094, 0xffffffff, 0x00090008,
 352         0x3c098, 0xffffffff, 0x00010000,
 353         0x3c09c, 0xffffffff, 0x00030002,
 354         0x3c0a0, 0xffffffff, 0x00040007,
 355         0x3c0a4, 0xffffffff, 0x00060005,
 356         0x3c0a8, 0xffffffff, 0x00090008,
 357         0x3c0ac, 0xffffffff, 0x00010000,
 358         0x3c0b0, 0xffffffff, 0x00030002,
 359         0x3c0b4, 0xffffffff, 0x00040007,
 360         0x3c0b8, 0xffffffff, 0x00060005,
 361         0x3c0bc, 0xffffffff, 0x00090008,
 362         0x3c000, 0xffffffff, 0x96e00200,
 363         0x8708, 0xffffffff, 0x00900100,
 364         0xc424, 0xffffffff, 0x0020003f,
 365         0x38, 0xffffffff, 0x0140001c,
 366         0x3c, 0x000f0000, 0x000f0000,
 367         0x220, 0xffffffff, 0xC060000C,
 368         0x224, 0xc0000fff, 0x00000100,
 369         0xf90, 0xffffffff, 0x00000100,
 370         0xf98, 0x00000101, 0x00000000,
 371         0x20a8, 0xffffffff, 0x00000104,
 372         0x55e4, 0xff000fff, 0x00000100,
 373         0x30cc, 0xc0000fff, 0x00000104,
 374         0xc1e4, 0x00000001, 0x00000001,
 375         0xd00c, 0xff000ff0, 0x00000100,
 376         0xd80c, 0xff000ff0, 0x00000100
 377 };
 378
 379 static const u32 kalindi_golden_spm_registers[] =
 380 {
 381         0x30800, 0xe0ffffff, 0xe0000000
 382 };
 383
 384 static const u32 kalindi_golden_common_registers[] =
 385 {
 386         0xc770, 0xffffffff, 0x00000800,
 387         0xc774, 0xffffffff, 0x00000800,
 388         0xc798, 0xffffffff, 0x00007fbf,
 389         0xc79c, 0xffffffff, 0x00007faf
 390 };
 391
 392 static const u32 kalindi_golden_registers[] =
 393 {
 394         0x3c000, 0xffffdfff, 0x6e944040,
 395         0x55e4, 0xff607fff, 0xfc000100,
 396         0x3c220, 0xff000fff, 0x00000100,
 397         0x3c224, 0xff000fff, 0x00000100,
 398         0x3c200, 0xfffc0fff, 0x00000100,
 399         0x6ed8, 0x00010101, 0x00010000,
 400         0x9830, 0xffffffff, 0x00000000,
 401         0x9834, 0xf00fffff, 0x00000400,
 402         0x5bb0, 0x000000f0, 0x00000070,
 403         0x5bc0, 0xf0311fff, 0x80300000,
 404         0x98f8, 0x73773777, 0x12010001,
 405         0x98fc, 0xffffffff, 0x00000010,
 406         0x9b7c, 0x00ff0000, 0x00fc0000,
 407         0x8030, 0x00001f0f, 0x0000100a,
 408         0x2f48, 0x73773777, 0x12010001,
 409         0x2408, 0x000fffff, 0x000c007f,
 410         0x8a14, 0xf000003f, 0x00000007,
 411         0x8b24, 0x3fff3fff, 0x00ffcfff,
 412         0x30a04, 0x0000ff0f, 0x00000000,
 413         0x28a4c, 0x07ffffff, 0x06000000,
 414         0x4d8, 0x00000fff, 0x00000100,
 415         0x3e78, 0x00000001, 0x00000002,
 416         0xc768, 0x00000008, 0x00000008,
 417         0x8c00, 0x000000ff, 0x00000003,
 418         0x214f8, 0x01ff01ff, 0x00000002,
 419         0x21498, 0x007ff800, 0x00200000,
 420         0x2015c, 0xffffffff, 0x00000f40,
 421         0x88c4, 0x001f3ae3, 0x00000082,
 422         0x88d4, 0x0000001f, 0x00000010,
 423         0x30934, 0xffffffff, 0x00000000
 424 };
 425
 426 static const u32 kalindi_mgcg_cgcg_init[] =
 427 {
 428         0xc420, 0xffffffff, 0xfffffffc,
 429         0x30800, 0xffffffff, 0xe0000000,
 430         0x3c2a0, 0xffffffff, 0x00000100,
 431         0x3c208, 0xffffffff, 0x00000100,
 432         0x3c2c0, 0xffffffff, 0x00000100,
 433         0x3c2c8, 0xffffffff, 0x00000100,
 434         0x3c2c4, 0xffffffff, 0x00000100,
 435         0x55e4, 0xffffffff, 0x00600100,
 436         0x3c280, 0xffffffff, 0x00000100,
 437         0x3c214, 0xffffffff, 0x06000100,
 438         0x3c220, 0xffffffff, 0x00000100,
 439         0x3c218, 0xffffffff, 0x06000100,
 440         0x3c204, 0xffffffff, 0x00000100,
 441         0x3c2e0, 0xffffffff, 0x00000100,
 442         0x3c224, 0xffffffff, 0x00000100,
 443         0x3c200, 0xffffffff, 0x00000100,
 444         0x3c230, 0xffffffff, 0x00000100,
 445         0x3c234, 0xffffffff, 0x00000100,
 446         0x3c250, 0xffffffff, 0x00000100,
 447         0x3c254, 0xffffffff, 0x00000100,
 448         0x3c258, 0xffffffff, 0x00000100,
 449         0x3c25c, 0xffffffff, 0x00000100,
 450         0x3c260, 0xffffffff, 0x00000100,
 451         0x3c27c, 0xffffffff, 0x00000100,
 452         0x3c278, 0xffffffff, 0x00000100,
 453         0x3c210, 0xffffffff, 0x06000100,
 454         0x3c290, 0xffffffff, 0x00000100,
 455         0x3c274, 0xffffffff, 0x00000100,
 456         0x3c2b4, 0xffffffff, 0x00000100,
 457         0x3c2b0, 0xffffffff, 0x00000100,
 458         0x3c270, 0xffffffff, 0x00000100,
 459         0x30800, 0xffffffff, 0xe0000000,
 460         0x3c020, 0xffffffff, 0x00010000,
 461         0x3c024, 0xffffffff, 0x00030002,
 462         0x3c028, 0xffffffff, 0x00040007,
 463         0x3c02c, 0xffffffff, 0x00060005,
 464         0x3c030, 0xffffffff, 0x00090008,
 465         0x3c034, 0xffffffff, 0x00010000,
 466         0x3c038, 0xffffffff, 0x00030002,
 467         0x3c03c, 0xffffffff, 0x00040007,
 468         0x3c040, 0xffffffff, 0x00060005,
 469         0x3c044, 0xffffffff, 0x00090008,
 470         0x3c000, 0xffffffff, 0x96e00200,
 471         0x8708, 0xffffffff, 0x00900100,
 472         0xc424, 0xffffffff, 0x0020003f,
 473         0x38, 0xffffffff, 0x0140001c,
 474         0x3c, 0x000f0000, 0x000f0000,
 475         0x220, 0xffffffff, 0xC060000C,
 476         0x224, 0xc0000fff, 0x00000100,
 477         0x20a8, 0xffffffff, 0x00000104,
 478         0x55e4, 0xff000fff, 0x00000100,
 479         0x30cc, 0xc0000fff, 0x00000104,
 480         0xc1e4, 0x00000001, 0x00000001,
 481         0xd00c, 0xff000ff0, 0x00000100,
 482         0xd80c, 0xff000ff0, 0x00000100
 483 };
 484
 485 static void cik_init_golden_registers(struct radeon_device *rdev)
 486 {
 487         switch (rdev->family) {
 488         case CHIP_BONAIRE:
 489                 radeon_program_register_sequence(rdev,
 490                                                  bonaire_mgcg_cgcg_init,
 491                                                  (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
 492                 radeon_program_register_sequence(rdev,
 493                                                  bonaire_golden_registers,
 494                                                  (const u32)ARRAY_SIZE(bonaire_golden_registers));
 495                 radeon_program_register_sequence(rdev,
 496                                                  bonaire_golden_common_registers,
 497                                                  (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
 498                 radeon_program_register_sequence(rdev,
 499                                                  bonaire_golden_spm_registers,
 500                                                  (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
 501                 break;
 502         case CHIP_KABINI:
 503                 radeon_program_register_sequence(rdev,
 504                                                  kalindi_mgcg_cgcg_init,
 505                                                  (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
 506                 radeon_program_register_sequence(rdev,
 507                                                  kalindi_golden_registers,
 508                                                  (const u32)ARRAY_SIZE(kalindi_golden_registers));
 509                 radeon_program_register_sequence(rdev,
 510                                                  kalindi_golden_common_registers,
 511                                                  (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
 512                 radeon_program_register_sequence(rdev,
 513                                                  kalindi_golden_spm_registers,
 514                                                  (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
 515                 break;
 516         case CHIP_KAVERI:
 517                 radeon_program_register_sequence(rdev,
 518                                                  spectre_mgcg_cgcg_init,
 519                                                  (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
 520                 radeon_program_register_sequence(rdev,
 521                                                  spectre_golden_registers,
 522                                                  (const u32)ARRAY_SIZE(spectre_golden_registers));
 523                 radeon_program_register_sequence(rdev,
 524                                                  spectre_golden_common_registers,
 525                                                  (const u32)ARRAY_SIZE(spectre_golden_common_registers));
 526                 radeon_program_register_sequence(rdev,
 527                                                  spectre_golden_spm_registers,
 528                                                  (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
 529                 break;
 530         default:
 531                 break;
 532         }
 533 }
 534
 535 /**
 536  * cik_get_xclk - get the xclk
 537  *
 538  * @rdev: radeon_device pointer
 539  *
 540  * Returns the reference clock used by the gfx engine
 541  * (CIK).
 542  */
 543 u32 cik_get_xclk(struct radeon_device *rdev)
 544 {
 545         u32 reference_clock = rdev->clock.spll.reference_freq;
 546
 547         if (rdev->flags & RADEON_IS_IGP) {
 548                 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
 549                         return reference_clock / 2;
 550         } else {
 551                 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
 552                         return reference_clock / 4;
 553         }
 554         return reference_clock;
 555 }
 556
 557 /**
 558  * cik_mm_rdoorbell - read a doorbell dword
 559  *
 560  * @rdev: radeon_device pointer
 561  * @offset: byte offset into the aperture
 562  *
 563  * Returns the value in the doorbell aperture at the
 564  * requested offset (CIK).
 565  */
 566 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 offset)
 567 {
 568         if (offset < rdev->doorbell.size) {
 569                 return readl(((void __iomem *)rdev->doorbell.ptr) + offset);
 570         } else {
 571                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", offset);
 572                 return 0;
 573         }
 574 }
 575
 576 /**
 577  * cik_mm_wdoorbell - write a doorbell dword
 578  *
 579  * @rdev: radeon_device pointer
 580  * @offset: byte offset into the aperture
 581  * @v: value to write
 582  *
 583  * Writes @v to the doorbell aperture at the
 584  * requested offset (CIK).
 585  */
 586 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 offset, u32 v)
 587 {
 588         if (offset < rdev->doorbell.size) {
 589                 writel(v, ((void __iomem *)rdev->doorbell.ptr) + offset);
 590         } else {
 591                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", offset);
 592         }
 593 }
 594
 595 #define BONAIRE_IO_MC_REGS_SIZE 36
 596
 597 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
 598 {
 599         {0x00000070, 0x04400000},
 600         {0x00000071, 0x80c01803},
 601         {0x00000072, 0x00004004},
 602         {0x00000073, 0x00000100},
 603         {0x00000074, 0x00ff0000},
 604         {0x00000075, 0x34000000},
 605         {0x00000076, 0x08000014},
 606         {0x00000077, 0x00cc08ec},
 607         {0x00000078, 0x00000400},
 608         {0x00000079, 0x00000000},
 609         {0x0000007a, 0x04090000},
 610         {0x0000007c, 0x00000000},
 611         {0x0000007e, 0x4408a8e8},
 612         {0x0000007f, 0x00000304},
 613         {0x00000080, 0x00000000},
 614         {0x00000082, 0x00000001},
 615         {0x00000083, 0x00000002},
 616         {0x00000084, 0xf3e4f400},
 617         {0x00000085, 0x052024e3},
 618         {0x00000087, 0x00000000},
 619         {0x00000088, 0x01000000},
 620         {0x0000008a, 0x1c0a0000},
 621         {0x0000008b, 0xff010000},
 622         {0x0000008d, 0xffffefff},
 623         {0x0000008e, 0xfff3efff},
 624         {0x0000008f, 0xfff3efbf},
 625         {0x00000092, 0xf7ffffff},
 626         {0x00000093, 0xffffff7f},
 627         {0x00000095, 0x00101101},
 628         {0x00000096, 0x00000fff},
 629         {0x00000097, 0x00116fff},
 630         {0x00000098, 0x60010000},
 631         {0x00000099, 0x10010000},
 632         {0x0000009a, 0x00006000},
 633         {0x0000009b, 0x00001000},
 634         {0x0000009f, 0x00b48000}
 635 };
 636
 637 /**
 638  * cik_srbm_select - select specific register instances
 639  *
 640  * @rdev: radeon_device pointer
 641  * @me: selected ME (micro engine)
 642  * @pipe: pipe
 643  * @queue: queue
 644  * @vmid: VMID
 645  *
 646  * Switches the currently active registers instances.  Some
 647  * registers are instanced per VMID, others are instanced per
 648  * me/pipe/queue combination.
 649  */
 650 static void cik_srbm_select(struct radeon_device *rdev,
 651                             u32 me, u32 pipe, u32 queue, u32 vmid)
 652 {
 653         u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
 654                              MEID(me & 0x3) |
 655                              VMID(vmid & 0xf) |
 656                              QUEUEID(queue & 0x7));
 657         WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
 658 }
 659
 660 /* ucode loading */
 661 /**
 662  * ci_mc_load_microcode - load MC ucode into the hw
 663  *
 664  * @rdev: radeon_device pointer
 665  *
 666  * Load the GDDR MC ucode into the hw (CIK).
 667  * Returns 0 on success, error on failure.
 668  */
 669 static int ci_mc_load_microcode(struct radeon_device *rdev)
 670 {
 671         const __be32 *fw_data;
 672         u32 running, blackout = 0;
 673         u32 *io_mc_regs;
 674         int i, ucode_size, regs_size;
 675
 676         if (!rdev->mc_fw)
 677                 return -EINVAL;
 678
 679         switch (rdev->family) {
 680         case CHIP_BONAIRE:
 681         default:
 682                 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
 683                 ucode_size = CIK_MC_UCODE_SIZE;
 684                 regs_size = BONAIRE_IO_MC_REGS_SIZE;
 685                 break;
 686         }
 687
 688         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
 689
 690         if (running == 0) {
 691                 if (running) {
 692                         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
 693                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
 694                 }
 695
 696                 /* reset the engine and set to writable */
 697                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 698                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
 699
 700                 /* load mc io regs */
 701                 for (i = 0; i < regs_size; i++) {
 702                         WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
 703                         WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
 704                 }
 705                 /* load the MC ucode */
 706                 fw_data = (const __be32 *)rdev->mc_fw->data;
 707                 for (i = 0; i < ucode_size; i++)
 708                         WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
 709
 710                 /* put the engine back into the active state */
 711                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 712                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
 713                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
 714
 715                 /* wait for training to complete */
 716                 for (i = 0; i < rdev->usec_timeout; i++) {
 717                         if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
 718                                 break;
 719                         udelay(1);
 720                 }
 721                 for (i = 0; i < rdev->usec_timeout; i++) {
 722                         if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
 723                                 break;
 724                         udelay(1);
 725                 }
 726
 727                 if (running)
 728                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
 729         }
 730
 731         return 0;
 732 }
 733
 734 /**
 735  * cik_init_microcode - load ucode images from disk
 736  *
 737  * @rdev: radeon_device pointer
 738  *
 739  * Use the firmware interface to load the ucode images into
 740  * the driver (not loaded into hw).
 741  * Returns 0 on success, error on failure.
 742  */
 743 static int cik_init_microcode(struct radeon_device *rdev)
 744 {
 745         struct platform_device *pdev;
 746         const char *chip_name;
 747         size_t pfp_req_size, me_req_size, ce_req_size,
 748                 mec_req_size, rlc_req_size, mc_req_size,
 749                 sdma_req_size;
 750         char fw_name[30];
 751         int err;
 752
 753         DRM_DEBUG("\n");
 754
 755         pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
 756         err = IS_ERR(pdev);
 757         if (err) {
 758                 printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
 759                 return -EINVAL;
 760         }
 761
 762         switch (rdev->family) {
 763         case CHIP_BONAIRE:
 764                 chip_name = "BONAIRE";
 765                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
 766                 me_req_size = CIK_ME_UCODE_SIZE * 4;
 767                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
 768                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
 769                 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
 770                 mc_req_size = CIK_MC_UCODE_SIZE * 4;
 771                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
 772                 break;
 773         case CHIP_KAVERI:
 774                 chip_name = "KAVERI";
 775                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
 776                 me_req_size = CIK_ME_UCODE_SIZE * 4;
 777                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
 778                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
 779                 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
 780                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
 781                 break;
 782         case CHIP_KABINI:
 783                 chip_name = "KABINI";
 784                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
 785                 me_req_size = CIK_ME_UCODE_SIZE * 4;
 786                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
 787                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
 788                 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
 789                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
 790                 break;
 791         default: BUG();
 792         }
 793
 794         DRM_INFO("Loading %s Microcode\n", chip_name);
 795
 796         snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
 797         err = request_firmware(&rdev->pfp_fw, fw_name, &pdev->dev);
 798         if (err)
 799                 goto out;
 800         if (rdev->pfp_fw->size != pfp_req_size) {
 801                 printk(KERN_ERR
 802                        "cik_cp: Bogus length %zu in firmware \"%s\"\n",
 803                        rdev->pfp_fw->size, fw_name);
 804                 err = -EINVAL;
 805                 goto out;
 806         }
 807
 808         snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
 809         err = request_firmware(&rdev->me_fw, fw_name, &pdev->dev);
 810         if (err)
 811                 goto out;
 812         if (rdev->me_fw->size != me_req_size) {
 813                 printk(KERN_ERR
 814                        "cik_cp: Bogus length %zu in firmware \"%s\"\n",
 815                        rdev->me_fw->size, fw_name);
 816                 err = -EINVAL;
 817         }
 818
 819         snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
 820         err = request_firmware(&rdev->ce_fw, fw_name, &pdev->dev);
 821         if (err)
 822                 goto out;
 823         if (rdev->ce_fw->size != ce_req_size) {
 824                 printk(KERN_ERR
 825                        "cik_cp: Bogus length %zu in firmware \"%s\"\n",
 826                        rdev->ce_fw->size, fw_name);
 827                 err = -EINVAL;
 828         }
 829
 830         snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
 831         err = request_firmware(&rdev->mec_fw, fw_name, &pdev->dev);
 832         if (err)
 833                 goto out;
 834         if (rdev->mec_fw->size != mec_req_size) {
 835                 printk(KERN_ERR
 836                        "cik_cp: Bogus length %zu in firmware \"%s\"\n",
 837                        rdev->mec_fw->size, fw_name);
 838                 err = -EINVAL;
 839         }
 840
 841         snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
 842         err = request_firmware(&rdev->rlc_fw, fw_name, &pdev->dev);
 843         if (err)
 844                 goto out;
 845         if (rdev->rlc_fw->size != rlc_req_size) {
 846                 printk(KERN_ERR
 847                        "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
 848                        rdev->rlc_fw->size, fw_name);
 849                 err = -EINVAL;
 850         }
 851
 852         snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
 853         err = request_firmware(&rdev->sdma_fw, fw_name, &pdev->dev);
 854         if (err)
 855                 goto out;
 856         if (rdev->sdma_fw->size != sdma_req_size) {
 857                 printk(KERN_ERR
 858                        "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
 859                        rdev->sdma_fw->size, fw_name);
 860                 err = -EINVAL;
 861         }
 862
 863         /* No MC ucode on APUs */
 864         if (!(rdev->flags & RADEON_IS_IGP)) {
 865                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
 866                 err = request_firmware(&rdev->mc_fw, fw_name, &pdev->dev);
 867                 if (err)
 868                         goto out;
 869                 if (rdev->mc_fw->size != mc_req_size) {
 870                         printk(KERN_ERR
 871                                "cik_mc: Bogus length %zu in firmware \"%s\"\n",
 872                                rdev->mc_fw->size, fw_name);
 873                         err = -EINVAL;
 874                 }
 875         }
 876
 877 out:
 878         platform_device_unregister(pdev);
 879
 880         if (err) {
 881                 if (err != -EINVAL)
 882                         printk(KERN_ERR
 883                                "cik_cp: Failed to load firmware \"%s\"\n",
 884                                fw_name);
 885                 release_firmware(rdev->pfp_fw);
 886                 rdev->pfp_fw = NULL;
 887                 release_firmware(rdev->me_fw);
 888                 rdev->me_fw = NULL;
 889                 release_firmware(rdev->ce_fw);
 890                 rdev->ce_fw = NULL;
 891                 release_firmware(rdev->rlc_fw);
 892                 rdev->rlc_fw = NULL;
 893                 release_firmware(rdev->mc_fw);
 894                 rdev->mc_fw = NULL;
 895         }
 896         return err;
 897 }
 898
 899 /*
 900  * Core functions
 901  */
 902 /**
 903  * cik_tiling_mode_table_init - init the hw tiling table
 904  *
 905  * @rdev: radeon_device pointer
 906  *
 907  * Starting with SI, the tiling setup is done globally in a
 908  * set of 32 tiling modes.  Rather than selecting each set of
 909  * parameters per surface as on older asics, we just select
 910  * which index in the tiling table we want to use, and the
 911  * surface uses those parameters (CIK).
 912  */
 913 static void cik_tiling_mode_table_init(struct radeon_device *rdev)
 914 {
 915         const u32 num_tile_mode_states = 32;
 916         const u32 num_secondary_tile_mode_states = 16;
 917         u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
 918         u32 num_pipe_configs;
 919         u32 num_rbs = rdev->config.cik.max_backends_per_se *
 920                 rdev->config.cik.max_shader_engines;
 921
 922         switch (rdev->config.cik.mem_row_size_in_kb) {
 923         case 1:
 924                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
 925                 break;
 926         case 2:
 927         default:
 928                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
 929                 break;
 930         case 4:
 931                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
 932                 break;
 933         }
 934
 935         num_pipe_configs = rdev->config.cik.max_tile_pipes;
 936         if (num_pipe_configs > 8)
 937                 num_pipe_configs = 8; /* ??? */
 938
 939         if (num_pipe_configs == 8) {
 940                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
 941                         switch (reg_offset) {
 942                         case 0:
 943                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 944                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
 945                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
 946                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
 947                                 break;
 948                         case 1:
 949                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 950                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
 951                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
 952                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
 953                                 break;
 954                         case 2:
 955                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 956                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
 957                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
 958                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
 959                                 break;
 960                         case 3:
 961                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 962                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
 963                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
 964                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
 965                                 break;
 966                         case 4:
 967                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 968                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
 969                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
 970                                                  TILE_SPLIT(split_equal_to_row_size));
 971                                 break;
 972                         case 5:
 973                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
 974                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
 975                                 break;
 976                         case 6:
 977                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
 978                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
 979                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
 980                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
 981                                 break;
 982                         case 7:
 983                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
 984                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
 985                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
 986                                                  TILE_SPLIT(split_equal_to_row_size));
 987                                 break;
 988                         case 8:
 989                                 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
 990                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
 991                                 break;
 992                         case 9:
 993                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
 994                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
 995                                 break;
 996                         case 10:
 997                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 998                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
 999                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1000                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1001                                 break;
1002                         case 11:
1003                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1004                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1005                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1006                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1007                                 break;
1008                         case 12:
1009                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1010                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1011                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1012                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1013                                 break;
1014                         case 13:
1015                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1016                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
1017                                 break;
1018                         case 14:
1019                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1020                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1021                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1022                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1023                                 break;
1024                         case 16:
1025                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1026                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1027                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1028                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1029                                 break;
1030                         case 17:
1031                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1032                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1033                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1034                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1035                                 break;
1036                         case 27:
1037                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1038                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
1039                                 break;
1040                         case 28:
1041                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1042                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1043                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1044                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1045                                 break;
1046                         case 29:
1047                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1048                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1049                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1050                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1051                                 break;
1052                         case 30:
1053                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1054                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1055                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1056                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1057                                 break;
1058                         default:
1059                                 gb_tile_moden = 0;
1060                                 break;
1061                         }
1062                         rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
1063                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
1064                 }
1065                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
1066                         switch (reg_offset) {
1067                         case 0:
1068                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1069                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1070                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1071                                                  NUM_BANKS(ADDR_SURF_16_BANK));
1072                                 break;
1073                         case 1:
1074                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1075                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1076                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1077                                                  NUM_BANKS(ADDR_SURF_16_BANK));
1078                                 break;
1079                         case 2:
1080                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1081                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1082                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1083                                                  NUM_BANKS(ADDR_SURF_16_BANK));
1084                                 break;
1085                         case 3:
1086                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1087                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1088                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1089                                                  NUM_BANKS(ADDR_SURF_16_BANK));
1090                                 break;
1091                         case 4:
1092                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1093                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1094                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1095                                                  NUM_BANKS(ADDR_SURF_8_BANK));
1096                                 break;
1097                         case 5:
1098                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1099                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1100                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1101                                                  NUM_BANKS(ADDR_SURF_4_BANK));
1102                                 break;
1103                         case 6:
1104                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1105                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1106                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1107                                                  NUM_BANKS(ADDR_SURF_2_BANK));
1108                                 break;
1109                         case 8:
1110                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1111                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
1112                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1113                                                  NUM_BANKS(ADDR_SURF_16_BANK));
1114                                 break;
1115                         case 9:
1116                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1117                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1118                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1119                                                  NUM_BANKS(ADDR_SURF_16_BANK));
1120                                 break;
1121                         case 10:
1122                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1123                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1124                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1125                                                  NUM_BANKS(ADDR_SURF_16_BANK));
1126                                 break;
1127                         case 11:
1128                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1129                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1130                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1131                                                  NUM_BANKS(ADDR_SURF_16_BANK));
1132                                 break;
1133                         case 12:
1134                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1135                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1136                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1137                                                  NUM_BANKS(ADDR_SURF_8_BANK));
1138                                 break;
1139                         case 13:
1140                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1141                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1142                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1143                                                  NUM_BANKS(ADDR_SURF_4_BANK));
1144                                 break;
1145                         case 14:
1146                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1147                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1148                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1149                                                  NUM_BANKS(ADDR_SURF_2_BANK));
1150                                 break;
1151                         default:
1152                                 gb_tile_moden = 0;
1153                                 break;
1154                         }
1155                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
1156                 }
1157         } else if (num_pipe_configs == 4) {
1158                 if (num_rbs == 4) {
1159                         for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1160                                 switch (reg_offset) {
1161                                 case 0:
1162                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1163                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1164                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1165                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
1166                                         break;
1167                                 case 1:
1168                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1169                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1170                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1171                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
1172                                         break;
1173                                 case 2:
1174                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1175                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1176                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1177                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
1178                                         break;
1179                                 case 3:
1180                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1181                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1182                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1183                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
1184                                         break;
1185                                 case 4:
1186                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1187                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1188                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1189                                                          TILE_SPLIT(split_equal_to_row_size));
1190                                         break;
1191                                 case 5:
1192                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1193                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1194                                         break;
1195                                 case 6:
1196                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1197                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1198                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1199                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
1200                                         break;
1201                                 case 7:
1202                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1203                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1204                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1205                                                          TILE_SPLIT(split_equal_to_row_size));
1206                                         break;
1207                                 case 8:
1208                                         gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
1209                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16));
1210                                         break;
1211                                 case 9:
1212                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1213                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
1214                                         break;
1215                                 case 10:
1216                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1217                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1218                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1219                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1220                                         break;
1221                                 case 11:
1222                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1223                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1224                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1225                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1226                                         break;
1227                                 case 12:
1228                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1229                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1230                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1231                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1232                                         break;
1233                                 case 13:
1234                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1235                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
1236                                         break;
1237                                 case 14:
1238                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1239                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1240                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1241                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1242                                         break;
1243                                 case 16:
1244                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1245                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1246                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1247                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1248                                         break;
1249                                 case 17:
1250                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1251                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1252                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1253                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1254                                         break;
1255                                 case 27:
1256                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1257                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
1258                                         break;
1259                                 case 28:
1260                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1261                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1262                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1263                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1264                                         break;
1265                                 case 29:
1266                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1267                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1268                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1269                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1270                                         break;
1271                                 case 30:
1272                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1273                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1274                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1275                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1276                                         break;
1277                                 default:
1278                                         gb_tile_moden = 0;
1279                                         break;
1280                                 }
1281                                 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
1282                                 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
1283                         }
1284                 } else if (num_rbs < 4) {
1285                         for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1286                                 switch (reg_offset) {
1287                                 case 0:
1288                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1289                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1290                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1291                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
1292                                         break;
1293                                 case 1:
1294                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1295                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1296                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1297                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
1298                                         break;
1299                                 case 2:
1300                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1301                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1302                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1303                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
1304                                         break;
1305                                 case 3:
1306                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1307                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1308                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1309                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
1310                                         break;
1311                                 case 4:
1312                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1313                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1314                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1315                                                          TILE_SPLIT(split_equal_to_row_size));
1316                                         break;
1317                                 case 5:
1318                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1319                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1320                                         break;
1321                                 case 6:
1322                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1323                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1324                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1325                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
1326                                         break;
1327                                 case 7:
1328                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1329                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1330                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1331                                                          TILE_SPLIT(split_equal_to_row_size));
1332                                         break;
1333                                 case 8:
1334                                         gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
1335                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16));
1336                                         break;
1337                                 case 9:
1338                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1339                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
1340                                         break;
1341                                 case 10:
1342                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1343                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1344                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1345                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1346                                         break;
1347                                 case 11:
1348                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1349                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1350                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1351                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1352                                         break;
1353                                 case 12:
1354                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1355                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1356                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1357                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1358                                         break;
1359                                 case 13:
1360                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1361                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
1362                                         break;
1363                                 case 14:
1364                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1365                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1366                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1367                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1368                                         break;
1369                                 case 16:
1370                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1371                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1372                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1373                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1374                                         break;
1375                                 case 17:
1376                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1377                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1378                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1379                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1380                                         break;
1381                                 case 27:
1382                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1383                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
1384                                         break;
1385                                 case 28:
1386                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1387                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1388                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1389                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1390                                         break;
1391                                 case 29:
1392                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1393                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1394                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1395                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1396                                         break;
1397                                 case 30:
1398                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1399                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1400                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1401                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1402                                         break;
1403                                 default:
1404                                         gb_tile_moden = 0;
1405                                         break;
1406                                 }
1407                                 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
1408                                 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
1409                         }
1410                 }
1411                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
1412                         switch (reg_offset) {
1413                         case 0:
1414                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1415                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1416                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1417                                                  NUM_BANKS(ADDR_SURF_16_BANK));
1418                                 break;
1419                         case 1:
1420                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1421                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1422                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1423                                                  NUM_BANKS(ADDR_SURF_16_BANK));
1424                                 break;
1425                         case 2:
1426                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1427                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1428                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1429                                                  NUM_BANKS(ADDR_SURF_16_BANK));
1430                                 break;
1431                         case 3:
1432                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1433                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1434                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1435                                                  NUM_BANKS(ADDR_SURF_16_BANK));
1436                                 break;
1437                         case 4:
1438                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1439                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1440                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1441                                                  NUM_BANKS(ADDR_SURF_16_BANK));
1442                                 break;
1443                         case 5:
1444                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1445                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1446                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1447                                                  NUM_BANKS(ADDR_SURF_8_BANK));
1448                                 break;
1449                         case 6:
1450                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1451                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1452                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1453                                                  NUM_BANKS(ADDR_SURF_4_BANK));
1454                                 break;
1455                         case 8:
1456                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
1457                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
1458                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1459                                                  NUM_BANKS(ADDR_SURF_16_BANK));
1460                                 break;
1461                         case 9:
1462                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
1463                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1464                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1465                                                  NUM_BANKS(ADDR_SURF_16_BANK));
1466                                 break;
1467                         case 10:
1468                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1469                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1470                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1471                                                  NUM_BANKS(ADDR_SURF_16_BANK));
1472                                 break;
1473                         case 11:
1474                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1475                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1476                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1477                                                  NUM_BANKS(ADDR_SURF_16_BANK));
1478                                 break;
1479                         case 12:
1480                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1481                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1482                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1483                                                  NUM_BANKS(ADDR_SURF_16_BANK));
1484                                 break;
1485                         case 13:
1486                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1487                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1488                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1489                                                  NUM_BANKS(ADDR_SURF_8_BANK));
1490                                 break;
1491                         case 14:
1492                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1493                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1494                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1495                                                  NUM_BANKS(ADDR_SURF_4_BANK));
1496                                 break;
1497                         default:
1498                                 gb_tile_moden = 0;
1499                                 break;
1500                         }
1501                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
1502                 }
1503         } else if (num_pipe_configs == 2) {
1504                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1505                         switch (reg_offset) {
1506                         case 0:
1507                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1508                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1509                                                  PIPE_CONFIG(ADDR_SURF_P2) |
1510                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
1511                                 break;
1512                         case 1:
1513                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1514                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1515                                                  PIPE_CONFIG(ADDR_SURF_P2) |
1516                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
1517                                 break;
1518                         case 2:
1519                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1520                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1521                                                  PIPE_CONFIG(ADDR_SURF_P2) |
1522                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
1523                                 break;
1524                         case 3:
1525                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1526                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1527                                                  PIPE_CONFIG(ADDR_SURF_P2) |
1528                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
1529                                 break;
1530                         case 4:
1531                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1532                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1533                                                  PIPE_CONFIG(ADDR_SURF_P2) |
1534                                                  TILE_SPLIT(split_equal_to_row_size));
1535                                 break;
1536                         case 5:
1537                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1538                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1539                                 break;
1540                         case 6:
1541                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1542                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1543                                                  PIPE_CONFIG(ADDR_SURF_P2) |
1544                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
1545                                 break;
1546                         case 7:
1547                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1548                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1549                                                  PIPE_CONFIG(ADDR_SURF_P2) |
1550                                                  TILE_SPLIT(split_equal_to_row_size));
1551                                 break;
1552                         case 8:
1553                                 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED);
1554                                 break;
1555                         case 9:
1556                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1557                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
1558                                 break;
1559                         case 10:
1560                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1561                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1562                                                  PIPE_CONFIG(ADDR_SURF_P2) |
1563                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1564                                 break;
1565                         case 11:
1566                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1567                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1568                                                  PIPE_CONFIG(ADDR_SURF_P2) |
1569                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1570                                 break;
1571                         case 12:
1572                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1573                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1574                                                  PIPE_CONFIG(ADDR_SURF_P2) |
1575                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1576                                 break;
1577                         case 13:
1578                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1579                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
1580                                 break;
1581                         case 14:
1582                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1583                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1584                                                  PIPE_CONFIG(ADDR_SURF_P2) |
1585                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1586                                 break;
1587                         case 16:
1588                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1589                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1590                                                  PIPE_CONFIG(ADDR_SURF_P2) |
1591                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1592                                 break;
1593                         case 17:
1594                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1595                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1596                                                  PIPE_CONFIG(ADDR_SURF_P2) |
1597                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1598                                 break;
1599                         case 27:
1600                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1601                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
1602                                 break;
1603                         case 28:
1604                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1605                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1606                                                  PIPE_CONFIG(ADDR_SURF_P2) |
1607                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1608                                 break;
1609                         case 29:
1610                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1611                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1612                                                  PIPE_CONFIG(ADDR_SURF_P2) |
1613                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1614                                 break;
1615                         case 30:
1616                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1617                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1618                                                  PIPE_CONFIG(ADDR_SURF_P2) |
1619                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1620                                 break;
1621                         default:
1622                                 gb_tile_moden = 0;
1623                                 break;
1624                         }
1625                         rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
1626                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
1627                 }
1628                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
1629                         switch (reg_offset) {
1630                         case 0:
1631                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
1632                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1633                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1634                                                  NUM_BANKS(ADDR_SURF_16_BANK));
1635                                 break;
1636                         case 1:
1637                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
1638                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1639                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1640                                                  NUM_BANKS(ADDR_SURF_16_BANK));
1641                                 break;
1642                         case 2:
1643                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1644                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1645                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1646                                                  NUM_BANKS(ADDR_SURF_16_BANK));
1647                                 break;
1648                         case 3:
1649                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1650                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1651                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1652                                                  NUM_BANKS(ADDR_SURF_16_BANK));
1653                                 break;
1654                         case 4:
1655                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1656                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1657                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1658                                                  NUM_BANKS(ADDR_SURF_16_BANK));
1659                                 break;
1660                         case 5:
1661                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1662                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1663                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1664                                                  NUM_BANKS(ADDR_SURF_16_BANK));
1665                                 break;
1666                         case 6:
1667                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1668                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1669                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1670                                                  NUM_BANKS(ADDR_SURF_8_BANK));
1671                                 break;
1672                         case 8:
1673                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
1674                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
1675                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1676                                                  NUM_BANKS(ADDR_SURF_16_BANK));
1677                                 break;
1678                         case 9:
1679                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
1680                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1681                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1682                                                  NUM_BANKS(ADDR_SURF_16_BANK));
1683                                 break;
1684                         case 10:
1685                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
1686                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1687                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1688                                                  NUM_BANKS(ADDR_SURF_16_BANK));
1689                                 break;
1690                         case 11:
1691                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
1692                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1693                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1694                                                  NUM_BANKS(ADDR_SURF_16_BANK));
1695                                 break;
1696                         case 12:
1697                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1698                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1699                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1700                                                  NUM_BANKS(ADDR_SURF_16_BANK));
1701                                 break;
1702                         case 13:
1703                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1704                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1705                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1706                                                  NUM_BANKS(ADDR_SURF_16_BANK));
1707                                 break;
1708                         case 14:
1709                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1710                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1711                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1712                                                  NUM_BANKS(ADDR_SURF_8_BANK));
1713                                 break;
1714                         default:
1715                                 gb_tile_moden = 0;
1716                                 break;
1717                         }
1718                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
1719                 }
1720         } else
1721                 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
1722 }
1723
1724 /**
1725  * cik_select_se_sh - select which SE, SH to address
1726  *
1727  * @rdev: radeon_device pointer
1728  * @se_num: shader engine to address
1729  * @sh_num: sh block to address
1730  *
1731  * Select which SE, SH combinations to address. Certain
1732  * registers are instanced per SE or SH.  0xffffffff means
1733  * broadcast to all SEs or SHs (CIK).
1734  */
1735 static void cik_select_se_sh(struct radeon_device *rdev,
1736                              u32 se_num, u32 sh_num)
1737 {
1738         u32 data = INSTANCE_BROADCAST_WRITES;
1739
1740         if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
1741                 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
1742         else if (se_num == 0xffffffff)
1743                 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
1744         else if (sh_num == 0xffffffff)
1745                 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
1746         else
1747                 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
1748         WREG32(GRBM_GFX_INDEX, data);
1749 }
1750
1751 /**
1752  * cik_create_bitmask - create a bitmask
1753  *
1754  * @bit_width: length of the mask
1755  *
1756  * create a variable length bit mask (CIK).
1757  * Returns the bitmask.
1758  */
1759 static u32 cik_create_bitmask(u32 bit_width)
1760 {
1761         u32 i, mask = 0;
1762
1763         for (i = 0; i < bit_width; i++) {
1764                 mask <<= 1;
1765                 mask |= 1;
1766         }
1767         return mask;
1768 }
1769
1770 /**
1771  * cik_select_se_sh - select which SE, SH to address
1772  *
1773  * @rdev: radeon_device pointer
1774  * @max_rb_num: max RBs (render backends) for the asic
1775  * @se_num: number of SEs (shader engines) for the asic
1776  * @sh_per_se: number of SH blocks per SE for the asic
1777  *
1778  * Calculates the bitmask of disabled RBs (CIK).
1779  * Returns the disabled RB bitmask.
1780  */
1781 static u32 cik_get_rb_disabled(struct radeon_device *rdev,
1782                               u32 max_rb_num, u32 se_num,
1783                               u32 sh_per_se)
1784 {
1785         u32 data, mask;
1786
1787         data = RREG32(CC_RB_BACKEND_DISABLE);
1788         if (data & 1)
1789                 data &= BACKEND_DISABLE_MASK;
1790         else
1791                 data = 0;
1792         data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
1793
1794         data >>= BACKEND_DISABLE_SHIFT;
1795
1796         mask = cik_create_bitmask(max_rb_num / se_num / sh_per_se);
1797
1798         return data & mask;
1799 }
1800
1801 /**
1802  * cik_setup_rb - setup the RBs on the asic
1803  *
1804  * @rdev: radeon_device pointer
1805  * @se_num: number of SEs (shader engines) for the asic
1806  * @sh_per_se: number of SH blocks per SE for the asic
1807  * @max_rb_num: max RBs (render backends) for the asic
1808  *
1809  * Configures per-SE/SH RB registers (CIK).
1810  */
1811 static void cik_setup_rb(struct radeon_device *rdev,
1812                          u32 se_num, u32 sh_per_se,
1813                          u32 max_rb_num)
1814 {
1815         int i, j;
1816         u32 data, mask;
1817         u32 disabled_rbs = 0;
1818         u32 enabled_rbs = 0;
1819
1820         for (i = 0; i < se_num; i++) {
1821                 for (j = 0; j < sh_per_se; j++) {
1822                         cik_select_se_sh(rdev, i, j);
1823                         data = cik_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
1824                         disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
1825                 }
1826         }
1827         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
1828
1829         mask = 1;
1830         for (i = 0; i < max_rb_num; i++) {
1831                 if (!(disabled_rbs & mask))
1832                         enabled_rbs |= mask;
1833                 mask <<= 1;
1834         }
1835
1836         for (i = 0; i < se_num; i++) {
1837                 cik_select_se_sh(rdev, i, 0xffffffff);
1838                 data = 0;
1839                 for (j = 0; j < sh_per_se; j++) {
1840                         switch (enabled_rbs & 3) {
1841                         case 1:
1842                                 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
1843                                 break;
1844                         case 2:
1845                                 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
1846                                 break;
1847                         case 3:
1848                         default:
1849                                 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
1850                                 break;
1851                         }
1852                         enabled_rbs >>= 2;
1853                 }
1854                 WREG32(PA_SC_RASTER_CONFIG, data);
1855         }
1856         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
1857 }
1858
1859 /**
1860  * cik_gpu_init - setup the 3D engine
1861  *
1862  * @rdev: radeon_device pointer
1863  *
1864  * Configures the 3D engine and tiling configuration
1865  * registers so that the 3D engine is usable.
1866  */
1867 static void cik_gpu_init(struct radeon_device *rdev)
1868 {
1869         u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
1870         u32 mc_shared_chmap, mc_arb_ramcfg;
1871         u32 hdp_host_path_cntl;
1872         u32 tmp;
1873         int i, j;
1874
1875         switch (rdev->family) {
1876         case CHIP_BONAIRE:
1877                 rdev->config.cik.max_shader_engines = 2;
1878                 rdev->config.cik.max_tile_pipes = 4;
1879                 rdev->config.cik.max_cu_per_sh = 7;
1880                 rdev->config.cik.max_sh_per_se = 1;
1881                 rdev->config.cik.max_backends_per_se = 2;
1882                 rdev->config.cik.max_texture_channel_caches = 4;
1883                 rdev->config.cik.max_gprs = 256;
1884                 rdev->config.cik.max_gs_threads = 32;
1885                 rdev->config.cik.max_hw_contexts = 8;
1886
1887                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
1888                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
1889                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
1890                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
1891                 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
1892                 break;
1893         case CHIP_KAVERI:
1894                 /* TODO */
1895                 break;
1896         case CHIP_KABINI:
1897         default:
1898                 rdev->config.cik.max_shader_engines = 1;
1899                 rdev->config.cik.max_tile_pipes = 2;
1900                 rdev->config.cik.max_cu_per_sh = 2;
1901                 rdev->config.cik.max_sh_per_se = 1;
1902                 rdev->config.cik.max_backends_per_se = 1;
1903                 rdev->config.cik.max_texture_channel_caches = 2;
1904                 rdev->config.cik.max_gprs = 256;
1905                 rdev->config.cik.max_gs_threads = 16;
1906                 rdev->config.cik.max_hw_contexts = 8;
1907
1908                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
1909                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
1910                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
1911                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
1912                 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
1913                 break;
1914         }
1915
1916         /* Initialize HDP */
1917         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1918                 WREG32((0x2c14 + j), 0x00000000);
1919                 WREG32((0x2c18 + j), 0x00000000);
1920                 WREG32((0x2c1c + j), 0x00000000);
1921                 WREG32((0x2c20 + j), 0x00000000);
1922                 WREG32((0x2c24 + j), 0x00000000);
1923         }
1924
1925         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
1926
1927         WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
1928
1929         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
1930         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
1931
1932         rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
1933         rdev->config.cik.mem_max_burst_length_bytes = 256;
1934         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
1935         rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
1936         if (rdev->config.cik.mem_row_size_in_kb > 4)
1937                 rdev->config.cik.mem_row_size_in_kb = 4;
1938         /* XXX use MC settings? */
1939         rdev->config.cik.shader_engine_tile_size = 32;
1940         rdev->config.cik.num_gpus = 1;
1941         rdev->config.cik.multi_gpu_tile_size = 64;
1942
1943         /* fix up row size */
1944         gb_addr_config &= ~ROW_SIZE_MASK;
1945         switch (rdev->config.cik.mem_row_size_in_kb) {
1946         case 1:
1947         default:
1948                 gb_addr_config |= ROW_SIZE(0);
1949                 break;
1950         case 2:
1951                 gb_addr_config |= ROW_SIZE(1);
1952                 break;
1953         case 4:
1954                 gb_addr_config |= ROW_SIZE(2);
1955                 break;
1956         }
1957
1958         /* setup tiling info dword.  gb_addr_config is not adequate since it does
1959          * not have bank info, so create a custom tiling dword.
1960          * bits 3:0   num_pipes
1961          * bits 7:4   num_banks
1962          * bits 11:8  group_size
1963          * bits 15:12 row_size
1964          */
1965         rdev->config.cik.tile_config = 0;
1966         switch (rdev->config.cik.num_tile_pipes) {
1967         case 1:
1968                 rdev->config.cik.tile_config |= (0 << 0);
1969                 break;
1970         case 2:
1971                 rdev->config.cik.tile_config |= (1 << 0);
1972                 break;
1973         case 4:
1974                 rdev->config.cik.tile_config |= (2 << 0);
1975                 break;
1976         case 8:
1977         default:
1978                 /* XXX what about 12? */
1979                 rdev->config.cik.tile_config |= (3 << 0);
1980                 break;
1981         }
1982         if ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT)
1983                 rdev->config.cik.tile_config |= 1 << 4;
1984         else
1985                 rdev->config.cik.tile_config |= 0 << 4;
1986         rdev->config.cik.tile_config |=
1987                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
1988         rdev->config.cik.tile_config |=
1989                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
1990
1991         WREG32(GB_ADDR_CONFIG, gb_addr_config);
1992         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
1993         WREG32(DMIF_ADDR_CALC, gb_addr_config);
1994         WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
1995         WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
1996         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
1997         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
1998         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
1999
2000         cik_tiling_mode_table_init(rdev);
2001
2002         cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
2003                      rdev->config.cik.max_sh_per_se,
2004                      rdev->config.cik.max_backends_per_se);
2005
2006         /* set HW defaults for 3D engine */
2007         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
2008
2009         WREG32(SX_DEBUG_1, 0x20);
2010
2011         WREG32(TA_CNTL_AUX, 0x00010000);
2012
2013         tmp = RREG32(SPI_CONFIG_CNTL);
2014         tmp |= 0x03000000;
2015         WREG32(SPI_CONFIG_CNTL, tmp);
2016
2017         WREG32(SQ_CONFIG, 1);
2018
2019         WREG32(DB_DEBUG, 0);
2020
2021         tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
2022         tmp |= 0x00000400;
2023         WREG32(DB_DEBUG2, tmp);
2024
2025         tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
2026         tmp |= 0x00020200;
2027         WREG32(DB_DEBUG3, tmp);
2028
2029         tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
2030         tmp |= 0x00018208;
2031         WREG32(CB_HW_CONTROL, tmp);
2032
2033         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
2034
2035         WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
2036                                  SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
2037                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
2038                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
2039
2040         WREG32(VGT_NUM_INSTANCES, 1);
2041
2042         WREG32(CP_PERFMON_CNTL, 0);
2043
2044         WREG32(SQ_CONFIG, 0);
2045
2046         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
2047                                           FORCE_EOV_MAX_REZ_CNT(255)));
2048
2049         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
2050                AUTO_INVLD_EN(ES_AND_GS_AUTO));
2051
2052         WREG32(VGT_GS_VERTEX_REUSE, 16);
2053         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
2054
2055         tmp = RREG32(HDP_MISC_CNTL);
2056         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
2057         WREG32(HDP_MISC_CNTL, tmp);
2058
2059         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
2060         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
2061
2062         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
2063         WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
2064
2065         udelay(50);
2066 }
2067
2068 /*
2069  * GPU scratch registers helpers function.
2070  */
2071 /**
2072  * cik_scratch_init - setup driver info for CP scratch regs
2073  *
2074  * @rdev: radeon_device pointer
2075  *
2076  * Set up the number and offset of the CP scratch registers.
2077  * NOTE: use of CP scratch registers is a legacy inferface and
2078  * is not used by default on newer asics (r6xx+).  On newer asics,
2079  * memory buffers are used for fences rather than scratch regs.
2080  */
2081 static void cik_scratch_init(struct radeon_device *rdev)
2082 {
2083         int i;
2084
2085         rdev->scratch.num_reg = 7;
2086         rdev->scratch.reg_base = SCRATCH_REG0;
2087         for (i = 0; i < rdev->scratch.num_reg; i++) {
2088                 rdev->scratch.free[i] = true;
2089                 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
2090         }
2091 }
2092
2093 /**
2094  * cik_ring_test - basic gfx ring test
2095  *
2096  * @rdev: radeon_device pointer
2097  * @ring: radeon_ring structure holding ring information
2098  *
2099  * Allocate a scratch register and write to it using the gfx ring (CIK).
2100  * Provides a basic gfx ring test to verify that the ring is working.
2101  * Used by cik_cp_gfx_resume();
2102  * Returns 0 on success, error on failure.
2103  */
2104 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
2105 {
2106         uint32_t scratch;
2107         uint32_t tmp = 0;
2108         unsigned i;
2109         int r;
2110
2111         r = radeon_scratch_get(rdev, &scratch);
2112         if (r) {
2113                 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
2114                 return r;
2115         }
2116         WREG32(scratch, 0xCAFEDEAD);
2117         r = radeon_ring_lock(rdev, ring, 3);
2118         if (r) {
2119                 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
2120                 radeon_scratch_free(rdev, scratch);
2121                 return r;
2122         }
2123         radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
2124         radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
2125         radeon_ring_write(ring, 0xDEADBEEF);
2126         radeon_ring_unlock_commit(rdev, ring);
2127
2128         for (i = 0; i < rdev->usec_timeout; i++) {
2129                 tmp = RREG32(scratch);
2130                 if (tmp == 0xDEADBEEF)
2131                         break;
2132                 DRM_UDELAY(1);
2133         }
2134         if (i < rdev->usec_timeout) {
2135                 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
2136         } else {
2137                 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
2138                           ring->idx, scratch, tmp);
2139                 r = -EINVAL;
2140         }
2141         radeon_scratch_free(rdev, scratch);
2142         return r;
2143 }
2144
2145 /**
2146  * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
2147  *
2148  * @rdev: radeon_device pointer
2149  * @fence: radeon fence object
2150  *
2151  * Emits a fence sequnce number on the gfx ring and flushes
2152  * GPU caches.
2153  */
2154 void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
2155                              struct radeon_fence *fence)
2156 {
2157         struct radeon_ring *ring = &rdev->ring[fence->ring];
2158         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
2159
2160         /* EVENT_WRITE_EOP - flush caches, send int */
2161         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
2162         radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
2163                                  EOP_TC_ACTION_EN |
2164                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
2165                                  EVENT_INDEX(5)));
2166         radeon_ring_write(ring, addr & 0xfffffffc);
2167         radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
2168         radeon_ring_write(ring, fence->seq);
2169         radeon_ring_write(ring, 0);
2170         /* HDP flush */
2171         /* We should be using the new WAIT_REG_MEM special op packet here
2172          * but it causes the CP to hang
2173          */
2174         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
2175         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
2176                                  WRITE_DATA_DST_SEL(0)));
2177         radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
2178         radeon_ring_write(ring, 0);
2179         radeon_ring_write(ring, 0);
2180 }
2181
2182 /**
2183  * cik_fence_compute_ring_emit - emit a fence on the compute ring
2184  *
2185  * @rdev: radeon_device pointer
2186  * @fence: radeon fence object
2187  *
2188  * Emits a fence sequnce number on the compute ring and flushes
2189  * GPU caches.
2190  */
2191 void cik_fence_compute_ring_emit(struct radeon_device *rdev,
2192                                  struct radeon_fence *fence)
2193 {
2194         struct radeon_ring *ring = &rdev->ring[fence->ring];
2195         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
2196
2197         /* RELEASE_MEM - flush caches, send int */
2198         radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
2199         radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
2200                                  EOP_TC_ACTION_EN |
2201                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
2202                                  EVENT_INDEX(5)));
2203         radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
2204         radeon_ring_write(ring, addr & 0xfffffffc);
2205         radeon_ring_write(ring, upper_32_bits(addr));
2206         radeon_ring_write(ring, fence->seq);
2207         radeon_ring_write(ring, 0);
2208         /* HDP flush */
2209         /* We should be using the new WAIT_REG_MEM special op packet here
2210          * but it causes the CP to hang
2211          */
2212         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
2213         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
2214                                  WRITE_DATA_DST_SEL(0)));
2215         radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
2216         radeon_ring_write(ring, 0);
2217         radeon_ring_write(ring, 0);
2218 }
2219
2220 void cik_semaphore_ring_emit(struct radeon_device *rdev,
2221                              struct radeon_ring *ring,
2222                              struct radeon_semaphore *semaphore,
2223                              bool emit_wait)
2224 {
2225         uint64_t addr = semaphore->gpu_addr;
2226         unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
2227
2228         radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
2229         radeon_ring_write(ring, addr & 0xffffffff);
2230         radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
2231 }
2232
2233 /*
2234  * IB stuff
2235  */
2236 /**
2237  * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
2238  *
2239  * @rdev: radeon_device pointer
2240  * @ib: radeon indirect buffer object
2241  *
2242  * Emits an DE (drawing engine) or CE (constant engine) IB
2243  * on the gfx ring.  IBs are usually generated by userspace
2244  * acceleration drivers and submitted to the kernel for
2245  * sheduling on the ring.  This function schedules the IB
2246  * on the gfx ring for execution by the GPU.
2247  */
2248 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2249 {
2250         struct radeon_ring *ring = &rdev->ring[ib->ring];
2251         u32 header, control = INDIRECT_BUFFER_VALID;
2252
2253         if (ib->is_const_ib) {
2254                 /* set switch buffer packet before const IB */
2255                 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
2256                 radeon_ring_write(ring, 0);
2257
2258                 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
2259         } else {
2260                 u32 next_rptr;
2261                 if (ring->rptr_save_reg) {
2262                         next_rptr = ring->wptr + 3 + 4;
2263                         radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
2264                         radeon_ring_write(ring, ((ring->rptr_save_reg -
2265                                                   PACKET3_SET_UCONFIG_REG_START) >> 2));
2266                         radeon_ring_write(ring, next_rptr);
2267                 } else if (rdev->wb.enabled) {
2268                         next_rptr = ring->wptr + 5 + 4;
2269                         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
2270                         radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
2271                         radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2272                         radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
2273                         radeon_ring_write(ring, next_rptr);
2274                 }
2275
2276                 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
2277         }
2278
2279         control |= ib->length_dw |
2280                 (ib->vm ? (ib->vm->id << 24) : 0);
2281
2282         radeon_ring_write(ring, header);
2283         radeon_ring_write(ring,
2284 #ifdef __BIG_ENDIAN
2285                           (2 << 0) |
2286 #endif
2287                           (ib->gpu_addr & 0xFFFFFFFC));
2288         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
2289         radeon_ring_write(ring, control);
2290 }
2291
2292 /**
2293  * cik_ib_test - basic gfx ring IB test
2294  *
2295  * @rdev: radeon_device pointer
2296  * @ring: radeon_ring structure holding ring information
2297  *
2298  * Allocate an IB and execute it on the gfx ring (CIK).
2299  * Provides a basic gfx ring test to verify that IBs are working.
2300  * Returns 0 on success, error on failure.
2301  */
2302 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
2303 {
2304         struct radeon_ib ib;
2305         uint32_t scratch;
2306         uint32_t tmp = 0;
2307         unsigned i;
2308         int r;
2309
2310         r = radeon_scratch_get(rdev, &scratch);
2311         if (r) {
2312                 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
2313                 return r;
2314         }
2315         WREG32(scratch, 0xCAFEDEAD);
2316         r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
2317         if (r) {
2318                 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
2319                 return r;
2320         }
2321         ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
2322         ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
2323         ib.ptr[2] = 0xDEADBEEF;
2324         ib.length_dw = 3;
2325         r = radeon_ib_schedule(rdev, &ib, NULL);
2326         if (r) {
2327                 radeon_scratch_free(rdev, scratch);
2328                 radeon_ib_free(rdev, &ib);
2329                 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
2330                 return r;
2331         }
2332         r = radeon_fence_wait(ib.fence, false);
2333         if (r) {
2334                 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
2335                 return r;
2336         }
2337         for (i = 0; i < rdev->usec_timeout; i++) {
2338                 tmp = RREG32(scratch);
2339                 if (tmp == 0xDEADBEEF)
2340                         break;
2341                 DRM_UDELAY(1);
2342         }
2343         if (i < rdev->usec_timeout) {
2344                 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
2345         } else {
2346                 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
2347                           scratch, tmp);
2348                 r = -EINVAL;
2349         }
2350         radeon_scratch_free(rdev, scratch);
2351         radeon_ib_free(rdev, &ib);
2352         return r;
2353 }
2354
2355 /*
2356  * CP.
2357  * On CIK, gfx and compute now have independant command processors.
2358  *
2359  * GFX
2360  * Gfx consists of a single ring and can process both gfx jobs and
2361  * compute jobs.  The gfx CP consists of three microengines (ME):
2362  * PFP - Pre-Fetch Parser
2363  * ME - Micro Engine
2364  * CE - Constant Engine
2365  * The PFP and ME make up what is considered the Drawing Engine (DE).
2366  * The CE is an asynchronous engine used for updating buffer desciptors
2367  * used by the DE so that they can be loaded into cache in parallel
2368  * while the DE is processing state update packets.
2369  *
2370  * Compute
2371  * The compute CP consists of two microengines (ME):
2372  * MEC1 - Compute MicroEngine 1
2373  * MEC2 - Compute MicroEngine 2
2374  * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
2375  * The queues are exposed to userspace and are programmed directly
2376  * by the compute runtime.
2377  */
2378 /**
2379  * cik_cp_gfx_enable - enable/disable the gfx CP MEs
2380  *
2381  * @rdev: radeon_device pointer
2382  * @enable: enable or disable the MEs
2383  *
2384  * Halts or unhalts the gfx MEs.
2385  */
2386 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
2387 {
2388         if (enable)
2389                 WREG32(CP_ME_CNTL, 0);
2390         else {
2391                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
2392                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
2393         }
2394         udelay(50);
2395 }
2396
2397 /**
2398  * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
2399  *
2400  * @rdev: radeon_device pointer
2401  *
2402  * Loads the gfx PFP, ME, and CE ucode.
2403  * Returns 0 for success, -EINVAL if the ucode is not available.
2404  */
2405 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
2406 {
2407         const __be32 *fw_data;
2408         int i;
2409
2410         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
2411                 return -EINVAL;
2412
2413         cik_cp_gfx_enable(rdev, false);
2414
2415         /* PFP */
2416         fw_data = (const __be32 *)rdev->pfp_fw->data;
2417         WREG32(CP_PFP_UCODE_ADDR, 0);
2418         for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
2419                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2420         WREG32(CP_PFP_UCODE_ADDR, 0);
2421
2422         /* CE */
2423         fw_data = (const __be32 *)rdev->ce_fw->data;
2424         WREG32(CP_CE_UCODE_ADDR, 0);
2425         for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
2426                 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
2427         WREG32(CP_CE_UCODE_ADDR, 0);
2428
2429         /* ME */
2430         fw_data = (const __be32 *)rdev->me_fw->data;
2431         WREG32(CP_ME_RAM_WADDR, 0);
2432         for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
2433                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2434         WREG32(CP_ME_RAM_WADDR, 0);
2435
2436         WREG32(CP_PFP_UCODE_ADDR, 0);
2437         WREG32(CP_CE_UCODE_ADDR, 0);
2438         WREG32(CP_ME_RAM_WADDR, 0);
2439         WREG32(CP_ME_RAM_RADDR, 0);
2440         return 0;
2441 }
2442
2443 /**
2444  * cik_cp_gfx_start - start the gfx ring
2445  *
2446  * @rdev: radeon_device pointer
2447  *
2448  * Enables the ring and loads the clear state context and other
2449  * packets required to init the ring.
2450  * Returns 0 for success, error for failure.
2451  */
2452 static int cik_cp_gfx_start(struct radeon_device *rdev)
2453 {
2454         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2455         int r, i;
2456
2457         /* init the CP */
2458         WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
2459         WREG32(CP_ENDIAN_SWAP, 0);
2460         WREG32(CP_DEVICE_ID, 1);
2461
2462         cik_cp_gfx_enable(rdev, true);
2463
2464         r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
2465         if (r) {
2466                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2467                 return r;
2468         }
2469
2470         /* init the CE partitions.  CE only used for gfx on CIK */
2471         radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
2472         radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
2473         radeon_ring_write(ring, 0xc000);
2474         radeon_ring_write(ring, 0xc000);
2475
2476         /* setup clear context state */
2477         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2478         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2479
2480         radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
2481         radeon_ring_write(ring, 0x80000000);
2482         radeon_ring_write(ring, 0x80000000);
2483
2484         for (i = 0; i < cik_default_size; i++)
2485                 radeon_ring_write(ring, cik_default_state[i]);
2486
2487         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2488         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2489
2490         /* set clear context state */
2491         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2492         radeon_ring_write(ring, 0);
2493
2494         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
2495         radeon_ring_write(ring, 0x00000316);
2496         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2497         radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
2498
2499         radeon_ring_unlock_commit(rdev, ring);
2500
2501         return 0;
2502 }
2503
2504 /**
2505  * cik_cp_gfx_fini - stop the gfx ring
2506  *
2507  * @rdev: radeon_device pointer
2508  *
2509  * Stop the gfx ring and tear down the driver ring
2510  * info.
2511  */
2512 static void cik_cp_gfx_fini(struct radeon_device *rdev)
2513 {
2514         cik_cp_gfx_enable(rdev, false);
2515         radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
2516 }
2517
2518 /**
2519  * cik_cp_gfx_resume - setup the gfx ring buffer registers
2520  *
2521  * @rdev: radeon_device pointer
2522  *
2523  * Program the location and size of the gfx ring buffer
2524  * and test it to make sure it's working.
2525  * Returns 0 for success, error for failure.
2526  */
2527 static int cik_cp_gfx_resume(struct radeon_device *rdev)
2528 {
2529         struct radeon_ring *ring;
2530         u32 tmp;
2531         u32 rb_bufsz;
2532         u64 rb_addr;
2533         int r;
2534
2535         WREG32(CP_SEM_WAIT_TIMER, 0x0);
2536         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
2537
2538         /* Set the write pointer delay */
2539         WREG32(CP_RB_WPTR_DELAY, 0);
2540
2541         /* set the RB to use vmid 0 */
2542         WREG32(CP_RB_VMID, 0);
2543
2544         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2545
2546         /* ring 0 - compute and gfx */
2547         /* Set ring buffer size */
2548         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2549         rb_bufsz = drm_order(ring->ring_size / 8);
2550         tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2551 #ifdef __BIG_ENDIAN
2552         tmp |= BUF_SWAP_32BIT;
2553 #endif
2554         WREG32(CP_RB0_CNTL, tmp);
2555
2556         /* Initialize the ring buffer's read and write pointers */
2557         WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
2558         ring->wptr = 0;
2559         WREG32(CP_RB0_WPTR, ring->wptr);
2560
2561         /* set the wb address wether it's enabled or not */
2562         WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
2563         WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2564
2565         /* scratch register shadowing is no longer supported */
2566         WREG32(SCRATCH_UMSK, 0);
2567
2568         if (!rdev->wb.enabled)
2569                 tmp |= RB_NO_UPDATE;
2570
2571         mdelay(1);
2572         WREG32(CP_RB0_CNTL, tmp);
2573
2574         rb_addr = ring->gpu_addr >> 8;
2575         WREG32(CP_RB0_BASE, rb_addr);
2576         WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
2577
2578         ring->rptr = RREG32(CP_RB0_RPTR);
2579
2580         /* start the ring */
2581         cik_cp_gfx_start(rdev);
2582         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
2583         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
2584         if (r) {
2585                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
2586                 return r;
2587         }
2588         return 0;
2589 }
2590
2591 u32 cik_compute_ring_get_rptr(struct radeon_device *rdev,
2592                               struct radeon_ring *ring)
2593 {
2594         u32 rptr;
2595
2596
2597
2598         if (rdev->wb.enabled) {
2599                 rptr = le32_to_cpu(rdev->wb.wb[ring->rptr_offs/4]);
2600         } else {
2601                 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
2602                 rptr = RREG32(CP_HQD_PQ_RPTR);
2603                 cik_srbm_select(rdev, 0, 0, 0, 0);
2604         }
2605         rptr = (rptr & ring->ptr_reg_mask) >> ring->ptr_reg_shift;
2606
2607         return rptr;
2608 }
2609
2610 u32 cik_compute_ring_get_wptr(struct radeon_device *rdev,
2611                               struct radeon_ring *ring)
2612 {
2613         u32 wptr;
2614
2615         if (rdev->wb.enabled) {
2616                 wptr = le32_to_cpu(rdev->wb.wb[ring->wptr_offs/4]);
2617         } else {
2618                 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
2619                 wptr = RREG32(CP_HQD_PQ_WPTR);
2620                 cik_srbm_select(rdev, 0, 0, 0, 0);
2621         }
2622         wptr = (wptr & ring->ptr_reg_mask) >> ring->ptr_reg_shift;
2623
2624         return wptr;
2625 }
2626
2627 void cik_compute_ring_set_wptr(struct radeon_device *rdev,
2628                                struct radeon_ring *ring)
2629 {
2630         u32 wptr = (ring->wptr << ring->ptr_reg_shift) & ring->ptr_reg_mask;
2631
2632         rdev->wb.wb[ring->wptr_offs/4] = cpu_to_le32(wptr);
2633         WDOORBELL32(ring->doorbell_offset, wptr);
2634 }
2635
2636 /**
2637  * cik_cp_compute_enable - enable/disable the compute CP MEs
2638  *
2639  * @rdev: radeon_device pointer
2640  * @enable: enable or disable the MEs
2641  *
2642  * Halts or unhalts the compute MEs.
2643  */
2644 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
2645 {
2646         if (enable)
2647                 WREG32(CP_MEC_CNTL, 0);
2648         else
2649                 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
2650         udelay(50);
2651 }
2652
2653 /**
2654  * cik_cp_compute_load_microcode - load the compute CP ME ucode
2655  *
2656  * @rdev: radeon_device pointer
2657  *
2658  * Loads the compute MEC1&2 ucode.
2659  * Returns 0 for success, -EINVAL if the ucode is not available.
2660  */
2661 static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
2662 {
2663         const __be32 *fw_data;
2664         int i;
2665
2666         if (!rdev->mec_fw)
2667                 return -EINVAL;
2668
2669         cik_cp_compute_enable(rdev, false);
2670
2671         /* MEC1 */
2672         fw_data = (const __be32 *)rdev->mec_fw->data;
2673         WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
2674         for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
2675                 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
2676         WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
2677
2678         if (rdev->family == CHIP_KAVERI) {
2679                 /* MEC2 */
2680                 fw_data = (const __be32 *)rdev->mec_fw->data;
2681                 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
2682                 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
2683                         WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
2684                 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
2685         }
2686
2687         return 0;
2688 }
2689
2690 /**
2691  * cik_cp_compute_start - start the compute queues
2692  *
2693  * @rdev: radeon_device pointer
2694  *
2695  * Enable the compute queues.
2696  * Returns 0 for success, error for failure.
2697  */
2698 static int cik_cp_compute_start(struct radeon_device *rdev)
2699 {
2700         cik_cp_compute_enable(rdev, true);
2701
2702         return 0;
2703 }
2704
2705 /**
2706  * cik_cp_compute_fini - stop the compute queues
2707  *
2708  * @rdev: radeon_device pointer
2709  *
2710  * Stop the compute queues and tear down the driver queue
2711  * info.
2712  */
2713 static void cik_cp_compute_fini(struct radeon_device *rdev)
2714 {
2715         int i, idx, r;
2716
2717         cik_cp_compute_enable(rdev, false);
2718
2719         for (i = 0; i < 2; i++) {
2720                 if (i == 0)
2721                         idx = CAYMAN_RING_TYPE_CP1_INDEX;
2722                 else
2723                         idx = CAYMAN_RING_TYPE_CP2_INDEX;
2724
2725                 if (rdev->ring[idx].mqd_obj) {
2726                         r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
2727                         if (unlikely(r != 0))
2728                                 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
2729
2730                         radeon_bo_unpin(rdev->ring[idx].mqd_obj);
2731                         radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
2732
2733                         radeon_bo_unref(&rdev->ring[idx].mqd_obj);
2734                         rdev->ring[idx].mqd_obj = NULL;
2735                 }
2736         }
2737 }
2738
2739 static void cik_mec_fini(struct radeon_device *rdev)
2740 {
2741         int r;
2742
2743         if (rdev->mec.hpd_eop_obj) {
2744                 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
2745                 if (unlikely(r != 0))
2746                         dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
2747                 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
2748                 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
2749
2750                 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
2751                 rdev->mec.hpd_eop_obj = NULL;
2752         }
2753 }
2754
2755 #define MEC_HPD_SIZE 2048
2756
2757 static int cik_mec_init(struct radeon_device *rdev)
2758 {
2759         int r;
2760         u32 *hpd;
2761
2762         /*
2763          * KV:    2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
2764          * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
2765          */
2766         if (rdev->family == CHIP_KAVERI)
2767                 rdev->mec.num_mec = 2;
2768         else
2769                 rdev->mec.num_mec = 1;
2770         rdev->mec.num_pipe = 4;
2771         rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
2772
2773         if (rdev->mec.hpd_eop_obj == NULL) {
2774                 r = radeon_bo_create(rdev,
2775                                      rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
2776                                      PAGE_SIZE, true,
2777                                      RADEON_GEM_DOMAIN_GTT, NULL,
2778                                      &rdev->mec.hpd_eop_obj);
2779                 if (r) {
2780                         dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
2781                         return r;
2782                 }
2783         }
2784
2785         r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
2786         if (unlikely(r != 0)) {
2787                 cik_mec_fini(rdev);
2788                 return r;
2789         }
2790         r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
2791                           &rdev->mec.hpd_eop_gpu_addr);
2792         if (r) {
2793                 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
2794                 cik_mec_fini(rdev);
2795                 return r;
2796         }
2797         r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
2798         if (r) {
2799                 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
2800                 cik_mec_fini(rdev);
2801                 return r;
2802         }
2803
2804         /* clear memory.  Not sure if this is required or not */
2805         memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
2806
2807         radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
2808         radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
2809
2810         return 0;
2811 }
2812
2813 struct hqd_registers
2814 {
2815         u32 cp_mqd_base_addr;
2816         u32 cp_mqd_base_addr_hi;
2817         u32 cp_hqd_active;
2818         u32 cp_hqd_vmid;
2819         u32 cp_hqd_persistent_state;
2820         u32 cp_hqd_pipe_priority;
2821         u32 cp_hqd_queue_priority;
2822         u32 cp_hqd_quantum;
2823         u32 cp_hqd_pq_base;
2824         u32 cp_hqd_pq_base_hi;
2825         u32 cp_hqd_pq_rptr;
2826         u32 cp_hqd_pq_rptr_report_addr;
2827         u32 cp_hqd_pq_rptr_report_addr_hi;
2828         u32 cp_hqd_pq_wptr_poll_addr;
2829         u32 cp_hqd_pq_wptr_poll_addr_hi;
2830         u32 cp_hqd_pq_doorbell_control;
2831         u32 cp_hqd_pq_wptr;
2832         u32 cp_hqd_pq_control;
2833         u32 cp_hqd_ib_base_addr;
2834         u32 cp_hqd_ib_base_addr_hi;
2835         u32 cp_hqd_ib_rptr;
2836         u32 cp_hqd_ib_control;
2837         u32 cp_hqd_iq_timer;
2838         u32 cp_hqd_iq_rptr;
2839         u32 cp_hqd_dequeue_request;
2840         u32 cp_hqd_dma_offload;
2841         u32 cp_hqd_sema_cmd;
2842         u32 cp_hqd_msg_type;
2843         u32 cp_hqd_atomic0_preop_lo;
2844         u32 cp_hqd_atomic0_preop_hi;
2845         u32 cp_hqd_atomic1_preop_lo;
2846         u32 cp_hqd_atomic1_preop_hi;
2847         u32 cp_hqd_hq_scheduler0;
2848         u32 cp_hqd_hq_scheduler1;
2849         u32 cp_mqd_control;
2850 };
2851
2852 struct bonaire_mqd
2853 {
2854         u32 header;
2855         u32 dispatch_initiator;
2856         u32 dimensions[3];
2857         u32 start_idx[3];
2858         u32 num_threads[3];
2859         u32 pipeline_stat_enable;
2860         u32 perf_counter_enable;
2861         u32 pgm[2];
2862         u32 tba[2];
2863         u32 tma[2];
2864         u32 pgm_rsrc[2];
2865         u32 vmid;
2866         u32 resource_limits;
2867         u32 static_thread_mgmt01[2];
2868         u32 tmp_ring_size;
2869         u32 static_thread_mgmt23[2];
2870         u32 restart[3];
2871         u32 thread_trace_enable;
2872         u32 reserved1;
2873         u32 user_data[16];
2874         u32 vgtcs_invoke_count[2];
2875         struct hqd_registers queue_state;
2876         u32 dequeue_cntr;
2877         u32 interrupt_queue[64];
2878 };
2879
2880 /**
2881  * cik_cp_compute_resume - setup the compute queue registers
2882  *
2883  * @rdev: radeon_device pointer
2884  *
2885  * Program the compute queues and test them to make sure they
2886  * are working.
2887  * Returns 0 for success, error for failure.
2888  */
2889 static int cik_cp_compute_resume(struct radeon_device *rdev)
2890 {
2891         int r, i, idx;
2892         u32 tmp;
2893         bool use_doorbell = true;
2894         u64 hqd_gpu_addr;
2895         u64 mqd_gpu_addr;
2896         u64 eop_gpu_addr;
2897         u64 wb_gpu_addr;
2898         u32 *buf;
2899         struct bonaire_mqd *mqd;
2900
2901         r = cik_cp_compute_start(rdev);
2902         if (r)
2903                 return r;
2904
2905         /* fix up chicken bits */
2906         tmp = RREG32(CP_CPF_DEBUG);
2907         tmp |= (1 << 23);
2908         WREG32(CP_CPF_DEBUG, tmp);
2909
2910         /* init the pipes */
2911         for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
2912                 int me = (i < 4) ? 1 : 2;
2913                 int pipe = (i < 4) ? i : (i - 4);
2914
2915                 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
2916
2917                 cik_srbm_select(rdev, me, pipe, 0, 0);
2918
2919                 /* write the EOP addr */
2920                 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
2921                 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
2922
2923                 /* set the VMID assigned */
2924                 WREG32(CP_HPD_EOP_VMID, 0);
2925
2926                 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
2927                 tmp = RREG32(CP_HPD_EOP_CONTROL);
2928                 tmp &= ~EOP_SIZE_MASK;
2929                 tmp |= drm_order(MEC_HPD_SIZE / 8);
2930                 WREG32(CP_HPD_EOP_CONTROL, tmp);
2931         }
2932         cik_srbm_select(rdev, 0, 0, 0, 0);
2933
2934         /* init the queues.  Just two for now. */
2935         for (i = 0; i < 2; i++) {
2936                 if (i == 0)
2937                         idx = CAYMAN_RING_TYPE_CP1_INDEX;
2938                 else
2939                         idx = CAYMAN_RING_TYPE_CP2_INDEX;
2940
2941                 if (rdev->ring[idx].mqd_obj == NULL) {
2942                         r = radeon_bo_create(rdev,
2943                                              sizeof(struct bonaire_mqd),
2944                                              PAGE_SIZE, true,
2945                                              RADEON_GEM_DOMAIN_GTT, NULL,
2946                                              &rdev->ring[idx].mqd_obj);
2947                         if (r) {
2948                                 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
2949                                 return r;
2950                         }
2951                 }
2952
2953                 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
2954                 if (unlikely(r != 0)) {
2955                         cik_cp_compute_fini(rdev);
2956                         return r;
2957                 }
2958                 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
2959                                   &mqd_gpu_addr);
2960                 if (r) {
2961                         dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
2962                         cik_cp_compute_fini(rdev);
2963                         return r;
2964                 }
2965                 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
2966                 if (r) {
2967                         dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
2968                         cik_cp_compute_fini(rdev);
2969                         return r;
2970                 }
2971
2972                 /* doorbell offset */
2973                 rdev->ring[idx].doorbell_offset =
2974                         (rdev->ring[idx].doorbell_page_num * PAGE_SIZE) + 0;
2975
2976                 /* init the mqd struct */
2977                 memset(buf, 0, sizeof(struct bonaire_mqd));
2978
2979                 mqd = (struct bonaire_mqd *)buf;
2980                 mqd->header = 0xC0310800;
2981                 mqd->static_thread_mgmt01[0] = 0xffffffff;
2982                 mqd->static_thread_mgmt01[1] = 0xffffffff;
2983                 mqd->static_thread_mgmt23[0] = 0xffffffff;
2984                 mqd->static_thread_mgmt23[1] = 0xffffffff;
2985
2986                 cik_srbm_select(rdev, rdev->ring[idx].me,
2987                                 rdev->ring[idx].pipe,
2988                                 rdev->ring[idx].queue, 0);
2989
2990                 /* disable wptr polling */
2991                 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
2992                 tmp &= ~WPTR_POLL_EN;
2993                 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
2994
2995                 /* enable doorbell? */
2996                 mqd->queue_state.cp_hqd_pq_doorbell_control =
2997                         RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
2998                 if (use_doorbell)
2999                         mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
3000                 else
3001                         mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
3002                 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
3003                        mqd->queue_state.cp_hqd_pq_doorbell_control);
3004
3005                 /* disable the queue if it's active */
3006                 mqd->queue_state.cp_hqd_dequeue_request = 0;
3007                 mqd->queue_state.cp_hqd_pq_rptr = 0;
3008                 mqd->queue_state.cp_hqd_pq_wptr= 0;
3009                 if (RREG32(CP_HQD_ACTIVE) & 1) {
3010                         WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
3011                         for (i = 0; i < rdev->usec_timeout; i++) {
3012                                 if (!(RREG32(CP_HQD_ACTIVE) & 1))
3013                                         break;
3014                                 udelay(1);
3015                         }
3016                         WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
3017                         WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
3018                         WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
3019                 }
3020
3021                 /* set the pointer to the MQD */
3022                 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
3023                 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
3024                 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
3025                 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
3026                 /* set MQD vmid to 0 */
3027                 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
3028                 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
3029                 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
3030
3031                 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3032                 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
3033                 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
3034                 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
3035                 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
3036                 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
3037
3038                 /* set up the HQD, this is similar to CP_RB0_CNTL */
3039                 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
3040                 mqd->queue_state.cp_hqd_pq_control &=
3041                         ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
3042
3043                 mqd->queue_state.cp_hqd_pq_control |=
3044                         drm_order(rdev->ring[idx].ring_size / 8);
3045                 mqd->queue_state.cp_hqd_pq_control |=
3046                         (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8);
3047 #ifdef __BIG_ENDIAN
3048                 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
3049 #endif
3050                 mqd->queue_state.cp_hqd_pq_control &=
3051                         ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
3052                 mqd->queue_state.cp_hqd_pq_control |=
3053                         PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
3054                 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
3055
3056                 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
3057                 if (i == 0)
3058                         wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
3059                 else
3060                         wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
3061                 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
3062                 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
3063                 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
3064                 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
3065                        mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
3066
3067                 /* set the wb address wether it's enabled or not */
3068                 if (i == 0)
3069                         wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
3070                 else
3071                         wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
3072                 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
3073                 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
3074                         upper_32_bits(wb_gpu_addr) & 0xffff;
3075                 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
3076                        mqd->queue_state.cp_hqd_pq_rptr_report_addr);
3077                 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
3078                        mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
3079
3080                 /* enable the doorbell if requested */
3081                 if (use_doorbell) {
3082                         mqd->queue_state.cp_hqd_pq_doorbell_control =
3083                                 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
3084                         mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
3085                         mqd->queue_state.cp_hqd_pq_doorbell_control |=
3086                                 DOORBELL_OFFSET(rdev->ring[idx].doorbell_offset / 4);
3087                         mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
3088                         mqd->queue_state.cp_hqd_pq_doorbell_control &=
3089                                 ~(DOORBELL_SOURCE | DOORBELL_HIT);
3090
3091                 } else {
3092                         mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
3093                 }
3094                 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
3095                        mqd->queue_state.cp_hqd_pq_doorbell_control);
3096
3097                 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3098                 rdev->ring[idx].wptr = 0;
3099                 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
3100                 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
3101                 rdev->ring[idx].rptr = RREG32(CP_HQD_PQ_RPTR);
3102                 mqd->queue_state.cp_hqd_pq_rptr = rdev->ring[idx].rptr;
3103
3104                 /* set the vmid for the queue */
3105                 mqd->queue_state.cp_hqd_vmid = 0;
3106                 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
3107
3108                 /* activate the queue */
3109                 mqd->queue_state.cp_hqd_active = 1;
3110                 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
3111
3112                 cik_srbm_select(rdev, 0, 0, 0, 0);
3113
3114                 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
3115                 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
3116
3117                 rdev->ring[idx].ready = true;
3118                 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
3119                 if (r)
3120                         rdev->ring[idx].ready = false;
3121         }
3122
3123         return 0;
3124 }
3125
3126 static void cik_cp_enable(struct radeon_device *rdev, bool enable)
3127 {
3128         cik_cp_gfx_enable(rdev, enable);
3129         cik_cp_compute_enable(rdev, enable);
3130 }
3131
3132 static int cik_cp_load_microcode(struct radeon_device *rdev)
3133 {
3134         int r;
3135
3136         r = cik_cp_gfx_load_microcode(rdev);
3137         if (r)
3138                 return r;
3139         r = cik_cp_compute_load_microcode(rdev);
3140         if (r)
3141                 return r;
3142
3143         return 0;
3144 }
3145
3146 static void cik_cp_fini(struct radeon_device *rdev)
3147 {
3148         cik_cp_gfx_fini(rdev);
3149         cik_cp_compute_fini(rdev);
3150 }
3151
3152 static int cik_cp_resume(struct radeon_device *rdev)
3153 {
3154         int r;
3155
3156         /* Reset all cp blocks */
3157         WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
3158         RREG32(GRBM_SOFT_RESET);
3159         mdelay(15);
3160         WREG32(GRBM_SOFT_RESET, 0);
3161         RREG32(GRBM_SOFT_RESET);
3162
3163         r = cik_cp_load_microcode(rdev);
3164         if (r)
3165                 return r;
3166
3167         r = cik_cp_gfx_resume(rdev);
3168         if (r)
3169                 return r;
3170         r = cik_cp_compute_resume(rdev);
3171         if (r)
3172                 return r;
3173
3174         return 0;
3175 }
3176
3177 /*
3178  * sDMA - System DMA
3179  * Starting with CIK, the GPU has new asynchronous
3180  * DMA engines.  These engines are used for compute
3181  * and gfx.  There are two DMA engines (SDMA0, SDMA1)
3182  * and each one supports 1 ring buffer used for gfx
3183  * and 2 queues used for compute.
3184  *
3185  * The programming model is very similar to the CP
3186  * (ring buffer, IBs, etc.), but sDMA has it's own
3187  * packet format that is different from the PM4 format
3188  * used by the CP. sDMA supports copying data, writing
3189  * embedded data, solid fills, and a number of other
3190  * things.  It also has support for tiling/detiling of
3191  * buffers.
3192  */
3193 /**
3194  * cik_sdma_ring_ib_execute - Schedule an IB on the DMA engine
3195  *
3196  * @rdev: radeon_device pointer
3197  * @ib: IB object to schedule
3198  *
3199  * Schedule an IB in the DMA ring (CIK).
3200  */
3201 void cik_sdma_ring_ib_execute(struct radeon_device *rdev,
3202                               struct radeon_ib *ib)
3203 {
3204         struct radeon_ring *ring = &rdev->ring[ib->ring];
3205         u32 extra_bits = (ib->vm ? ib->vm->id : 0) & 0xf;
3206
3207         if (rdev->wb.enabled) {
3208                 u32 next_rptr = ring->wptr + 5;
3209                 while ((next_rptr & 7) != 4)
3210                         next_rptr++;
3211                 next_rptr += 4;
3212                 radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0));
3213                 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3214                 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3215                 radeon_ring_write(ring, 1); /* number of DWs to follow */
3216                 radeon_ring_write(ring, next_rptr);
3217         }
3218
3219         /* IB packet must end on a 8 DW boundary */
3220         while ((ring->wptr & 7) != 4)
3221                 radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
3222         radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_INDIRECT_BUFFER, 0, extra_bits));
3223         radeon_ring_write(ring, ib->gpu_addr & 0xffffffe0); /* base must be 32 byte aligned */
3224         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xffffffff);
3225         radeon_ring_write(ring, ib->length_dw);
3226
3227 }
3228
3229 /**
3230  * cik_sdma_fence_ring_emit - emit a fence on the DMA ring
3231  *
3232  * @rdev: radeon_device pointer
3233  * @fence: radeon fence object
3234  *
3235  * Add a DMA fence packet to the ring to write
3236  * the fence seq number and DMA trap packet to generate
3237  * an interrupt if needed (CIK).
3238  */
3239 void cik_sdma_fence_ring_emit(struct radeon_device *rdev,
3240                               struct radeon_fence *fence)
3241 {
3242         struct radeon_ring *ring = &rdev->ring[fence->ring];
3243         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3244         u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(1) |
3245                           SDMA_POLL_REG_MEM_EXTRA_FUNC(3)); /* == */
3246         u32 ref_and_mask;
3247
3248         if (fence->ring == R600_RING_TYPE_DMA_INDEX)
3249                 ref_and_mask = SDMA0;
3250         else
3251                 ref_and_mask = SDMA1;
3252
3253         /* write the fence */
3254         radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_FENCE, 0, 0));
3255         radeon_ring_write(ring, addr & 0xffffffff);
3256         radeon_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
3257         radeon_ring_write(ring, fence->seq);
3258         /* generate an interrupt */
3259         radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_TRAP, 0, 0));
3260         /* flush HDP */
3261         radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, extra_bits));
3262         radeon_ring_write(ring, GPU_HDP_FLUSH_DONE);
3263         radeon_ring_write(ring, GPU_HDP_FLUSH_REQ);
3264         radeon_ring_write(ring, ref_and_mask); /* REFERENCE */
3265         radeon_ring_write(ring, ref_and_mask); /* MASK */
3266         radeon_ring_write(ring, (4 << 16) | 10); /* RETRY_COUNT, POLL_INTERVAL */
3267 }
3268
3269 /**
3270  * cik_sdma_semaphore_ring_emit - emit a semaphore on the dma ring
3271  *
3272  * @rdev: radeon_device pointer
3273  * @ring: radeon_ring structure holding ring information
3274  * @semaphore: radeon semaphore object
3275  * @emit_wait: wait or signal semaphore
3276  *
3277  * Add a DMA semaphore packet to the ring wait on or signal
3278  * other rings (CIK).
3279  */
3280 void cik_sdma_semaphore_ring_emit(struct radeon_device *rdev,
3281                                   struct radeon_ring *ring,
3282                                   struct radeon_semaphore *semaphore,
3283                                   bool emit_wait)
3284 {
3285         u64 addr = semaphore->gpu_addr;
3286         u32 extra_bits = emit_wait ? 0 : SDMA_SEMAPHORE_EXTRA_S;
3287
3288         radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SEMAPHORE, 0, extra_bits));
3289         radeon_ring_write(ring, addr & 0xfffffff8);
3290         radeon_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
3291 }
3292
3293 /**
3294  * cik_sdma_gfx_stop - stop the gfx async dma engines
3295  *
3296  * @rdev: radeon_device pointer
3297  *
3298  * Stop the gfx async dma ring buffers (CIK).
3299  */
3300 static void cik_sdma_gfx_stop(struct radeon_device *rdev)
3301 {
3302         u32 rb_cntl, reg_offset;
3303         int i;
3304
3305         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3306
3307         for (i = 0; i < 2; i++) {
3308                 if (i == 0)
3309                         reg_offset = SDMA0_REGISTER_OFFSET;
3310                 else
3311                         reg_offset = SDMA1_REGISTER_OFFSET;
3312                 rb_cntl = RREG32(SDMA0_GFX_RB_CNTL + reg_offset);
3313                 rb_cntl &= ~SDMA_RB_ENABLE;
3314                 WREG32(SDMA0_GFX_RB_CNTL + reg_offset, rb_cntl);
3315                 WREG32(SDMA0_GFX_IB_CNTL + reg_offset, 0);
3316         }
3317 }
3318
3319 /**
3320  * cik_sdma_rlc_stop - stop the compute async dma engines
3321  *
3322  * @rdev: radeon_device pointer
3323  *
3324  * Stop the compute async dma queues (CIK).
3325  */
3326 static void cik_sdma_rlc_stop(struct radeon_device *rdev)
3327 {
3328         /* XXX todo */
3329 }
3330
3331 /**
3332  * cik_sdma_enable - stop the async dma engines
3333  *
3334  * @rdev: radeon_device pointer
3335  * @enable: enable/disable the DMA MEs.
3336  *
3337  * Halt or unhalt the async dma engines (CIK).
3338  */
3339 static void cik_sdma_enable(struct radeon_device *rdev, bool enable)
3340 {
3341         u32 me_cntl, reg_offset;
3342         int i;
3343
3344         for (i = 0; i < 2; i++) {
3345                 if (i == 0)
3346                         reg_offset = SDMA0_REGISTER_OFFSET;
3347                 else
3348                         reg_offset = SDMA1_REGISTER_OFFSET;
3349                 me_cntl = RREG32(SDMA0_ME_CNTL + reg_offset);
3350                 if (enable)
3351                         me_cntl &= ~SDMA_HALT;
3352                 else
3353                         me_cntl |= SDMA_HALT;
3354                 WREG32(SDMA0_ME_CNTL + reg_offset, me_cntl);
3355         }
3356 }
3357
3358 /**
3359  * cik_sdma_gfx_resume - setup and start the async dma engines
3360  *
3361  * @rdev: radeon_device pointer
3362  *
3363  * Set up the gfx DMA ring buffers and enable them (CIK).
3364  * Returns 0 for success, error for failure.
3365  */
3366 static int cik_sdma_gfx_resume(struct radeon_device *rdev)
3367 {
3368         struct radeon_ring *ring;
3369         u32 rb_cntl, ib_cntl;
3370         u32 rb_bufsz;
3371         u32 reg_offset, wb_offset;
3372         int i, r;
3373
3374         for (i = 0; i < 2; i++) {
3375                 if (i == 0) {
3376                         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
3377                         reg_offset = SDMA0_REGISTER_OFFSET;
3378                         wb_offset = R600_WB_DMA_RPTR_OFFSET;
3379                 } else {
3380                         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
3381                         reg_offset = SDMA1_REGISTER_OFFSET;
3382                         wb_offset = CAYMAN_WB_DMA1_RPTR_OFFSET;
3383                 }
3384
3385                 WREG32(SDMA0_SEM_INCOMPLETE_TIMER_CNTL + reg_offset, 0);
3386                 WREG32(SDMA0_SEM_WAIT_FAIL_TIMER_CNTL + reg_offset, 0);
3387
3388                 /* Set ring buffer size in dwords */
3389                 rb_bufsz = drm_order(ring->ring_size / 4);
3390                 rb_cntl = rb_bufsz << 1;
3391 #ifdef __BIG_ENDIAN
3392                 rb_cntl |= SDMA_RB_SWAP_ENABLE | SDMA_RPTR_WRITEBACK_SWAP_ENABLE;
3393 #endif
3394                 WREG32(SDMA0_GFX_RB_CNTL + reg_offset, rb_cntl);
3395
3396                 /* Initialize the ring buffer's read and write pointers */
3397                 WREG32(SDMA0_GFX_RB_RPTR + reg_offset, 0);
3398                 WREG32(SDMA0_GFX_RB_WPTR + reg_offset, 0);
3399
3400                 /* set the wb address whether it's enabled or not */
3401                 WREG32(SDMA0_GFX_RB_RPTR_ADDR_HI + reg_offset,
3402                        upper_32_bits(rdev->wb.gpu_addr + wb_offset) & 0xFFFFFFFF);
3403                 WREG32(SDMA0_GFX_RB_RPTR_ADDR_LO + reg_offset,
3404                        ((rdev->wb.gpu_addr + wb_offset) & 0xFFFFFFFC));
3405
3406                 if (rdev->wb.enabled)
3407                         rb_cntl |= SDMA_RPTR_WRITEBACK_ENABLE;
3408
3409                 WREG32(SDMA0_GFX_RB_BASE + reg_offset, ring->gpu_addr >> 8);
3410                 WREG32(SDMA0_GFX_RB_BASE_HI + reg_offset, ring->gpu_addr >> 40);
3411
3412                 ring->wptr = 0;
3413                 WREG32(SDMA0_GFX_RB_WPTR + reg_offset, ring->wptr << 2);
3414
3415                 ring->rptr = RREG32(SDMA0_GFX_RB_RPTR + reg_offset) >> 2;
3416
3417                 /* enable DMA RB */
3418                 WREG32(SDMA0_GFX_RB_CNTL + reg_offset, rb_cntl | SDMA_RB_ENABLE);
3419
3420                 ib_cntl = SDMA_IB_ENABLE;
3421 #ifdef __BIG_ENDIAN
3422                 ib_cntl |= SDMA_IB_SWAP_ENABLE;
3423 #endif
3424                 /* enable DMA IBs */
3425                 WREG32(SDMA0_GFX_IB_CNTL + reg_offset, ib_cntl);
3426
3427                 ring->ready = true;
3428
3429                 r = radeon_ring_test(rdev, ring->idx, ring);
3430                 if (r) {
3431                         ring->ready = false;
3432                         return r;
3433                 }
3434         }
3435
3436         radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
3437
3438         return 0;
3439 }
3440
3441 /**
3442  * cik_sdma_rlc_resume - setup and start the async dma engines
3443  *
3444  * @rdev: radeon_device pointer
3445  *
3446  * Set up the compute DMA queues and enable them (CIK).
3447  * Returns 0 for success, error for failure.
3448  */
3449 static int cik_sdma_rlc_resume(struct radeon_device *rdev)
3450 {
3451         /* XXX todo */
3452         return 0;
3453 }
3454
3455 /**
3456  * cik_sdma_load_microcode - load the sDMA ME ucode
3457  *
3458  * @rdev: radeon_device pointer
3459  *
3460  * Loads the sDMA0/1 ucode.
3461  * Returns 0 for success, -EINVAL if the ucode is not available.
3462  */
3463 static int cik_sdma_load_microcode(struct radeon_device *rdev)
3464 {
3465         const __be32 *fw_data;
3466         int i;
3467
3468         if (!rdev->sdma_fw)
3469                 return -EINVAL;
3470
3471         /* stop the gfx rings and rlc compute queues */
3472         cik_sdma_gfx_stop(rdev);
3473         cik_sdma_rlc_stop(rdev);
3474
3475         /* halt the MEs */
3476         cik_sdma_enable(rdev, false);
3477
3478         /* sdma0 */
3479         fw_data = (const __be32 *)rdev->sdma_fw->data;
3480         WREG32(SDMA0_UCODE_ADDR + SDMA0_REGISTER_OFFSET, 0);
3481         for (i = 0; i < CIK_SDMA_UCODE_SIZE; i++)
3482                 WREG32(SDMA0_UCODE_DATA + SDMA0_REGISTER_OFFSET, be32_to_cpup(fw_data++));
3483         WREG32(SDMA0_UCODE_DATA + SDMA0_REGISTER_OFFSET, CIK_SDMA_UCODE_VERSION);
3484
3485         /* sdma1 */
3486         fw_data = (const __be32 *)rdev->sdma_fw->data;
3487         WREG32(SDMA0_UCODE_ADDR + SDMA1_REGISTER_OFFSET, 0);
3488         for (i = 0; i < CIK_SDMA_UCODE_SIZE; i++)
3489                 WREG32(SDMA0_UCODE_DATA + SDMA1_REGISTER_OFFSET, be32_to_cpup(fw_data++));
3490         WREG32(SDMA0_UCODE_DATA + SDMA1_REGISTER_OFFSET, CIK_SDMA_UCODE_VERSION);
3491
3492         WREG32(SDMA0_UCODE_ADDR + SDMA0_REGISTER_OFFSET, 0);
3493         WREG32(SDMA0_UCODE_ADDR + SDMA1_REGISTER_OFFSET, 0);
3494         return 0;
3495 }
3496
3497 /**
3498  * cik_sdma_resume - setup and start the async dma engines
3499  *
3500  * @rdev: radeon_device pointer
3501  *
3502  * Set up the DMA engines and enable them (CIK).
3503  * Returns 0 for success, error for failure.
3504  */
3505 static int cik_sdma_resume(struct radeon_device *rdev)
3506 {
3507         int r;
3508
3509         /* Reset dma */
3510         WREG32(SRBM_SOFT_RESET, SOFT_RESET_SDMA | SOFT_RESET_SDMA1);
3511         RREG32(SRBM_SOFT_RESET);
3512         udelay(50);
3513         WREG32(SRBM_SOFT_RESET, 0);
3514         RREG32(SRBM_SOFT_RESET);
3515
3516         r = cik_sdma_load_microcode(rdev);
3517         if (r)
3518                 return r;
3519
3520         /* unhalt the MEs */
3521         cik_sdma_enable(rdev, true);
3522
3523         /* start the gfx rings and rlc compute queues */
3524         r = cik_sdma_gfx_resume(rdev);
3525         if (r)
3526                 return r;
3527         r = cik_sdma_rlc_resume(rdev);
3528         if (r)
3529                 return r;
3530
3531         return 0;
3532 }
3533
3534 /**
3535  * cik_sdma_fini - tear down the async dma engines
3536  *
3537  * @rdev: radeon_device pointer
3538  *
3539  * Stop the async dma engines and free the rings (CIK).
3540  */
3541 static void cik_sdma_fini(struct radeon_device *rdev)
3542 {
3543         /* stop the gfx rings and rlc compute queues */
3544         cik_sdma_gfx_stop(rdev);
3545         cik_sdma_rlc_stop(rdev);
3546         /* halt the MEs */
3547         cik_sdma_enable(rdev, false);
3548         radeon_ring_fini(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX]);
3549         radeon_ring_fini(rdev, &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]);
3550         /* XXX - compute dma queue tear down */
3551 }
3552
3553 /**
3554  * cik_copy_dma - copy pages using the DMA engine
3555  *
3556  * @rdev: radeon_device pointer
3557  * @src_offset: src GPU address
3558  * @dst_offset: dst GPU address
3559  * @num_gpu_pages: number of GPU pages to xfer
3560  * @fence: radeon fence object
3561  *
3562  * Copy GPU paging using the DMA engine (CIK).
3563  * Used by the radeon ttm implementation to move pages if
3564  * registered as the asic copy callback.
3565  */
3566 int cik_copy_dma(struct radeon_device *rdev,
3567                  uint64_t src_offset, uint64_t dst_offset,
3568                  unsigned num_gpu_pages,
3569                  struct radeon_fence **fence)
3570 {
3571         struct radeon_semaphore *sem = NULL;
3572         int ring_index = rdev->asic->copy.dma_ring_index;
3573         struct radeon_ring *ring = &rdev->ring[ring_index];
3574         u32 size_in_bytes, cur_size_in_bytes;
3575         int i, num_loops;
3576         int r = 0;
3577
3578         r = radeon_semaphore_create(rdev, &sem);
3579         if (r) {
3580                 DRM_ERROR("radeon: moving bo (%d).\n", r);
3581                 return r;
3582         }
3583
3584         size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3585         num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3586         r = radeon_ring_lock(rdev, ring, num_loops * 7 + 14);
3587         if (r) {
3588                 DRM_ERROR("radeon: moving bo (%d).\n", r);
3589                 radeon_semaphore_free(rdev, &sem, NULL);
3590                 return r;
3591         }
3592
3593         if (radeon_fence_need_sync(*fence, ring->idx)) {
3594                 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
3595                                             ring->idx);
3596                 radeon_fence_note_sync(*fence, ring->idx);
3597         } else {
3598                 radeon_semaphore_free(rdev, &sem, NULL);
3599         }
3600
3601         for (i = 0; i < num_loops; i++) {
3602                 cur_size_in_bytes = size_in_bytes;
3603                 if (cur_size_in_bytes > 0x1fffff)
3604                         cur_size_in_bytes = 0x1fffff;
3605                 size_in_bytes -= cur_size_in_bytes;
3606                 radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_LINEAR, 0));
3607                 radeon_ring_write(ring, cur_size_in_bytes);
3608                 radeon_ring_write(ring, 0); /* src/dst endian swap */
3609                 radeon_ring_write(ring, src_offset & 0xffffffff);
3610                 radeon_ring_write(ring, upper_32_bits(src_offset) & 0xffffffff);
3611                 radeon_ring_write(ring, dst_offset & 0xfffffffc);
3612                 radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xffffffff);
3613                 src_offset += cur_size_in_bytes;
3614                 dst_offset += cur_size_in_bytes;
3615         }
3616
3617         r = radeon_fence_emit(rdev, fence, ring->idx);
3618         if (r) {
3619                 radeon_ring_unlock_undo(rdev, ring);
3620                 return r;
3621         }
3622
3623         radeon_ring_unlock_commit(rdev, ring);
3624         radeon_semaphore_free(rdev, &sem, *fence);
3625
3626         return r;
3627 }
3628
3629 /**
3630  * cik_sdma_ring_test - simple async dma engine test
3631  *
3632  * @rdev: radeon_device pointer
3633  * @ring: radeon_ring structure holding ring information
3634  *
3635  * Test the DMA engine by writing using it to write an
3636  * value to memory. (CIK).
3637  * Returns 0 for success, error for failure.
3638  */
3639 int cik_sdma_ring_test(struct radeon_device *rdev,
3640                        struct radeon_ring *ring)
3641 {
3642         unsigned i;
3643         int r;
3644         void __iomem *ptr = (void *)rdev->vram_scratch.ptr;
3645         u32 tmp;
3646
3647         if (!ptr) {
3648                 DRM_ERROR("invalid vram scratch pointer\n");
3649                 return -EINVAL;
3650         }
3651
3652         tmp = 0xCAFEDEAD;
3653         writel(tmp, ptr);
3654
3655         r = radeon_ring_lock(rdev, ring, 4);
3656         if (r) {
3657                 DRM_ERROR("radeon: dma failed to lock ring %d (%d).\n", ring->idx, r);
3658                 return r;
3659         }
3660         radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0));
3661         radeon_ring_write(ring, rdev->vram_scratch.gpu_addr & 0xfffffffc);
3662         radeon_ring_write(ring, upper_32_bits(rdev->vram_scratch.gpu_addr) & 0xffffffff);
3663         radeon_ring_write(ring, 1); /* number of DWs to follow */
3664         radeon_ring_write(ring, 0xDEADBEEF);
3665         radeon_ring_unlock_commit(rdev, ring);
3666
3667         for (i = 0; i < rdev->usec_timeout; i++) {
3668                 tmp = readl(ptr);
3669                 if (tmp == 0xDEADBEEF)
3670                         break;
3671                 DRM_UDELAY(1);
3672         }
3673
3674         if (i < rdev->usec_timeout) {
3675                 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3676         } else {
3677                 DRM_ERROR("radeon: ring %d test failed (0x%08X)\n",
3678                           ring->idx, tmp);
3679                 r = -EINVAL;
3680         }
3681         return r;
3682 }
3683
3684 /**
3685  * cik_sdma_ib_test - test an IB on the DMA engine
3686  *
3687  * @rdev: radeon_device pointer
3688  * @ring: radeon_ring structure holding ring information
3689  *
3690  * Test a simple IB in the DMA ring (CIK).
3691  * Returns 0 on success, error on failure.
3692  */
3693 int cik_sdma_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3694 {
3695         struct radeon_ib ib;
3696         unsigned i;
3697         int r;
3698         void __iomem *ptr = (void *)rdev->vram_scratch.ptr;
3699         u32 tmp = 0;
3700
3701         if (!ptr) {
3702                 DRM_ERROR("invalid vram scratch pointer\n");
3703                 return -EINVAL;
3704         }
3705
3706         tmp = 0xCAFEDEAD;
3707         writel(tmp, ptr);
3708
3709         r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3710         if (r) {
3711                 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3712                 return r;
3713         }
3714
3715         ib.ptr[0] = SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0);
3716         ib.ptr[1] = rdev->vram_scratch.gpu_addr & 0xfffffffc;
3717         ib.ptr[2] = upper_32_bits(rdev->vram_scratch.gpu_addr) & 0xffffffff;
3718         ib.ptr[3] = 1;
3719         ib.ptr[4] = 0xDEADBEEF;
3720         ib.length_dw = 5;
3721
3722         r = radeon_ib_schedule(rdev, &ib, NULL);
3723         if (r) {
3724                 radeon_ib_free(rdev, &ib);
3725                 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3726                 return r;
3727         }
3728         r = radeon_fence_wait(ib.fence, false);
3729         if (r) {
3730                 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3731                 return r;
3732         }
3733         for (i = 0; i < rdev->usec_timeout; i++) {
3734                 tmp = readl(ptr);
3735                 if (tmp == 0xDEADBEEF)
3736                         break;
3737                 DRM_UDELAY(1);
3738         }
3739         if (i < rdev->usec_timeout) {
3740                 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3741         } else {
3742                 DRM_ERROR("radeon: ib test failed (0x%08X)\n", tmp);
3743                 r = -EINVAL;
3744         }
3745         radeon_ib_free(rdev, &ib);
3746         return r;
3747 }
3748
3749
3750 static void cik_print_gpu_status_regs(struct radeon_device *rdev)
3751 {
3752         dev_info(rdev->dev, "  GRBM_STATUS=0x%08X\n",
3753                 RREG32(GRBM_STATUS));
3754         dev_info(rdev->dev, "  GRBM_STATUS2=0x%08X\n",
3755                 RREG32(GRBM_STATUS2));
3756         dev_info(rdev->dev, "  GRBM_STATUS_SE0=0x%08X\n",
3757                 RREG32(GRBM_STATUS_SE0));
3758         dev_info(rdev->dev, "  GRBM_STATUS_SE1=0x%08X\n",
3759                 RREG32(GRBM_STATUS_SE1));
3760         dev_info(rdev->dev, "  GRBM_STATUS_SE2=0x%08X\n",
3761                 RREG32(GRBM_STATUS_SE2));
3762         dev_info(rdev->dev, "  GRBM_STATUS_SE3=0x%08X\n",
3763                 RREG32(GRBM_STATUS_SE3));
3764         dev_info(rdev->dev, "  SRBM_STATUS=0x%08X\n",
3765                 RREG32(SRBM_STATUS));
3766         dev_info(rdev->dev, "  SRBM_STATUS2=0x%08X\n",
3767                 RREG32(SRBM_STATUS2));
3768         dev_info(rdev->dev, "  SDMA0_STATUS_REG   = 0x%08X\n",
3769                 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
3770         dev_info(rdev->dev, "  SDMA1_STATUS_REG   = 0x%08X\n",
3771                  RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
3772         dev_info(rdev->dev, "  CP_STAT = 0x%08x\n", RREG32(CP_STAT));
3773         dev_info(rdev->dev, "  CP_STALLED_STAT1 = 0x%08x\n",
3774                  RREG32(CP_STALLED_STAT1));
3775         dev_info(rdev->dev, "  CP_STALLED_STAT2 = 0x%08x\n",
3776                  RREG32(CP_STALLED_STAT2));
3777         dev_info(rdev->dev, "  CP_STALLED_STAT3 = 0x%08x\n",
3778                  RREG32(CP_STALLED_STAT3));
3779         dev_info(rdev->dev, "  CP_CPF_BUSY_STAT = 0x%08x\n",
3780                  RREG32(CP_CPF_BUSY_STAT));
3781         dev_info(rdev->dev, "  CP_CPF_STALLED_STAT1 = 0x%08x\n",
3782                  RREG32(CP_CPF_STALLED_STAT1));
3783         dev_info(rdev->dev, "  CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
3784         dev_info(rdev->dev, "  CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
3785         dev_info(rdev->dev, "  CP_CPC_STALLED_STAT1 = 0x%08x\n",
3786                  RREG32(CP_CPC_STALLED_STAT1));
3787         dev_info(rdev->dev, "  CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
3788 }
3789
3790 /**
3791  * cik_gpu_check_soft_reset - check which blocks are busy
3792  *
3793  * @rdev: radeon_device pointer
3794  *
3795  * Check which blocks are busy and return the relevant reset
3796  * mask to be used by cik_gpu_soft_reset().
3797  * Returns a mask of the blocks to be reset.
3798  */
3799 static u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
3800 {
3801         u32 reset_mask = 0;
3802         u32 tmp;
3803
3804         /* GRBM_STATUS */
3805         tmp = RREG32(GRBM_STATUS);
3806         if (tmp & (PA_BUSY | SC_BUSY |
3807                    BCI_BUSY | SX_BUSY |
3808                    TA_BUSY | VGT_BUSY |
3809                    DB_BUSY | CB_BUSY |
3810                    GDS_BUSY | SPI_BUSY |
3811                    IA_BUSY | IA_BUSY_NO_DMA))
3812                 reset_mask |= RADEON_RESET_GFX;
3813
3814         if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
3815                 reset_mask |= RADEON_RESET_CP;
3816
3817         /* GRBM_STATUS2 */
3818         tmp = RREG32(GRBM_STATUS2);
3819         if (tmp & RLC_BUSY)
3820                 reset_mask |= RADEON_RESET_RLC;
3821
3822         /* SDMA0_STATUS_REG */
3823         tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
3824         if (!(tmp & SDMA_IDLE))
3825                 reset_mask |= RADEON_RESET_DMA;
3826
3827         /* SDMA1_STATUS_REG */
3828         tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
3829         if (!(tmp & SDMA_IDLE))
3830                 reset_mask |= RADEON_RESET_DMA1;
3831
3832         /* SRBM_STATUS2 */
3833         tmp = RREG32(SRBM_STATUS2);
3834         if (tmp & SDMA_BUSY)
3835                 reset_mask |= RADEON_RESET_DMA;
3836
3837         if (tmp & SDMA1_BUSY)
3838                 reset_mask |= RADEON_RESET_DMA1;
3839
3840         /* SRBM_STATUS */
3841         tmp = RREG32(SRBM_STATUS);
3842
3843         if (tmp & IH_BUSY)
3844                 reset_mask |= RADEON_RESET_IH;
3845
3846         if (tmp & SEM_BUSY)
3847                 reset_mask |= RADEON_RESET_SEM;
3848
3849         if (tmp & GRBM_RQ_PENDING)
3850                 reset_mask |= RADEON_RESET_GRBM;
3851
3852         if (tmp & VMC_BUSY)
3853                 reset_mask |= RADEON_RESET_VMC;
3854
3855         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3856                    MCC_BUSY | MCD_BUSY))
3857                 reset_mask |= RADEON_RESET_MC;
3858
3859         if (evergreen_is_display_hung(rdev))
3860                 reset_mask |= RADEON_RESET_DISPLAY;
3861
3862         /* Skip MC reset as it's mostly likely not hung, just busy */
3863         if (reset_mask & RADEON_RESET_MC) {
3864                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3865                 reset_mask &= ~RADEON_RESET_MC;
3866         }
3867
3868         return reset_mask;
3869 }
3870
3871 /**
3872  * cik_gpu_soft_reset - soft reset GPU
3873  *
3874  * @rdev: radeon_device pointer
3875  * @reset_mask: mask of which blocks to reset
3876  *
3877  * Soft reset the blocks specified in @reset_mask.
3878  */
3879 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3880 {
3881         struct evergreen_mc_save save;
3882         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3883         u32 tmp;
3884
3885         if (reset_mask == 0)
3886                 return;
3887
3888         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3889
3890         cik_print_gpu_status_regs(rdev);
3891         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
3892                  RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3893         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3894                  RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3895
3896         /* stop the rlc */
3897         cik_rlc_stop(rdev);
3898
3899         /* Disable GFX parsing/prefetching */
3900         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3901
3902         /* Disable MEC parsing/prefetching */
3903         WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
3904
3905         if (reset_mask & RADEON_RESET_DMA) {
3906                 /* sdma0 */
3907                 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
3908                 tmp |= SDMA_HALT;
3909                 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
3910         }
3911         if (reset_mask & RADEON_RESET_DMA1) {
3912                 /* sdma1 */
3913                 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
3914                 tmp |= SDMA_HALT;
3915                 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
3916         }
3917
3918         evergreen_mc_stop(rdev, &save);
3919         if (evergreen_mc_wait_for_idle(rdev)) {
3920                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3921         }
3922
3923         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
3924                 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
3925
3926         if (reset_mask & RADEON_RESET_CP) {
3927                 grbm_soft_reset |= SOFT_RESET_CP;
3928
3929                 srbm_soft_reset |= SOFT_RESET_GRBM;
3930         }
3931
3932         if (reset_mask & RADEON_RESET_DMA)
3933                 srbm_soft_reset |= SOFT_RESET_SDMA;
3934
3935         if (reset_mask & RADEON_RESET_DMA1)
3936                 srbm_soft_reset |= SOFT_RESET_SDMA1;
3937
3938         if (reset_mask & RADEON_RESET_DISPLAY)
3939                 srbm_soft_reset |= SOFT_RESET_DC;
3940
3941         if (reset_mask & RADEON_RESET_RLC)
3942                 grbm_soft_reset |= SOFT_RESET_RLC;
3943
3944         if (reset_mask & RADEON_RESET_SEM)
3945                 srbm_soft_reset |= SOFT_RESET_SEM;
3946
3947         if (reset_mask & RADEON_RESET_IH)
3948                 srbm_soft_reset |= SOFT_RESET_IH;
3949
3950         if (reset_mask & RADEON_RESET_GRBM)
3951                 srbm_soft_reset |= SOFT_RESET_GRBM;
3952
3953         if (reset_mask & RADEON_RESET_VMC)
3954                 srbm_soft_reset |= SOFT_RESET_VMC;
3955
3956         if (!(rdev->flags & RADEON_IS_IGP)) {
3957                 if (reset_mask & RADEON_RESET_MC)
3958                         srbm_soft_reset |= SOFT_RESET_MC;
3959         }
3960
3961         if (grbm_soft_reset) {
3962                 tmp = RREG32(GRBM_SOFT_RESET);
3963                 tmp |= grbm_soft_reset;
3964                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3965                 WREG32(GRBM_SOFT_RESET, tmp);
3966                 tmp = RREG32(GRBM_SOFT_RESET);
3967
3968                 udelay(50);
3969
3970                 tmp &= ~grbm_soft_reset;
3971                 WREG32(GRBM_SOFT_RESET, tmp);
3972                 tmp = RREG32(GRBM_SOFT_RESET);
3973         }
3974
3975         if (srbm_soft_reset) {
3976                 tmp = RREG32(SRBM_SOFT_RESET);
3977                 tmp |= srbm_soft_reset;
3978                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3979                 WREG32(SRBM_SOFT_RESET, tmp);
3980                 tmp = RREG32(SRBM_SOFT_RESET);
3981
3982                 udelay(50);
3983
3984                 tmp &= ~srbm_soft_reset;
3985                 WREG32(SRBM_SOFT_RESET, tmp);
3986                 tmp = RREG32(SRBM_SOFT_RESET);
3987         }
3988
3989         /* Wait a little for things to settle down */
3990         udelay(50);
3991
3992         evergreen_mc_resume(rdev, &save);
3993         udelay(50);
3994
3995         cik_print_gpu_status_regs(rdev);
3996 }
3997
3998 /**
3999  * cik_asic_reset - soft reset GPU
4000  *
4001  * @rdev: radeon_device pointer
4002  *
4003  * Look up which blocks are hung and attempt
4004  * to reset them.
4005  * Returns 0 for success.
4006  */
4007 int cik_asic_reset(struct radeon_device *rdev)
4008 {
4009         u32 reset_mask;
4010
4011         reset_mask = cik_gpu_check_soft_reset(rdev);
4012
4013         if (reset_mask)
4014                 r600_set_bios_scratch_engine_hung(rdev, true);
4015
4016         cik_gpu_soft_reset(rdev, reset_mask);
4017
4018         reset_mask = cik_gpu_check_soft_reset(rdev);
4019
4020         if (!reset_mask)
4021                 r600_set_bios_scratch_engine_hung(rdev, false);
4022
4023         return 0;
4024 }
4025
4026 /**
4027  * cik_gfx_is_lockup - check if the 3D engine is locked up
4028  *
4029  * @rdev: radeon_device pointer
4030  * @ring: radeon_ring structure holding ring information
4031  *
4032  * Check if the 3D engine is locked up (CIK).
4033  * Returns true if the engine is locked, false if not.
4034  */
4035 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4036 {
4037         u32 reset_mask = cik_gpu_check_soft_reset(rdev);
4038
4039         if (!(reset_mask & (RADEON_RESET_GFX |
4040                             RADEON_RESET_COMPUTE |
4041                             RADEON_RESET_CP))) {
4042                 radeon_ring_lockup_update(ring);
4043                 return false;
4044         }
4045         /* force CP activities */
4046         radeon_ring_force_activity(rdev, ring);
4047         return radeon_ring_test_lockup(rdev, ring);
4048 }
4049
4050 /**
4051  * cik_sdma_is_lockup - Check if the DMA engine is locked up
4052  *
4053  * @rdev: radeon_device pointer
4054  * @ring: radeon_ring structure holding ring information
4055  *
4056  * Check if the async DMA engine is locked up (CIK).
4057  * Returns true if the engine appears to be locked up, false if not.
4058  */
4059 bool cik_sdma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4060 {
4061         u32 reset_mask = cik_gpu_check_soft_reset(rdev);
4062         u32 mask;
4063
4064         if (ring->idx == R600_RING_TYPE_DMA_INDEX)
4065                 mask = RADEON_RESET_DMA;
4066         else
4067                 mask = RADEON_RESET_DMA1;
4068
4069         if (!(reset_mask & mask)) {
4070                 radeon_ring_lockup_update(ring);
4071                 return false;
4072         }
4073         /* force ring activities */
4074         radeon_ring_force_activity(rdev, ring);
4075         return radeon_ring_test_lockup(rdev, ring);
4076 }
4077
4078 /* MC */
4079 /**
4080  * cik_mc_program - program the GPU memory controller
4081  *
4082  * @rdev: radeon_device pointer
4083  *
4084  * Set the location of vram, gart, and AGP in the GPU's
4085  * physical address space (CIK).
4086  */
4087 static void cik_mc_program(struct radeon_device *rdev)
4088 {
4089         struct evergreen_mc_save save;
4090         u32 tmp;
4091         int i, j;
4092
4093         /* Initialize HDP */
4094         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
4095                 WREG32((0x2c14 + j), 0x00000000);
4096                 WREG32((0x2c18 + j), 0x00000000);
4097                 WREG32((0x2c1c + j), 0x00000000);
4098                 WREG32((0x2c20 + j), 0x00000000);
4099                 WREG32((0x2c24 + j), 0x00000000);
4100         }
4101         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
4102
4103         evergreen_mc_stop(rdev, &save);
4104         if (radeon_mc_wait_for_idle(rdev)) {
4105                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4106         }
4107         /* Lockout access through VGA aperture*/
4108         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
4109         /* Update configuration */
4110         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
4111                rdev->mc.vram_start >> 12);
4112         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
4113                rdev->mc.vram_end >> 12);
4114         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
4115                rdev->vram_scratch.gpu_addr >> 12);
4116         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
4117         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
4118         WREG32(MC_VM_FB_LOCATION, tmp);
4119         /* XXX double check these! */
4120         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
4121         WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
4122         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
4123         WREG32(MC_VM_AGP_BASE, 0);
4124         WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
4125         WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
4126         if (radeon_mc_wait_for_idle(rdev)) {
4127                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4128         }
4129         evergreen_mc_resume(rdev, &save);
4130         /* we need to own VRAM, so turn off the VGA renderer here
4131          * to stop it overwriting our objects */
4132         rv515_vga_render_disable(rdev);
4133 }
4134
4135 /**
4136  * cik_mc_init - initialize the memory controller driver params
4137  *
4138  * @rdev: radeon_device pointer
4139  *
4140  * Look up the amount of vram, vram width, and decide how to place
4141  * vram and gart within the GPU's physical address space (CIK).
4142  * Returns 0 for success.
4143  */
4144 static int cik_mc_init(struct radeon_device *rdev)
4145 {
4146         u32 tmp;
4147         int chansize, numchan;
4148
4149         /* Get VRAM informations */
4150         rdev->mc.vram_is_ddr = true;
4151         tmp = RREG32(MC_ARB_RAMCFG);
4152         if (tmp & CHANSIZE_MASK) {
4153                 chansize = 64;
4154         } else {
4155                 chansize = 32;
4156         }
4157         tmp = RREG32(MC_SHARED_CHMAP);
4158         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
4159         case 0:
4160         default:
4161                 numchan = 1;
4162                 break;
4163         case 1:
4164                 numchan = 2;
4165                 break;
4166         case 2:
4167                 numchan = 4;
4168                 break;
4169         case 3:
4170                 numchan = 8;
4171                 break;
4172         case 4:
4173                 numchan = 3;
4174                 break;
4175         case 5:
4176                 numchan = 6;
4177                 break;
4178         case 6:
4179                 numchan = 10;
4180                 break;
4181         case 7:
4182                 numchan = 12;
4183                 break;
4184         case 8:
4185                 numchan = 16;
4186                 break;
4187         }
4188         rdev->mc.vram_width = numchan * chansize;
4189         /* Could aper size report 0 ? */
4190         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
4191         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
4192         /* size in MB on si */
4193         rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
4194         rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
4195         rdev->mc.visible_vram_size = rdev->mc.aper_size;
4196         si_vram_gtt_location(rdev, &rdev->mc);
4197         radeon_update_bandwidth_info(rdev);
4198
4199         return 0;
4200 }
4201
4202 /*
4203  * GART
4204  * VMID 0 is the physical GPU addresses as used by the kernel.
4205  * VMIDs 1-15 are used for userspace clients and are handled
4206  * by the radeon vm/hsa code.
4207  */
4208 /**
4209  * cik_pcie_gart_tlb_flush - gart tlb flush callback
4210  *
4211  * @rdev: radeon_device pointer
4212  *
4213  * Flush the TLB for the VMID 0 page table (CIK).
4214  */
4215 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
4216 {
4217         /* flush hdp cache */
4218         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
4219
4220         /* bits 0-15 are the VM contexts0-15 */
4221         WREG32(VM_INVALIDATE_REQUEST, 0x1);
4222 }
4223
4224 /**
4225  * cik_pcie_gart_enable - gart enable
4226  *
4227  * @rdev: radeon_device pointer
4228  *
4229  * This sets up the TLBs, programs the page tables for VMID0,
4230  * sets up the hw for VMIDs 1-15 which are allocated on
4231  * demand, and sets up the global locations for the LDS, GDS,
4232  * and GPUVM for FSA64 clients (CIK).
4233  * Returns 0 for success, errors for failure.
4234  */
4235 static int cik_pcie_gart_enable(struct radeon_device *rdev)
4236 {
4237         int r, i;
4238
4239         if (rdev->gart.robj == NULL) {
4240                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
4241                 return -EINVAL;
4242         }
4243         r = radeon_gart_table_vram_pin(rdev);
4244         if (r)
4245                 return r;
4246         radeon_gart_restore(rdev);
4247         /* Setup TLB control */
4248         WREG32(MC_VM_MX_L1_TLB_CNTL,
4249                (0xA << 7) |
4250                ENABLE_L1_TLB |
4251                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4252                ENABLE_ADVANCED_DRIVER_MODEL |
4253                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4254         /* Setup L2 cache */
4255         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
4256                ENABLE_L2_FRAGMENT_PROCESSING |
4257                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4258                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4259                EFFECTIVE_L2_QUEUE_SIZE(7) |
4260                CONTEXT1_IDENTITY_ACCESS_MODE(1));
4261         WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
4262         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4263                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
4264         /* setup context0 */
4265         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
4266         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
4267         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
4268         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
4269                         (u32)(rdev->dummy_page.addr >> 12));
4270         WREG32(VM_CONTEXT0_CNTL2, 0);
4271         WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
4272                                   RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
4273
4274         WREG32(0x15D4, 0);
4275         WREG32(0x15D8, 0);
4276         WREG32(0x15DC, 0);
4277
4278         /* empty context1-15 */
4279         /* FIXME start with 4G, once using 2 level pt switch to full
4280          * vm size space
4281          */
4282         /* set vm size, must be a multiple of 4 */
4283         WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
4284         WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
4285         for (i = 1; i < 16; i++) {
4286                 if (i < 8)
4287                         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
4288                                rdev->gart.table_addr >> 12);
4289                 else
4290                         WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
4291                                rdev->gart.table_addr >> 12);
4292         }
4293
4294         /* enable context1-15 */
4295         WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
4296                (u32)(rdev->dummy_page.addr >> 12));
4297         WREG32(VM_CONTEXT1_CNTL2, 4);
4298         WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
4299                                 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4300                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4301                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4302                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4303                                 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
4304                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
4305                                 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
4306                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
4307                                 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
4308                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
4309                                 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4310                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
4311
4312         /* TC cache setup ??? */
4313         WREG32(TC_CFG_L1_LOAD_POLICY0, 0);
4314         WREG32(TC_CFG_L1_LOAD_POLICY1, 0);
4315         WREG32(TC_CFG_L1_STORE_POLICY, 0);
4316
4317         WREG32(TC_CFG_L2_LOAD_POLICY0, 0);
4318         WREG32(TC_CFG_L2_LOAD_POLICY1, 0);
4319         WREG32(TC_CFG_L2_STORE_POLICY0, 0);
4320         WREG32(TC_CFG_L2_STORE_POLICY1, 0);
4321         WREG32(TC_CFG_L2_ATOMIC_POLICY, 0);
4322
4323         WREG32(TC_CFG_L1_VOLATILE, 0);
4324         WREG32(TC_CFG_L2_VOLATILE, 0);
4325
4326         if (rdev->family == CHIP_KAVERI) {
4327                 u32 tmp = RREG32(CHUB_CONTROL);
4328                 tmp &= ~BYPASS_VM;
4329                 WREG32(CHUB_CONTROL, tmp);
4330         }
4331
4332         /* XXX SH_MEM regs */
4333         /* where to put LDS, scratch, GPUVM in FSA64 space */
4334         for (i = 0; i < 16; i++) {
4335                 cik_srbm_select(rdev, 0, 0, 0, i);
4336                 /* CP and shaders */
4337                 WREG32(SH_MEM_CONFIG, 0);
4338                 WREG32(SH_MEM_APE1_BASE, 1);
4339                 WREG32(SH_MEM_APE1_LIMIT, 0);
4340                 WREG32(SH_MEM_BASES, 0);
4341                 /* SDMA GFX */
4342                 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
4343                 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
4344                 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
4345                 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
4346                 /* XXX SDMA RLC - todo */
4347         }
4348         cik_srbm_select(rdev, 0, 0, 0, 0);
4349
4350         cik_pcie_gart_tlb_flush(rdev);
4351         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
4352                  (unsigned)(rdev->mc.gtt_size >> 20),
4353                  (unsigned long long)rdev->gart.table_addr);
4354         rdev->gart.ready = true;
4355         return 0;
4356 }
4357
4358 /**
4359  * cik_pcie_gart_disable - gart disable
4360  *
4361  * @rdev: radeon_device pointer
4362  *
4363  * This disables all VM page table (CIK).
4364  */
4365 static void cik_pcie_gart_disable(struct radeon_device *rdev)
4366 {
4367         /* Disable all tables */
4368         WREG32(VM_CONTEXT0_CNTL, 0);
4369         WREG32(VM_CONTEXT1_CNTL, 0);
4370         /* Setup TLB control */
4371         WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4372                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4373         /* Setup L2 cache */
4374         WREG32(VM_L2_CNTL,
4375                ENABLE_L2_FRAGMENT_PROCESSING |
4376                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4377                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4378                EFFECTIVE_L2_QUEUE_SIZE(7) |
4379                CONTEXT1_IDENTITY_ACCESS_MODE(1));
4380         WREG32(VM_L2_CNTL2, 0);
4381         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4382                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
4383         radeon_gart_table_vram_unpin(rdev);
4384 }
4385
4386 /**
4387  * cik_pcie_gart_fini - vm fini callback
4388  *
4389  * @rdev: radeon_device pointer
4390  *
4391  * Tears down the driver GART/VM setup (CIK).
4392  */
4393 static void cik_pcie_gart_fini(struct radeon_device *rdev)
4394 {
4395         cik_pcie_gart_disable(rdev);
4396         radeon_gart_table_vram_free(rdev);
4397         radeon_gart_fini(rdev);
4398 }
4399
4400 /* vm parser */
4401 /**
4402  * cik_ib_parse - vm ib_parse callback
4403  *
4404  * @rdev: radeon_device pointer
4405  * @ib: indirect buffer pointer
4406  *
4407  * CIK uses hw IB checking so this is a nop (CIK).
4408  */
4409 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4410 {
4411         return 0;
4412 }
4413
4414 /*
4415  * vm
4416  * VMID 0 is the physical GPU addresses as used by the kernel.
4417  * VMIDs 1-15 are used for userspace clients and are handled
4418  * by the radeon vm/hsa code.
4419  */
4420 /**
4421  * cik_vm_init - cik vm init callback
4422  *
4423  * @rdev: radeon_device pointer
4424  *
4425  * Inits cik specific vm parameters (number of VMs, base of vram for
4426  * VMIDs 1-15) (CIK).
4427  * Returns 0 for success.
4428  */
4429 int cik_vm_init(struct radeon_device *rdev)
4430 {
4431         /* number of VMs */
4432         rdev->vm_manager.nvm = 16;
4433         /* base offset of vram pages */
4434         if (rdev->flags & RADEON_IS_IGP) {
4435                 u64 tmp = RREG32(MC_VM_FB_OFFSET);
4436                 tmp <<= 22;
4437                 rdev->vm_manager.vram_base_offset = tmp;
4438         } else
4439                 rdev->vm_manager.vram_base_offset = 0;
4440
4441         return 0;
4442 }
4443
4444 /**
4445  * cik_vm_fini - cik vm fini callback
4446  *
4447  * @rdev: radeon_device pointer
4448  *
4449  * Tear down any asic specific VM setup (CIK).
4450  */
4451 void cik_vm_fini(struct radeon_device *rdev)
4452 {
4453 }
4454
4455 /**
4456  * cik_vm_flush - cik vm flush using the CP
4457  *
4458  * @rdev: radeon_device pointer
4459  *
4460  * Update the page table base and flush the VM TLB
4461  * using the CP (CIK).
4462  */
4463 void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
4464 {
4465         struct radeon_ring *ring = &rdev->ring[ridx];
4466
4467         if (vm == NULL)
4468                 return;
4469
4470         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4471         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4472                                  WRITE_DATA_DST_SEL(0)));
4473         if (vm->id < 8) {
4474                 radeon_ring_write(ring,
4475                                   (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
4476         } else {
4477                 radeon_ring_write(ring,
4478                                   (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
4479         }
4480         radeon_ring_write(ring, 0);
4481         radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
4482
4483         /* update SH_MEM_* regs */
4484         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4485         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4486                                  WRITE_DATA_DST_SEL(0)));
4487         radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
4488         radeon_ring_write(ring, 0);
4489         radeon_ring_write(ring, VMID(vm->id));
4490
4491         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
4492         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4493                                  WRITE_DATA_DST_SEL(0)));
4494         radeon_ring_write(ring, SH_MEM_BASES >> 2);
4495         radeon_ring_write(ring, 0);
4496
4497         radeon_ring_write(ring, 0); /* SH_MEM_BASES */
4498         radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
4499         radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
4500         radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
4501
4502         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4503         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4504                                  WRITE_DATA_DST_SEL(0)));
4505         radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
4506         radeon_ring_write(ring, 0);
4507         radeon_ring_write(ring, VMID(0));
4508
4509         /* HDP flush */
4510         /* We should be using the WAIT_REG_MEM packet here like in
4511          * cik_fence_ring_emit(), but it causes the CP to hang in this
4512          * context...
4513          */
4514         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4515         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4516                                  WRITE_DATA_DST_SEL(0)));
4517         radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
4518         radeon_ring_write(ring, 0);
4519         radeon_ring_write(ring, 0);
4520
4521         /* bits 0-15 are the VM contexts0-15 */
4522         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4523         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4524                                  WRITE_DATA_DST_SEL(0)));
4525         radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
4526         radeon_ring_write(ring, 0);
4527         radeon_ring_write(ring, 1 << vm->id);
4528
4529         /* compute doesn't have PFP */
4530         if (ridx == RADEON_RING_TYPE_GFX_INDEX) {
4531                 /* sync PFP to ME, otherwise we might get invalid PFP reads */
4532                 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4533                 radeon_ring_write(ring, 0x0);
4534         }
4535 }
4536
4537 /**
4538  * cik_vm_set_page - update the page tables using sDMA
4539  *
4540  * @rdev: radeon_device pointer
4541  * @ib: indirect buffer to fill with commands
4542  * @pe: addr of the page entry
4543  * @addr: dst addr to write into pe
4544  * @count: number of page entries to update
4545  * @incr: increase next addr by incr bytes
4546  * @flags: access flags
4547  *
4548  * Update the page tables using CP or sDMA (CIK).
4549  */
4550 void cik_vm_set_page(struct radeon_device *rdev,
4551                      struct radeon_ib *ib,
4552                      uint64_t pe,
4553                      uint64_t addr, unsigned count,
4554                      uint32_t incr, uint32_t flags)
4555 {
4556         uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
4557         uint64_t value;
4558         unsigned ndw;
4559
4560         if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) {
4561                 /* CP */
4562                 while (count) {
4563                         ndw = 2 + count * 2;
4564                         if (ndw > 0x3FFE)
4565                                 ndw = 0x3FFE;
4566
4567                         ib->ptr[ib->length_dw++] = PACKET3(PACKET3_WRITE_DATA, ndw);
4568                         ib->ptr[ib->length_dw++] = (WRITE_DATA_ENGINE_SEL(0) |
4569                                                     WRITE_DATA_DST_SEL(1));
4570                         ib->ptr[ib->length_dw++] = pe;
4571                         ib->ptr[ib->length_dw++] = upper_32_bits(pe);
4572                         for (; ndw > 2; ndw -= 2, --count, pe += 8) {
4573                                 if (flags & RADEON_VM_PAGE_SYSTEM) {
4574                                         value = radeon_vm_map_gart(rdev, addr);
4575                                         value &= 0xFFFFFFFFFFFFF000ULL;
4576                                 } else if (flags & RADEON_VM_PAGE_VALID) {
4577                                         value = addr;
4578                                 } else {
4579                                         value = 0;
4580                                 }
4581                                 addr += incr;
4582                                 value |= r600_flags;
4583                                 ib->ptr[ib->length_dw++] = value;
4584                                 ib->ptr[ib->length_dw++] = upper_32_bits(value);
4585                         }
4586                 }
4587         } else {
4588                 /* DMA */
4589                 if (flags & RADEON_VM_PAGE_SYSTEM) {
4590                         while (count) {
4591                                 ndw = count * 2;
4592                                 if (ndw > 0xFFFFE)
4593                                         ndw = 0xFFFFE;
4594
4595                                 /* for non-physically contiguous pages (system) */
4596                                 ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0);
4597                                 ib->ptr[ib->length_dw++] = pe;
4598                                 ib->ptr[ib->length_dw++] = upper_32_bits(pe);
4599                                 ib->ptr[ib->length_dw++] = ndw;
4600                                 for (; ndw > 0; ndw -= 2, --count, pe += 8) {
4601                                         if (flags & RADEON_VM_PAGE_SYSTEM) {
4602                                                 value = radeon_vm_map_gart(rdev, addr);
4603                                                 value &= 0xFFFFFFFFFFFFF000ULL;
4604                                         } else if (flags & RADEON_VM_PAGE_VALID) {
4605                                                 value = addr;
4606                                         } else {
4607                                                 value = 0;
4608                                         }
4609                                         addr += incr;
4610                                         value |= r600_flags;
4611                                         ib->ptr[ib->length_dw++] = value;
4612                                         ib->ptr[ib->length_dw++] = upper_32_bits(value);
4613                                 }
4614                         }
4615                 } else {
4616                         while (count) {
4617                                 ndw = count;
4618                                 if (ndw > 0x7FFFF)
4619                                         ndw = 0x7FFFF;
4620
4621                                 if (flags & RADEON_VM_PAGE_VALID)
4622                                         value = addr;
4623                                 else
4624                                         value = 0;
4625                                 /* for physically contiguous pages (vram) */
4626                                 ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_GENERATE_PTE_PDE, 0, 0);
4627                                 ib->ptr[ib->length_dw++] = pe; /* dst addr */
4628                                 ib->ptr[ib->length_dw++] = upper_32_bits(pe);
4629                                 ib->ptr[ib->length_dw++] = r600_flags; /* mask */
4630                                 ib->ptr[ib->length_dw++] = 0;
4631                                 ib->ptr[ib->length_dw++] = value; /* value */
4632                                 ib->ptr[ib->length_dw++] = upper_32_bits(value);
4633                                 ib->ptr[ib->length_dw++] = incr; /* increment size */
4634                                 ib->ptr[ib->length_dw++] = 0;
4635                                 ib->ptr[ib->length_dw++] = ndw; /* number of entries */
4636                                 pe += ndw * 8;
4637                                 addr += ndw * incr;
4638                                 count -= ndw;
4639                         }
4640                 }
4641                 while (ib->length_dw & 0x7)
4642                         ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0);
4643         }
4644 }
4645
4646 /**
4647  * cik_dma_vm_flush - cik vm flush using sDMA
4648  *
4649  * @rdev: radeon_device pointer
4650  *
4651  * Update the page table base and flush the VM TLB
4652  * using sDMA (CIK).
4653  */
4654 void cik_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
4655 {
4656         struct radeon_ring *ring = &rdev->ring[ridx];
4657         u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(1) |
4658                           SDMA_POLL_REG_MEM_EXTRA_FUNC(3)); /* == */
4659         u32 ref_and_mask;
4660
4661         if (vm == NULL)
4662                 return;
4663
4664         if (ridx == R600_RING_TYPE_DMA_INDEX)
4665                 ref_and_mask = SDMA0;
4666         else
4667                 ref_and_mask = SDMA1;
4668
4669         radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
4670         if (vm->id < 8) {
4671                 radeon_ring_write(ring, (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
4672         } else {
4673                 radeon_ring_write(ring, (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
4674         }
4675         radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
4676
4677         /* update SH_MEM_* regs */
4678         radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
4679         radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
4680         radeon_ring_write(ring, VMID(vm->id));
4681
4682         radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
4683         radeon_ring_write(ring, SH_MEM_BASES >> 2);
4684         radeon_ring_write(ring, 0);
4685
4686         radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
4687         radeon_ring_write(ring, SH_MEM_CONFIG >> 2);
4688         radeon_ring_write(ring, 0);
4689
4690         radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
4691         radeon_ring_write(ring, SH_MEM_APE1_BASE >> 2);
4692         radeon_ring_write(ring, 1);
4693
4694         radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
4695         radeon_ring_write(ring, SH_MEM_APE1_LIMIT >> 2);
4696         radeon_ring_write(ring, 0);
4697
4698         radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
4699         radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
4700         radeon_ring_write(ring, VMID(0));
4701
4702         /* flush HDP */
4703         radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, extra_bits));
4704         radeon_ring_write(ring, GPU_HDP_FLUSH_DONE);
4705         radeon_ring_write(ring, GPU_HDP_FLUSH_REQ);
4706         radeon_ring_write(ring, ref_and_mask); /* REFERENCE */
4707         radeon_ring_write(ring, ref_and_mask); /* MASK */
4708         radeon_ring_write(ring, (4 << 16) | 10); /* RETRY_COUNT, POLL_INTERVAL */
4709
4710         /* flush TLB */
4711         radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
4712         radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
4713         radeon_ring_write(ring, 1 << vm->id);
4714 }
4715
4716 /*
4717  * RLC
4718  * The RLC is a multi-purpose microengine that handles a
4719  * variety of functions, the most important of which is
4720  * the interrupt controller.
4721  */
4722 /**
4723  * cik_rlc_stop - stop the RLC ME
4724  *
4725  * @rdev: radeon_device pointer
4726  *
4727  * Halt the RLC ME (MicroEngine) (CIK).
4728  */
4729 static void cik_rlc_stop(struct radeon_device *rdev)
4730 {
4731         int i, j, k;
4732         u32 mask, tmp;
4733
4734         tmp = RREG32(CP_INT_CNTL_RING0);
4735         tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4736         WREG32(CP_INT_CNTL_RING0, tmp);
4737
4738         RREG32(CB_CGTT_SCLK_CTRL);
4739         RREG32(CB_CGTT_SCLK_CTRL);
4740         RREG32(CB_CGTT_SCLK_CTRL);
4741         RREG32(CB_CGTT_SCLK_CTRL);
4742
4743         tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
4744         WREG32(RLC_CGCG_CGLS_CTRL, tmp);
4745
4746         WREG32(RLC_CNTL, 0);
4747
4748         for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
4749                 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
4750                         cik_select_se_sh(rdev, i, j);
4751                         for (k = 0; k < rdev->usec_timeout; k++) {
4752                                 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
4753                                         break;
4754                                 udelay(1);
4755                         }
4756                 }
4757         }
4758         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
4759
4760         mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
4761         for (k = 0; k < rdev->usec_timeout; k++) {
4762                 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
4763                         break;
4764                 udelay(1);
4765         }
4766 }
4767
4768 /**
4769  * cik_rlc_start - start the RLC ME
4770  *
4771  * @rdev: radeon_device pointer
4772  *
4773  * Unhalt the RLC ME (MicroEngine) (CIK).
4774  */
4775 static void cik_rlc_start(struct radeon_device *rdev)
4776 {
4777         u32 tmp;
4778
4779         WREG32(RLC_CNTL, RLC_ENABLE);
4780
4781         tmp = RREG32(CP_INT_CNTL_RING0);
4782         tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4783         WREG32(CP_INT_CNTL_RING0, tmp);
4784
4785         udelay(50);
4786 }
4787
4788 /**
4789  * cik_rlc_resume - setup the RLC hw
4790  *
4791  * @rdev: radeon_device pointer
4792  *
4793  * Initialize the RLC registers, load the ucode,
4794  * and start the RLC (CIK).
4795  * Returns 0 for success, -EINVAL if the ucode is not available.
4796  */
4797 static int cik_rlc_resume(struct radeon_device *rdev)
4798 {
4799         u32 i, size;
4800         u32 clear_state_info[3];
4801         const __be32 *fw_data;
4802
4803         if (!rdev->rlc_fw)
4804                 return -EINVAL;
4805
4806         switch (rdev->family) {
4807         case CHIP_BONAIRE:
4808         default:
4809                 size = BONAIRE_RLC_UCODE_SIZE;
4810                 break;
4811         case CHIP_KAVERI:
4812                 size = KV_RLC_UCODE_SIZE;
4813                 break;
4814         case CHIP_KABINI:
4815                 size = KB_RLC_UCODE_SIZE;
4816                 break;
4817         }
4818
4819         cik_rlc_stop(rdev);
4820
4821         WREG32(GRBM_SOFT_RESET, SOFT_RESET_RLC);
4822         RREG32(GRBM_SOFT_RESET);
4823         udelay(50);
4824         WREG32(GRBM_SOFT_RESET, 0);
4825         RREG32(GRBM_SOFT_RESET);
4826         udelay(50);
4827
4828         WREG32(RLC_LB_CNTR_INIT, 0);
4829         WREG32(RLC_LB_CNTR_MAX, 0x00008000);
4830
4831         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
4832         WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
4833         WREG32(RLC_LB_PARAMS, 0x00600408);
4834         WREG32(RLC_LB_CNTL, 0x80000004);
4835
4836         WREG32(RLC_MC_CNTL, 0);
4837         WREG32(RLC_UCODE_CNTL, 0);
4838
4839         fw_data = (const __be32 *)rdev->rlc_fw->data;
4840                 WREG32(RLC_GPM_UCODE_ADDR, 0);
4841         for (i = 0; i < size; i++)
4842                 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
4843         WREG32(RLC_GPM_UCODE_ADDR, 0);
4844
4845         /* XXX */
4846         clear_state_info[0] = 0;//upper_32_bits(rdev->rlc.save_restore_gpu_addr);
4847         clear_state_info[1] = 0;//rdev->rlc.save_restore_gpu_addr;
4848         clear_state_info[2] = 0;//cik_default_size;
4849         WREG32(RLC_GPM_SCRATCH_ADDR, 0x3d);
4850         for (i = 0; i < 3; i++)
4851                 WREG32(RLC_GPM_SCRATCH_DATA, clear_state_info[i]);
4852         WREG32(RLC_DRIVER_DMA_STATUS, 0);
4853
4854         cik_rlc_start(rdev);
4855
4856         return 0;
4857 }
4858
4859 /*
4860  * Interrupts
4861  * Starting with r6xx, interrupts are handled via a ring buffer.
4862  * Ring buffers are areas of GPU accessible memory that the GPU
4863  * writes interrupt vectors into and the host reads vectors out of.
4864  * There is a rptr (read pointer) that determines where the
4865  * host is currently reading, and a wptr (write pointer)
4866  * which determines where the GPU has written.  When the
4867  * pointers are equal, the ring is idle.  When the GPU
4868  * writes vectors to the ring buffer, it increments the
4869  * wptr.  When there is an interrupt, the host then starts
4870  * fetching commands and processing them until the pointers are
4871  * equal again at which point it updates the rptr.
4872  */
4873
4874 /**
4875  * cik_enable_interrupts - Enable the interrupt ring buffer
4876  *
4877  * @rdev: radeon_device pointer
4878  *
4879  * Enable the interrupt ring buffer (CIK).
4880  */
4881 static void cik_enable_interrupts(struct radeon_device *rdev)
4882 {
4883         u32 ih_cntl = RREG32(IH_CNTL);
4884         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
4885
4886         ih_cntl |= ENABLE_INTR;
4887         ih_rb_cntl |= IH_RB_ENABLE;
4888         WREG32(IH_CNTL, ih_cntl);
4889         WREG32(IH_RB_CNTL, ih_rb_cntl);
4890         rdev->ih.enabled = true;
4891 }
4892
4893 /**
4894  * cik_disable_interrupts - Disable the interrupt ring buffer
4895  *
4896  * @rdev: radeon_device pointer
4897  *
4898  * Disable the interrupt ring buffer (CIK).
4899  */
4900 static void cik_disable_interrupts(struct radeon_device *rdev)
4901 {
4902         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
4903         u32 ih_cntl = RREG32(IH_CNTL);
4904
4905         ih_rb_cntl &= ~IH_RB_ENABLE;
4906         ih_cntl &= ~ENABLE_INTR;
4907         WREG32(IH_RB_CNTL, ih_rb_cntl);
4908         WREG32(IH_CNTL, ih_cntl);
4909         /* set rptr, wptr to 0 */
4910         WREG32(IH_RB_RPTR, 0);
4911         WREG32(IH_RB_WPTR, 0);
4912         rdev->ih.enabled = false;
4913         rdev->ih.rptr = 0;
4914 }
4915
4916 /**
4917  * cik_disable_interrupt_state - Disable all interrupt sources
4918  *
4919  * @rdev: radeon_device pointer
4920  *
4921  * Clear all interrupt enable bits used by the driver (CIK).
4922  */
4923 static void cik_disable_interrupt_state(struct radeon_device *rdev)
4924 {
4925         u32 tmp;
4926
4927         /* gfx ring */
4928         WREG32(CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4929         /* sdma */
4930         tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
4931         WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4932         tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
4933         WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
4934         /* compute queues */
4935         WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
4936         WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
4937         WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
4938         WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
4939         WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
4940         WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
4941         WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
4942         WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
4943         /* grbm */
4944         WREG32(GRBM_INT_CNTL, 0);
4945         /* vline/vblank, etc. */
4946         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4947         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4948         if (rdev->num_crtc >= 4) {
4949                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4950                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4951         }
4952         if (rdev->num_crtc >= 6) {
4953                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4954                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4955         }
4956
4957         /* dac hotplug */
4958         WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
4959
4960         /* digital hotplug */
4961         tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4962         WREG32(DC_HPD1_INT_CONTROL, tmp);
4963         tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4964         WREG32(DC_HPD2_INT_CONTROL, tmp);
4965         tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4966         WREG32(DC_HPD3_INT_CONTROL, tmp);
4967         tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4968         WREG32(DC_HPD4_INT_CONTROL, tmp);
4969         tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4970         WREG32(DC_HPD5_INT_CONTROL, tmp);
4971         tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4972         WREG32(DC_HPD6_INT_CONTROL, tmp);
4973
4974 }
4975
4976 /**
4977  * cik_irq_init - init and enable the interrupt ring
4978  *
4979  * @rdev: radeon_device pointer
4980  *
4981  * Allocate a ring buffer for the interrupt controller,
4982  * enable the RLC, disable interrupts, enable the IH
4983  * ring buffer and enable it (CIK).
4984  * Called at device load and reume.
4985  * Returns 0 for success, errors for failure.
4986  */
4987 static int cik_irq_init(struct radeon_device *rdev)
4988 {
4989         int ret = 0;
4990         int rb_bufsz;
4991         u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
4992
4993         /* allocate ring */
4994         ret = r600_ih_ring_alloc(rdev);
4995         if (ret)
4996                 return ret;
4997
4998         /* disable irqs */
4999         cik_disable_interrupts(rdev);
5000
5001         /* init rlc */
5002         ret = cik_rlc_resume(rdev);
5003         if (ret) {
5004                 r600_ih_ring_fini(rdev);
5005                 return ret;
5006         }
5007
5008         /* setup interrupt control */
5009         /* XXX this should actually be a bus address, not an MC address. same on older asics */
5010         WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
5011         interrupt_cntl = RREG32(INTERRUPT_CNTL);
5012         /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
5013          * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
5014          */
5015         interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
5016         /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
5017         interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
5018         WREG32(INTERRUPT_CNTL, interrupt_cntl);
5019
5020         WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
5021         rb_bufsz = drm_order(rdev->ih.ring_size / 4);
5022
5023         ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
5024                       IH_WPTR_OVERFLOW_CLEAR |
5025                       (rb_bufsz << 1));
5026
5027         if (rdev->wb.enabled)
5028                 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
5029
5030         /* set the writeback address whether it's enabled or not */
5031         WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
5032         WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
5033
5034         WREG32(IH_RB_CNTL, ih_rb_cntl);
5035
5036         /* set rptr, wptr to 0 */
5037         WREG32(IH_RB_RPTR, 0);
5038         WREG32(IH_RB_WPTR, 0);
5039
5040         /* Default settings for IH_CNTL (disabled at first) */
5041         ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
5042         /* RPTR_REARM only works if msi's are enabled */
5043         if (rdev->msi_enabled)
5044                 ih_cntl |= RPTR_REARM;
5045         WREG32(IH_CNTL, ih_cntl);
5046
5047         /* force the active interrupt state to all disabled */
5048         cik_disable_interrupt_state(rdev);
5049
5050         pci_set_master(rdev->pdev);
5051
5052         /* enable irqs */
5053         cik_enable_interrupts(rdev);
5054
5055         return ret;
5056 }
5057
5058 /**
5059  * cik_irq_set - enable/disable interrupt sources
5060  *
5061  * @rdev: radeon_device pointer
5062  *
5063  * Enable interrupt sources on the GPU (vblanks, hpd,
5064  * etc.) (CIK).
5065  * Returns 0 for success, errors for failure.
5066  */
5067 int cik_irq_set(struct radeon_device *rdev)
5068 {
5069         u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE |
5070                 PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
5071         u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
5072         u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
5073         u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
5074         u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
5075         u32 grbm_int_cntl = 0;
5076         u32 dma_cntl, dma_cntl1;
5077
5078         if (!rdev->irq.installed) {
5079                 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
5080                 return -EINVAL;
5081         }
5082         /* don't enable anything if the ih is disabled */
5083         if (!rdev->ih.enabled) {
5084                 cik_disable_interrupts(rdev);
5085                 /* force the active interrupt state to all disabled */
5086                 cik_disable_interrupt_state(rdev);
5087                 return 0;
5088         }
5089
5090         hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
5091         hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
5092         hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
5093         hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
5094         hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
5095         hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
5096
5097         dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5098         dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5099
5100         cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
5101         cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
5102         cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
5103         cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
5104         cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
5105         cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
5106         cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
5107         cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
5108
5109         /* enable CP interrupts on all rings */
5110         if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
5111                 DRM_DEBUG("cik_irq_set: sw int gfx\n");
5112                 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
5113         }
5114         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
5115                 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
5116                 DRM_DEBUG("si_irq_set: sw int cp1\n");
5117                 if (ring->me == 1) {
5118                         switch (ring->pipe) {
5119                         case 0:
5120                                 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
5121                                 break;
5122                         case 1:
5123                                 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
5124                                 break;
5125                         case 2:
5126                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
5127                                 break;
5128                         case 3:
5129                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
5130                                 break;
5131                         default:
5132                                 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
5133                                 break;
5134                         }
5135                 } else if (ring->me == 2) {
5136                         switch (ring->pipe) {
5137                         case 0:
5138                                 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
5139                                 break;
5140                         case 1:
5141                                 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
5142                                 break;
5143                         case 2:
5144                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
5145                                 break;
5146                         case 3:
5147                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
5148                                 break;
5149                         default:
5150                                 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
5151                                 break;
5152                         }
5153                 } else {
5154                         DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
5155                 }
5156         }
5157         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
5158                 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
5159                 DRM_DEBUG("si_irq_set: sw int cp2\n");
5160                 if (ring->me == 1) {
5161                         switch (ring->pipe) {
5162                         case 0:
5163                                 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
5164                                 break;
5165                         case 1:
5166                                 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
5167                                 break;
5168                         case 2:
5169                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
5170                                 break;
5171                         case 3:
5172                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
5173                                 break;
5174                         default:
5175                                 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
5176                                 break;
5177                         }
5178                 } else if (ring->me == 2) {
5179                         switch (ring->pipe) {
5180                         case 0:
5181                                 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
5182                                 break;
5183                         case 1:
5184                                 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
5185                                 break;
5186                         case 2:
5187                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
5188                                 break;
5189                         case 3:
5190                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
5191                                 break;
5192                         default:
5193                                 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
5194                                 break;
5195                         }
5196                 } else {
5197                         DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
5198                 }
5199         }
5200
5201         if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
5202                 DRM_DEBUG("cik_irq_set: sw int dma\n");
5203                 dma_cntl |= TRAP_ENABLE;
5204         }
5205
5206         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
5207                 DRM_DEBUG("cik_irq_set: sw int dma1\n");
5208                 dma_cntl1 |= TRAP_ENABLE;
5209         }
5210
5211         if (rdev->irq.crtc_vblank_int[0] ||
5212             atomic_read(&rdev->irq.pflip[0])) {
5213                 DRM_DEBUG("cik_irq_set: vblank 0\n");
5214                 crtc1 |= VBLANK_INTERRUPT_MASK;
5215         }
5216         if (rdev->irq.crtc_vblank_int[1] ||
5217             atomic_read(&rdev->irq.pflip[1])) {
5218                 DRM_DEBUG("cik_irq_set: vblank 1\n");
5219                 crtc2 |= VBLANK_INTERRUPT_MASK;
5220         }
5221         if (rdev->irq.crtc_vblank_int[2] ||
5222             atomic_read(&rdev->irq.pflip[2])) {
5223                 DRM_DEBUG("cik_irq_set: vblank 2\n");
5224                 crtc3 |= VBLANK_INTERRUPT_MASK;
5225         }
5226         if (rdev->irq.crtc_vblank_int[3] ||
5227             atomic_read(&rdev->irq.pflip[3])) {
5228                 DRM_DEBUG("cik_irq_set: vblank 3\n");
5229                 crtc4 |= VBLANK_INTERRUPT_MASK;
5230         }
5231         if (rdev->irq.crtc_vblank_int[4] ||
5232             atomic_read(&rdev->irq.pflip[4])) {
5233                 DRM_DEBUG("cik_irq_set: vblank 4\n");
5234                 crtc5 |= VBLANK_INTERRUPT_MASK;
5235         }
5236         if (rdev->irq.crtc_vblank_int[5] ||
5237             atomic_read(&rdev->irq.pflip[5])) {
5238                 DRM_DEBUG("cik_irq_set: vblank 5\n");
5239                 crtc6 |= VBLANK_INTERRUPT_MASK;
5240         }
5241         if (rdev->irq.hpd[0]) {
5242                 DRM_DEBUG("cik_irq_set: hpd 1\n");
5243                 hpd1 |= DC_HPDx_INT_EN;
5244         }
5245         if (rdev->irq.hpd[1]) {
5246                 DRM_DEBUG("cik_irq_set: hpd 2\n");
5247                 hpd2 |= DC_HPDx_INT_EN;
5248         }
5249         if (rdev->irq.hpd[2]) {
5250                 DRM_DEBUG("cik_irq_set: hpd 3\n");
5251                 hpd3 |= DC_HPDx_INT_EN;
5252         }
5253         if (rdev->irq.hpd[3]) {
5254                 DRM_DEBUG("cik_irq_set: hpd 4\n");
5255                 hpd4 |= DC_HPDx_INT_EN;
5256         }
5257         if (rdev->irq.hpd[4]) {
5258                 DRM_DEBUG("cik_irq_set: hpd 5\n");
5259                 hpd5 |= DC_HPDx_INT_EN;
5260         }
5261         if (rdev->irq.hpd[5]) {
5262                 DRM_DEBUG("cik_irq_set: hpd 6\n");
5263                 hpd6 |= DC_HPDx_INT_EN;
5264         }
5265
5266         WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
5267
5268         WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
5269         WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
5270
5271         WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
5272         WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
5273         WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
5274         WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
5275         WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
5276         WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
5277         WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
5278         WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
5279
5280         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
5281
5282         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
5283         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
5284         if (rdev->num_crtc >= 4) {
5285                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
5286                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
5287         }
5288         if (rdev->num_crtc >= 6) {
5289                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
5290                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
5291         }
5292
5293         WREG32(DC_HPD1_INT_CONTROL, hpd1);
5294         WREG32(DC_HPD2_INT_CONTROL, hpd2);
5295         WREG32(DC_HPD3_INT_CONTROL, hpd3);
5296         WREG32(DC_HPD4_INT_CONTROL, hpd4);
5297         WREG32(DC_HPD5_INT_CONTROL, hpd5);
5298         WREG32(DC_HPD6_INT_CONTROL, hpd6);
5299
5300         return 0;
5301 }
5302
5303 /**
5304  * cik_irq_ack - ack interrupt sources
5305  *
5306  * @rdev: radeon_device pointer
5307  *
5308  * Ack interrupt sources on the GPU (vblanks, hpd,
5309  * etc.) (CIK).  Certain interrupts sources are sw
5310  * generated and do not require an explicit ack.
5311  */
5312 static inline void cik_irq_ack(struct radeon_device *rdev)
5313 {
5314         u32 tmp;
5315
5316         rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
5317         rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
5318         rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
5319         rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
5320         rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
5321         rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
5322         rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
5323
5324         if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
5325                 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
5326         if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
5327                 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
5328         if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
5329                 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
5330         if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
5331                 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
5332
5333         if (rdev->num_crtc >= 4) {
5334                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
5335                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
5336                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
5337                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
5338                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
5339                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
5340                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
5341                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
5342         }
5343
5344         if (rdev->num_crtc >= 6) {
5345                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
5346                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
5347                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
5348                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
5349                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
5350                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
5351                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
5352                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
5353         }
5354
5355         if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
5356                 tmp = RREG32(DC_HPD1_INT_CONTROL);
5357                 tmp |= DC_HPDx_INT_ACK;
5358                 WREG32(DC_HPD1_INT_CONTROL, tmp);
5359         }
5360         if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
5361                 tmp = RREG32(DC_HPD2_INT_CONTROL);
5362                 tmp |= DC_HPDx_INT_ACK;
5363                 WREG32(DC_HPD2_INT_CONTROL, tmp);
5364         }
5365         if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
5366                 tmp = RREG32(DC_HPD3_INT_CONTROL);
5367                 tmp |= DC_HPDx_INT_ACK;
5368                 WREG32(DC_HPD3_INT_CONTROL, tmp);
5369         }
5370         if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
5371                 tmp = RREG32(DC_HPD4_INT_CONTROL);
5372                 tmp |= DC_HPDx_INT_ACK;
5373                 WREG32(DC_HPD4_INT_CONTROL, tmp);
5374         }
5375         if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
5376                 tmp = RREG32(DC_HPD5_INT_CONTROL);
5377                 tmp |= DC_HPDx_INT_ACK;
5378                 WREG32(DC_HPD5_INT_CONTROL, tmp);
5379         }
5380         if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
5381                 tmp = RREG32(DC_HPD5_INT_CONTROL);
5382                 tmp |= DC_HPDx_INT_ACK;
5383                 WREG32(DC_HPD6_INT_CONTROL, tmp);
5384         }
5385 }
5386
5387 /**
5388  * cik_irq_disable - disable interrupts
5389  *
5390  * @rdev: radeon_device pointer
5391  *
5392  * Disable interrupts on the hw (CIK).
5393  */
5394 static void cik_irq_disable(struct radeon_device *rdev)
5395 {
5396         cik_disable_interrupts(rdev);
5397         /* Wait and acknowledge irq */
5398         mdelay(1);
5399         cik_irq_ack(rdev);
5400         cik_disable_interrupt_state(rdev);
5401 }
5402
5403 /**
5404  * cik_irq_disable - disable interrupts for suspend
5405  *
5406  * @rdev: radeon_device pointer
5407  *
5408  * Disable interrupts and stop the RLC (CIK).
5409  * Used for suspend.
5410  */
5411 static void cik_irq_suspend(struct radeon_device *rdev)
5412 {
5413         cik_irq_disable(rdev);
5414         cik_rlc_stop(rdev);
5415 }
5416
5417 /**
5418  * cik_irq_fini - tear down interrupt support
5419  *
5420  * @rdev: radeon_device pointer
5421  *
5422  * Disable interrupts on the hw and free the IH ring
5423  * buffer (CIK).
5424  * Used for driver unload.
5425  */
5426 static void cik_irq_fini(struct radeon_device *rdev)
5427 {
5428         cik_irq_suspend(rdev);
5429         r600_ih_ring_fini(rdev);
5430 }
5431
5432 /**
5433  * cik_get_ih_wptr - get the IH ring buffer wptr
5434  *
5435  * @rdev: radeon_device pointer
5436  *
5437  * Get the IH ring buffer wptr from either the register
5438  * or the writeback memory buffer (CIK).  Also check for
5439  * ring buffer overflow and deal with it.
5440  * Used by cik_irq_process().
5441  * Returns the value of the wptr.
5442  */
5443 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
5444 {
5445         u32 wptr, tmp;
5446
5447         if (rdev->wb.enabled)
5448                 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
5449         else
5450                 wptr = RREG32(IH_RB_WPTR);
5451
5452         if (wptr & RB_OVERFLOW) {
5453                 /* When a ring buffer overflow happen start parsing interrupt
5454                  * from the last not overwritten vector (wptr + 16). Hopefully
5455                  * this should allow us to catchup.
5456                  */
5457                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
5458                         wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
5459                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
5460                 tmp = RREG32(IH_RB_CNTL);
5461                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
5462                 WREG32(IH_RB_CNTL, tmp);
5463         }
5464         return (wptr & rdev->ih.ptr_mask);
5465 }
5466
5467 /*        CIK IV Ring
5468  * Each IV ring entry is 128 bits:
5469  * [7:0]    - interrupt source id
5470  * [31:8]   - reserved
5471  * [59:32]  - interrupt source data
5472  * [63:60]  - reserved
5473  * [71:64]  - RINGID
5474  *            CP:
5475  *            ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
5476  *            QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
5477  *                     - for gfx, hw shader state (0=PS...5=LS, 6=CS)
5478  *            ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
5479  *            PIPE_ID - ME0 0=3D
5480  *                    - ME1&2 compute dispatcher (4 pipes each)
5481  *            SDMA:
5482  *            INSTANCE_ID [1:0], QUEUE_ID[1:0]
5483  *            INSTANCE_ID - 0 = sdma0, 1 = sdma1
5484  *            QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
5485  * [79:72]  - VMID
5486  * [95:80]  - PASID
5487  * [127:96] - reserved
5488  */
5489 /**
5490  * cik_irq_process - interrupt handler
5491  *
5492  * @rdev: radeon_device pointer
5493  *
5494  * Interrupt hander (CIK).  Walk the IH ring,
5495  * ack interrupts and schedule work to handle
5496  * interrupt events.
5497  * Returns irq process return code.
5498  */
5499 int cik_irq_process(struct radeon_device *rdev)
5500 {
5501         struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
5502         struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
5503         u32 wptr;
5504         u32 rptr;
5505         u32 src_id, src_data, ring_id;
5506         u8 me_id, pipe_id, queue_id;
5507         u32 ring_index;
5508         bool queue_hotplug = false;
5509         bool queue_reset = false;
5510
5511         if (!rdev->ih.enabled || rdev->shutdown)
5512                 return IRQ_NONE;
5513
5514         wptr = cik_get_ih_wptr(rdev);
5515
5516 restart_ih:
5517         /* is somebody else already processing irqs? */
5518         if (atomic_xchg(&rdev->ih.lock, 1))
5519                 return IRQ_NONE;
5520
5521         rptr = rdev->ih.rptr;
5522         DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
5523
5524         /* Order reading of wptr vs. reading of IH ring data */
5525         rmb();
5526
5527         /* display interrupts */
5528         cik_irq_ack(rdev);
5529
5530         while (rptr != wptr) {
5531                 /* wptr/rptr are in bytes! */
5532                 ring_index = rptr / 4;
5533                 src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
5534                 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
5535                 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
5536
5537                 switch (src_id) {
5538                 case 1: /* D1 vblank/vline */
5539                         switch (src_data) {
5540                         case 0: /* D1 vblank */
5541                                 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
5542                                         if (rdev->irq.crtc_vblank_int[0]) {
5543                                                 drm_handle_vblank(rdev->ddev, 0);
5544                                                 rdev->pm.vblank_sync = true;
5545                                                 wake_up(&rdev->irq.vblank_queue);
5546                                         }
5547                                         if (atomic_read(&rdev->irq.pflip[0]))
5548                                                 radeon_crtc_handle_flip(rdev, 0);
5549                                         rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
5550                                         DRM_DEBUG("IH: D1 vblank\n");
5551                                 }
5552                                 break;
5553                         case 1: /* D1 vline */
5554                                 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
5555                                         rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
5556                                         DRM_DEBUG("IH: D1 vline\n");
5557                                 }
5558                                 break;
5559                         default:
5560                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5561                                 break;
5562                         }
5563                         break;
5564                 case 2: /* D2 vblank/vline */
5565                         switch (src_data) {
5566                         case 0: /* D2 vblank */
5567                                 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
5568                                         if (rdev->irq.crtc_vblank_int[1]) {
5569                                                 drm_handle_vblank(rdev->ddev, 1);
5570                                                 rdev->pm.vblank_sync = true;
5571                                                 wake_up(&rdev->irq.vblank_queue);
5572                                         }
5573                                         if (atomic_read(&rdev->irq.pflip[1]))
5574                                                 radeon_crtc_handle_flip(rdev, 1);
5575                                         rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
5576                                         DRM_DEBUG("IH: D2 vblank\n");
5577                                 }
5578                                 break;
5579                         case 1: /* D2 vline */
5580                                 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
5581                                         rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
5582                                         DRM_DEBUG("IH: D2 vline\n");
5583                                 }
5584                                 break;
5585                         default:
5586                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5587                                 break;
5588                         }
5589                         break;
5590                 case 3: /* D3 vblank/vline */
5591                         switch (src_data) {
5592                         case 0: /* D3 vblank */
5593                                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
5594                                         if (rdev->irq.crtc_vblank_int[2]) {
5595                                                 drm_handle_vblank(rdev->ddev, 2);
5596                                                 rdev->pm.vblank_sync = true;
5597                                                 wake_up(&rdev->irq.vblank_queue);
5598                                         }
5599                                         if (atomic_read(&rdev->irq.pflip[2]))
5600                                                 radeon_crtc_handle_flip(rdev, 2);
5601                                         rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
5602                                         DRM_DEBUG("IH: D3 vblank\n");
5603                                 }
5604                                 break;
5605                         case 1: /* D3 vline */
5606                                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
5607                                         rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
5608                                         DRM_DEBUG("IH: D3 vline\n");
5609                                 }
5610                                 break;
5611                         default:
5612                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5613                                 break;
5614                         }
5615                         break;
5616                 case 4: /* D4 vblank/vline */
5617                         switch (src_data) {
5618                         case 0: /* D4 vblank */
5619                                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
5620                                         if (rdev->irq.crtc_vblank_int[3]) {
5621                                                 drm_handle_vblank(rdev->ddev, 3);
5622                                                 rdev->pm.vblank_sync = true;
5623                                                 wake_up(&rdev->irq.vblank_queue);
5624                                         }
5625                                         if (atomic_read(&rdev->irq.pflip[3]))
5626                                                 radeon_crtc_handle_flip(rdev, 3);
5627                                         rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
5628                                         DRM_DEBUG("IH: D4 vblank\n");
5629                                 }
5630                                 break;
5631                         case 1: /* D4 vline */
5632                                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
5633                                         rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
5634                                         DRM_DEBUG("IH: D4 vline\n");
5635                                 }
5636                                 break;
5637                         default:
5638                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5639                                 break;
5640                         }
5641                         break;
5642                 case 5: /* D5 vblank/vline */
5643                         switch (src_data) {
5644                         case 0: /* D5 vblank */
5645                                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
5646                                         if (rdev->irq.crtc_vblank_int[4]) {
5647                                                 drm_handle_vblank(rdev->ddev, 4);
5648                                                 rdev->pm.vblank_sync = true;
5649                                                 wake_up(&rdev->irq.vblank_queue);
5650                                         }
5651                                         if (atomic_read(&rdev->irq.pflip[4]))
5652                                                 radeon_crtc_handle_flip(rdev, 4);
5653                                         rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
5654                                         DRM_DEBUG("IH: D5 vblank\n");
5655                                 }
5656                                 break;
5657                         case 1: /* D5 vline */
5658                                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
5659                                         rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
5660                                         DRM_DEBUG("IH: D5 vline\n");
5661                                 }
5662                                 break;
5663                         default:
5664                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5665                                 break;
5666                         }
5667                         break;
5668                 case 6: /* D6 vblank/vline */
5669                         switch (src_data) {
5670                         case 0: /* D6 vblank */
5671                                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
5672                                         if (rdev->irq.crtc_vblank_int[5]) {
5673                                                 drm_handle_vblank(rdev->ddev, 5);
5674                                                 rdev->pm.vblank_sync = true;
5675                                                 wake_up(&rdev->irq.vblank_queue);
5676                                         }
5677                                         if (atomic_read(&rdev->irq.pflip[5]))
5678                                                 radeon_crtc_handle_flip(rdev, 5);
5679                                         rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
5680                                         DRM_DEBUG("IH: D6 vblank\n");
5681                                 }
5682                                 break;
5683                         case 1: /* D6 vline */
5684                                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
5685                                         rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
5686                                         DRM_DEBUG("IH: D6 vline\n");
5687                                 }
5688                                 break;
5689                         default:
5690                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5691                                 break;
5692                         }
5693                         break;
5694                 case 42: /* HPD hotplug */
5695                         switch (src_data) {
5696                         case 0:
5697                                 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
5698                                         rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
5699                                         queue_hotplug = true;
5700                                         DRM_DEBUG("IH: HPD1\n");
5701                                 }
5702                                 break;
5703                         case 1:
5704                                 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
5705                                         rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
5706                                         queue_hotplug = true;
5707                                         DRM_DEBUG("IH: HPD2\n");
5708                                 }
5709                                 break;
5710                         case 2:
5711                                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
5712                                         rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
5713                                         queue_hotplug = true;
5714                                         DRM_DEBUG("IH: HPD3\n");
5715                                 }
5716                                 break;
5717                         case 3:
5718                                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
5719                                         rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
5720                                         queue_hotplug = true;
5721                                         DRM_DEBUG("IH: HPD4\n");
5722                                 }
5723                                 break;
5724                         case 4:
5725                                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
5726                                         rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
5727                                         queue_hotplug = true;
5728                                         DRM_DEBUG("IH: HPD5\n");
5729                                 }
5730                                 break;
5731                         case 5:
5732                                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
5733                                         rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
5734                                         queue_hotplug = true;
5735                                         DRM_DEBUG("IH: HPD6\n");
5736                                 }
5737                                 break;
5738                         default:
5739                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5740                                 break;
5741                         }
5742                         break;
5743                 case 146:
5744                 case 147:
5745                         dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
5746                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
5747                                 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
5748                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5749                                 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
5750                         /* reset addr and status */
5751                         WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
5752                         break;
5753                 case 176: /* GFX RB CP_INT */
5754                 case 177: /* GFX IB CP_INT */
5755                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5756                         break;
5757                 case 181: /* CP EOP event */
5758                         DRM_DEBUG("IH: CP EOP\n");
5759                         /* XXX check the bitfield order! */
5760                         me_id = (ring_id & 0x60) >> 5;
5761                         pipe_id = (ring_id & 0x18) >> 3;
5762                         queue_id = (ring_id & 0x7) >> 0;
5763                         switch (me_id) {
5764                         case 0:
5765                                 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5766                                 break;
5767                         case 1:
5768                         case 2:
5769                                 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
5770                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5771                                 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
5772                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5773                                 break;
5774                         }
5775                         break;
5776                 case 184: /* CP Privileged reg access */
5777                         DRM_ERROR("Illegal register access in command stream\n");
5778                         /* XXX check the bitfield order! */
5779                         me_id = (ring_id & 0x60) >> 5;
5780                         pipe_id = (ring_id & 0x18) >> 3;
5781                         queue_id = (ring_id & 0x7) >> 0;
5782                         switch (me_id) {
5783                         case 0:
5784                                 /* This results in a full GPU reset, but all we need to do is soft
5785                                  * reset the CP for gfx
5786                                  */
5787                                 queue_reset = true;
5788                                 break;
5789                         case 1:
5790                                 /* XXX compute */
5791                                 queue_reset = true;
5792                                 break;
5793                         case 2:
5794                                 /* XXX compute */
5795                                 queue_reset = true;
5796                                 break;
5797                         }
5798                         break;
5799                 case 185: /* CP Privileged inst */
5800                         DRM_ERROR("Illegal instruction in command stream\n");
5801                         /* XXX check the bitfield order! */
5802                         me_id = (ring_id & 0x60) >> 5;
5803                         pipe_id = (ring_id & 0x18) >> 3;
5804                         queue_id = (ring_id & 0x7) >> 0;
5805                         switch (me_id) {
5806                         case 0:
5807                                 /* This results in a full GPU reset, but all we need to do is soft
5808                                  * reset the CP for gfx
5809                                  */
5810                                 queue_reset = true;
5811                                 break;
5812                         case 1:
5813                                 /* XXX compute */
5814                                 queue_reset = true;
5815                                 break;
5816                         case 2:
5817                                 /* XXX compute */
5818                                 queue_reset = true;
5819                                 break;
5820                         }
5821                         break;
5822                 case 224: /* SDMA trap event */
5823                         /* XXX check the bitfield order! */
5824                         me_id = (ring_id & 0x3) >> 0;
5825                         queue_id = (ring_id & 0xc) >> 2;
5826                         DRM_DEBUG("IH: SDMA trap\n");
5827                         switch (me_id) {
5828                         case 0:
5829                                 switch (queue_id) {
5830                                 case 0:
5831                                         radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
5832                                         break;
5833                                 case 1:
5834                                         /* XXX compute */
5835                                         break;
5836                                 case 2:
5837                                         /* XXX compute */
5838                                         break;
5839                                 }
5840                                 break;
5841                         case 1:
5842                                 switch (queue_id) {
5843                                 case 0:
5844                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5845                                         break;
5846                                 case 1:
5847                                         /* XXX compute */
5848                                         break;
5849                                 case 2:
5850                                         /* XXX compute */
5851                                         break;
5852                                 }
5853                                 break;
5854                         }
5855                         break;
5856                 case 241: /* SDMA Privileged inst */
5857                 case 247: /* SDMA Privileged inst */
5858                         DRM_ERROR("Illegal instruction in SDMA command stream\n");
5859                         /* XXX check the bitfield order! */
5860                         me_id = (ring_id & 0x3) >> 0;
5861                         queue_id = (ring_id & 0xc) >> 2;
5862                         switch (me_id) {
5863                         case 0:
5864                                 switch (queue_id) {
5865                                 case 0:
5866                                         queue_reset = true;
5867                                         break;
5868                                 case 1:
5869                                         /* XXX compute */
5870                                         queue_reset = true;
5871                                         break;
5872                                 case 2:
5873                                         /* XXX compute */
5874                                         queue_reset = true;
5875                                         break;
5876                                 }
5877                                 break;
5878                         case 1:
5879                                 switch (queue_id) {
5880                                 case 0:
5881                                         queue_reset = true;
5882                                         break;
5883                                 case 1:
5884                                         /* XXX compute */
5885                                         queue_reset = true;
5886                                         break;
5887                                 case 2:
5888                                         /* XXX compute */
5889                                         queue_reset = true;
5890                                         break;
5891                                 }
5892                                 break;
5893                         }
5894                         break;
5895                 case 233: /* GUI IDLE */
5896                         DRM_DEBUG("IH: GUI idle\n");
5897                         break;
5898                 default:
5899                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5900                         break;
5901                 }
5902
5903                 /* wptr/rptr are in bytes! */
5904                 rptr += 16;
5905                 rptr &= rdev->ih.ptr_mask;
5906         }
5907         if (queue_hotplug)
5908                 schedule_work(&rdev->hotplug_work);
5909         if (queue_reset)
5910                 schedule_work(&rdev->reset_work);
5911         rdev->ih.rptr = rptr;
5912         WREG32(IH_RB_RPTR, rdev->ih.rptr);
5913         atomic_set(&rdev->ih.lock, 0);
5914
5915         /* make sure wptr hasn't changed while processing */
5916         wptr = cik_get_ih_wptr(rdev);
5917         if (wptr != rptr)
5918                 goto restart_ih;
5919
5920         return IRQ_HANDLED;
5921 }
5922
5923 /*
5924  * startup/shutdown callbacks
5925  */
5926 /**
5927  * cik_startup - program the asic to a functional state
5928  *
5929  * @rdev: radeon_device pointer
5930  *
5931  * Programs the asic to a functional state (CIK).
5932  * Called by cik_init() and cik_resume().
5933  * Returns 0 for success, error for failure.
5934  */
5935 static int cik_startup(struct radeon_device *rdev)
5936 {
5937         struct radeon_ring *ring;
5938         int r;
5939
5940         if (rdev->flags & RADEON_IS_IGP) {
5941                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
5942                     !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
5943                         r = cik_init_microcode(rdev);
5944                         if (r) {
5945                                 DRM_ERROR("Failed to load firmware!\n");
5946                                 return r;
5947                         }
5948                 }
5949         } else {
5950                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
5951                     !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
5952                     !rdev->mc_fw) {
5953                         r = cik_init_microcode(rdev);
5954                         if (r) {
5955                                 DRM_ERROR("Failed to load firmware!\n");
5956                                 return r;
5957                         }
5958                 }
5959
5960                 r = ci_mc_load_microcode(rdev);
5961                 if (r) {
5962                         DRM_ERROR("Failed to load MC firmware!\n");
5963                         return r;
5964                 }
5965         }
5966
5967         r = r600_vram_scratch_init(rdev);
5968         if (r)
5969                 return r;
5970
5971         cik_mc_program(rdev);
5972         r = cik_pcie_gart_enable(rdev);
5973         if (r)
5974                 return r;
5975         cik_gpu_init(rdev);
5976
5977         /* allocate rlc buffers */
5978         r = si_rlc_init(rdev);
5979         if (r) {
5980                 DRM_ERROR("Failed to init rlc BOs!\n");
5981                 return r;
5982         }
5983
5984         /* allocate wb buffer */
5985         r = radeon_wb_init(rdev);
5986         if (r)
5987                 return r;
5988
5989         /* allocate mec buffers */
5990         r = cik_mec_init(rdev);
5991         if (r) {
5992                 DRM_ERROR("Failed to init MEC BOs!\n");
5993                 return r;
5994         }
5995
5996         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5997         if (r) {
5998                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5999                 return r;
6000         }
6001
6002         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6003         if (r) {
6004                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6005                 return r;
6006         }
6007
6008         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6009         if (r) {
6010                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6011                 return r;
6012         }
6013
6014         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
6015         if (r) {
6016                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6017                 return r;
6018         }
6019
6020         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6021         if (r) {
6022                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6023                 return r;
6024         }
6025
6026         r = cik_uvd_resume(rdev);
6027         if (!r) {
6028                 r = radeon_fence_driver_start_ring(rdev,
6029                                                    R600_RING_TYPE_UVD_INDEX);
6030                 if (r)
6031                         dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
6032         }
6033         if (r)
6034                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
6035
6036         /* Enable IRQ */
6037         if (!rdev->irq.installed) {
6038                 r = radeon_irq_kms_init(rdev);
6039                 if (r)
6040                         return r;
6041         }
6042
6043         r = cik_irq_init(rdev);
6044         if (r) {
6045                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
6046                 radeon_irq_kms_fini(rdev);
6047                 return r;
6048         }
6049         cik_irq_set(rdev);
6050
6051         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6052         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
6053                              CP_RB0_RPTR, CP_RB0_WPTR,
6054                              0, 0xfffff, RADEON_CP_PACKET2);
6055         if (r)
6056                 return r;
6057
6058         /* set up the compute queues */
6059         /* type-2 packets are deprecated on MEC, use type-3 instead */
6060         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6061         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
6062                              CP_HQD_PQ_RPTR, CP_HQD_PQ_WPTR,
6063                              0, 0xfffff, PACKET3(PACKET3_NOP, 0x3FFF));
6064         if (r)
6065                 return r;
6066         ring->me = 1; /* first MEC */
6067         ring->pipe = 0; /* first pipe */
6068         ring->queue = 0; /* first queue */
6069         ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
6070
6071         /* type-2 packets are deprecated on MEC, use type-3 instead */
6072         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6073         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
6074                              CP_HQD_PQ_RPTR, CP_HQD_PQ_WPTR,
6075                              0, 0xffffffff, PACKET3(PACKET3_NOP, 0x3FFF));
6076         if (r)
6077                 return r;
6078         /* dGPU only have 1 MEC */
6079         ring->me = 1; /* first MEC */
6080         ring->pipe = 0; /* first pipe */
6081         ring->queue = 1; /* second queue */
6082         ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
6083
6084         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6085         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
6086                              SDMA0_GFX_RB_RPTR + SDMA0_REGISTER_OFFSET,
6087                              SDMA0_GFX_RB_WPTR + SDMA0_REGISTER_OFFSET,
6088                              2, 0xfffffffc, SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
6089         if (r)
6090                 return r;
6091
6092         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6093         r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
6094                              SDMA0_GFX_RB_RPTR + SDMA1_REGISTER_OFFSET,
6095                              SDMA0_GFX_RB_WPTR + SDMA1_REGISTER_OFFSET,
6096                              2, 0xfffffffc, SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
6097         if (r)
6098                 return r;
6099
6100         r = cik_cp_resume(rdev);
6101         if (r)
6102                 return r;
6103
6104         r = cik_sdma_resume(rdev);
6105         if (r)
6106                 return r;
6107
6108         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6109         if (ring->ring_size) {
6110                 r = radeon_ring_init(rdev, ring, ring->ring_size,
6111                                      R600_WB_UVD_RPTR_OFFSET,
6112                                      UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
6113                                      0, 0xfffff, RADEON_CP_PACKET2);
6114                 if (!r)
6115                         r = r600_uvd_init(rdev);
6116                 if (r)
6117                         DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
6118         }
6119
6120         r = radeon_ib_pool_init(rdev);
6121         if (r) {
6122                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
6123                 return r;
6124         }
6125
6126         r = radeon_vm_manager_init(rdev);
6127         if (r) {
6128                 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
6129                 return r;
6130         }
6131
6132         return 0;
6133 }
6134
6135 /**
6136  * cik_resume - resume the asic to a functional state
6137  *
6138  * @rdev: radeon_device pointer
6139  *
6140  * Programs the asic to a functional state (CIK).
6141  * Called at resume.
6142  * Returns 0 for success, error for failure.
6143  */
6144 int cik_resume(struct radeon_device *rdev)
6145 {
6146         int r;
6147
6148         /* post card */
6149         atom_asic_init(rdev->mode_info.atom_context);
6150
6151         /* init golden registers */
6152         cik_init_golden_registers(rdev);
6153
6154         rdev->accel_working = true;
6155         r = cik_startup(rdev);
6156         if (r) {
6157                 DRM_ERROR("cik startup failed on resume\n");
6158                 rdev->accel_working = false;
6159                 return r;
6160         }
6161
6162         return r;
6163
6164 }
6165
6166 /**
6167  * cik_suspend - suspend the asic
6168  *
6169  * @rdev: radeon_device pointer
6170  *
6171  * Bring the chip into a state suitable for suspend (CIK).
6172  * Called at suspend.
6173  * Returns 0 for success.
6174  */
6175 int cik_suspend(struct radeon_device *rdev)
6176 {
6177         radeon_vm_manager_fini(rdev);
6178         cik_cp_enable(rdev, false);
6179         cik_sdma_enable(rdev, false);
6180         r600_uvd_rbc_stop(rdev);
6181         radeon_uvd_suspend(rdev);
6182         cik_irq_suspend(rdev);
6183         radeon_wb_disable(rdev);
6184         cik_pcie_gart_disable(rdev);
6185         return 0;
6186 }
6187
6188 /* Plan is to move initialization in that function and use
6189  * helper function so that radeon_device_init pretty much
6190  * do nothing more than calling asic specific function. This
6191  * should also allow to remove a bunch of callback function
6192  * like vram_info.
6193  */
6194 /**
6195  * cik_init - asic specific driver and hw init
6196  *
6197  * @rdev: radeon_device pointer
6198  *
6199  * Setup asic specific driver variables and program the hw
6200  * to a functional state (CIK).
6201  * Called at driver startup.
6202  * Returns 0 for success, errors for failure.
6203  */
6204 int cik_init(struct radeon_device *rdev)
6205 {
6206         struct radeon_ring *ring;
6207         int r;
6208
6209         /* Read BIOS */
6210         if (!radeon_get_bios(rdev)) {
6211                 if (ASIC_IS_AVIVO(rdev))
6212                         return -EINVAL;
6213         }
6214         /* Must be an ATOMBIOS */
6215         if (!rdev->is_atom_bios) {
6216                 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
6217                 return -EINVAL;
6218         }
6219         r = radeon_atombios_init(rdev);
6220         if (r)
6221                 return r;
6222
6223         /* Post card if necessary */
6224         if (!radeon_card_posted(rdev)) {
6225                 if (!rdev->bios) {
6226                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
6227                         return -EINVAL;
6228                 }
6229                 DRM_INFO("GPU not posted. posting now...\n");
6230                 atom_asic_init(rdev->mode_info.atom_context);
6231         }
6232         /* init golden registers */
6233         cik_init_golden_registers(rdev);
6234         /* Initialize scratch registers */
6235         cik_scratch_init(rdev);
6236         /* Initialize surface registers */
6237         radeon_surface_init(rdev);
6238         /* Initialize clocks */
6239         radeon_get_clock_info(rdev->ddev);
6240
6241         /* Fence driver */
6242         r = radeon_fence_driver_init(rdev);
6243         if (r)
6244                 return r;
6245
6246         /* initialize memory controller */
6247         r = cik_mc_init(rdev);
6248         if (r)
6249                 return r;
6250         /* Memory manager */
6251         r = radeon_bo_init(rdev);
6252         if (r)
6253                 return r;
6254
6255         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6256         ring->ring_obj = NULL;
6257         r600_ring_init(rdev, ring, 1024 * 1024);
6258
6259         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6260         ring->ring_obj = NULL;
6261         r600_ring_init(rdev, ring, 1024 * 1024);
6262         r = radeon_doorbell_get(rdev, &ring->doorbell_page_num);
6263         if (r)
6264                 return r;
6265
6266         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6267         ring->ring_obj = NULL;
6268         r600_ring_init(rdev, ring, 1024 * 1024);
6269         r = radeon_doorbell_get(rdev, &ring->doorbell_page_num);
6270         if (r)
6271                 return r;
6272
6273         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6274         ring->ring_obj = NULL;
6275         r600_ring_init(rdev, ring, 256 * 1024);
6276
6277         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6278         ring->ring_obj = NULL;
6279         r600_ring_init(rdev, ring, 256 * 1024);
6280
6281         r = radeon_uvd_init(rdev);
6282         if (!r) {
6283                 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6284                 ring->ring_obj = NULL;
6285                 r600_ring_init(rdev, ring, 4096);
6286         }
6287
6288         rdev->ih.ring_obj = NULL;
6289         r600_ih_ring_init(rdev, 64 * 1024);
6290
6291         r = r600_pcie_gart_init(rdev);
6292         if (r)
6293                 return r;
6294
6295         rdev->accel_working = true;
6296         r = cik_startup(rdev);
6297         if (r) {
6298                 dev_err(rdev->dev, "disabling GPU acceleration\n");
6299                 cik_cp_fini(rdev);
6300                 cik_sdma_fini(rdev);
6301                 cik_irq_fini(rdev);
6302                 si_rlc_fini(rdev);
6303                 cik_mec_fini(rdev);
6304                 radeon_wb_fini(rdev);
6305                 radeon_ib_pool_fini(rdev);
6306                 radeon_vm_manager_fini(rdev);
6307                 radeon_irq_kms_fini(rdev);
6308                 cik_pcie_gart_fini(rdev);
6309                 rdev->accel_working = false;
6310         }
6311
6312         /* Don't start up if the MC ucode is missing.
6313          * The default clocks and voltages before the MC ucode
6314          * is loaded are not suffient for advanced operations.
6315          */
6316         if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
6317                 DRM_ERROR("radeon: MC ucode required for NI+.\n");
6318                 return -EINVAL;
6319         }
6320
6321         return 0;
6322 }
6323
6324 /**
6325  * cik_fini - asic specific driver and hw fini
6326  *
6327  * @rdev: radeon_device pointer
6328  *
6329  * Tear down the asic specific driver variables and program the hw
6330  * to an idle state (CIK).
6331  * Called at driver unload.
6332  */
6333 void cik_fini(struct radeon_device *rdev)
6334 {
6335         cik_cp_fini(rdev);
6336         cik_sdma_fini(rdev);
6337         cik_irq_fini(rdev);
6338         si_rlc_fini(rdev);
6339         cik_mec_fini(rdev);
6340         radeon_wb_fini(rdev);
6341         radeon_vm_manager_fini(rdev);
6342         radeon_ib_pool_fini(rdev);
6343         radeon_irq_kms_fini(rdev);
6344         radeon_uvd_fini(rdev);
6345         cik_pcie_gart_fini(rdev);
6346         r600_vram_scratch_fini(rdev);
6347         radeon_gem_fini(rdev);
6348         radeon_fence_driver_fini(rdev);
6349         radeon_bo_fini(rdev);
6350         radeon_atombios_fini(rdev);
6351         kfree(rdev->bios);
6352         rdev->bios = NULL;
6353 }
6354
6355 /* display watermark setup */
6356 /**
6357  * dce8_line_buffer_adjust - Set up the line buffer
6358  *
6359  * @rdev: radeon_device pointer
6360  * @radeon_crtc: the selected display controller
6361  * @mode: the current display mode on the selected display
6362  * controller
6363  *
6364  * Setup up the line buffer allocation for
6365  * the selected display controller (CIK).
6366  * Returns the line buffer size in pixels.
6367  */
6368 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
6369                                    struct radeon_crtc *radeon_crtc,
6370                                    struct drm_display_mode *mode)
6371 {
6372         u32 tmp;
6373
6374         /*
6375          * Line Buffer Setup
6376          * There are 6 line buffers, one for each display controllers.
6377          * There are 3 partitions per LB. Select the number of partitions
6378          * to enable based on the display width.  For display widths larger
6379          * than 4096, you need use to use 2 display controllers and combine
6380          * them using the stereo blender.
6381          */
6382         if (radeon_crtc->base.enabled && mode) {
6383                 if (mode->crtc_hdisplay < 1920)
6384                         tmp = 1;
6385                 else if (mode->crtc_hdisplay < 2560)
6386                         tmp = 2;
6387                 else if (mode->crtc_hdisplay < 4096)
6388                         tmp = 0;
6389                 else {
6390                         DRM_DEBUG_KMS("Mode too big for LB!\n");
6391                         tmp = 0;
6392                 }
6393         } else
6394                 tmp = 1;
6395
6396         WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
6397                LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
6398
6399         if (radeon_crtc->base.enabled && mode) {
6400                 switch (tmp) {
6401                 case 0:
6402                 default:
6403                         return 4096 * 2;
6404                 case 1:
6405                         return 1920 * 2;
6406                 case 2:
6407                         return 2560 * 2;
6408                 }
6409         }
6410
6411         /* controller not enabled, so no lb used */
6412         return 0;
6413 }
6414
6415 /**
6416  * cik_get_number_of_dram_channels - get the number of dram channels
6417  *
6418  * @rdev: radeon_device pointer
6419  *
6420  * Look up the number of video ram channels (CIK).
6421  * Used for display watermark bandwidth calculations
6422  * Returns the number of dram channels
6423  */
6424 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
6425 {
6426         u32 tmp = RREG32(MC_SHARED_CHMAP);
6427
6428         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
6429         case 0:
6430         default:
6431                 return 1;
6432         case 1:
6433                 return 2;
6434         case 2:
6435                 return 4;
6436         case 3:
6437                 return 8;
6438         case 4:
6439                 return 3;
6440         case 5:
6441                 return 6;
6442         case 6:
6443                 return 10;
6444         case 7:
6445                 return 12;
6446         case 8:
6447                 return 16;
6448         }
6449 }
6450
6451 struct dce8_wm_params {
6452         u32 dram_channels; /* number of dram channels */
6453         u32 yclk;          /* bandwidth per dram data pin in kHz */
6454         u32 sclk;          /* engine clock in kHz */
6455         u32 disp_clk;      /* display clock in kHz */
6456         u32 src_width;     /* viewport width */
6457         u32 active_time;   /* active display time in ns */
6458         u32 blank_time;    /* blank time in ns */
6459         bool interlaced;    /* mode is interlaced */
6460         fixed20_12 vsc;    /* vertical scale ratio */
6461         u32 num_heads;     /* number of active crtcs */
6462         u32 bytes_per_pixel; /* bytes per pixel display + overlay */
6463         u32 lb_size;       /* line buffer allocated to pipe */
6464         u32 vtaps;         /* vertical scaler taps */
6465 };
6466
6467 /**
6468  * dce8_dram_bandwidth - get the dram bandwidth
6469  *
6470  * @wm: watermark calculation data
6471  *
6472  * Calculate the raw dram bandwidth (CIK).
6473  * Used for display watermark bandwidth calculations
6474  * Returns the dram bandwidth in MBytes/s
6475  */
6476 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
6477 {
6478         /* Calculate raw DRAM Bandwidth */
6479         fixed20_12 dram_efficiency; /* 0.7 */
6480         fixed20_12 yclk, dram_channels, bandwidth;
6481         fixed20_12 a;
6482
6483         a.full = dfixed_const(1000);
6484         yclk.full = dfixed_const(wm->yclk);
6485         yclk.full = dfixed_div(yclk, a);
6486         dram_channels.full = dfixed_const(wm->dram_channels * 4);
6487         a.full = dfixed_const(10);
6488         dram_efficiency.full = dfixed_const(7);
6489         dram_efficiency.full = dfixed_div(dram_efficiency, a);
6490         bandwidth.full = dfixed_mul(dram_channels, yclk);
6491         bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
6492
6493         return dfixed_trunc(bandwidth);
6494 }
6495
6496 /**
6497  * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
6498  *
6499  * @wm: watermark calculation data
6500  *
6501  * Calculate the dram bandwidth used for display (CIK).
6502  * Used for display watermark bandwidth calculations
6503  * Returns the dram bandwidth for display in MBytes/s
6504  */
6505 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
6506 {
6507         /* Calculate DRAM Bandwidth and the part allocated to display. */
6508         fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
6509         fixed20_12 yclk, dram_channels, bandwidth;
6510         fixed20_12 a;
6511
6512         a.full = dfixed_const(1000);
6513         yclk.full = dfixed_const(wm->yclk);
6514         yclk.full = dfixed_div(yclk, a);
6515         dram_channels.full = dfixed_const(wm->dram_channels * 4);
6516         a.full = dfixed_const(10);
6517         disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
6518         disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
6519         bandwidth.full = dfixed_mul(dram_channels, yclk);
6520         bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
6521
6522         return dfixed_trunc(bandwidth);
6523 }
6524
6525 /**
6526  * dce8_data_return_bandwidth - get the data return bandwidth
6527  *
6528  * @wm: watermark calculation data
6529  *
6530  * Calculate the data return bandwidth used for display (CIK).
6531  * Used for display watermark bandwidth calculations
6532  * Returns the data return bandwidth in MBytes/s
6533  */
6534 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
6535 {
6536         /* Calculate the display Data return Bandwidth */
6537         fixed20_12 return_efficiency; /* 0.8 */
6538         fixed20_12 sclk, bandwidth;
6539         fixed20_12 a;
6540
6541         a.full = dfixed_const(1000);
6542         sclk.full = dfixed_const(wm->sclk);
6543         sclk.full = dfixed_div(sclk, a);
6544         a.full = dfixed_const(10);
6545         return_efficiency.full = dfixed_const(8);
6546         return_efficiency.full = dfixed_div(return_efficiency, a);
6547         a.full = dfixed_const(32);
6548         bandwidth.full = dfixed_mul(a, sclk);
6549         bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
6550
6551         return dfixed_trunc(bandwidth);
6552 }
6553
6554 /**
6555  * dce8_dmif_request_bandwidth - get the dmif bandwidth
6556  *
6557  * @wm: watermark calculation data
6558  *
6559  * Calculate the dmif bandwidth used for display (CIK).
6560  * Used for display watermark bandwidth calculations
6561  * Returns the dmif bandwidth in MBytes/s
6562  */
6563 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
6564 {
6565         /* Calculate the DMIF Request Bandwidth */
6566         fixed20_12 disp_clk_request_efficiency; /* 0.8 */
6567         fixed20_12 disp_clk, bandwidth;
6568         fixed20_12 a, b;
6569
6570         a.full = dfixed_const(1000);
6571         disp_clk.full = dfixed_const(wm->disp_clk);
6572         disp_clk.full = dfixed_div(disp_clk, a);
6573         a.full = dfixed_const(32);
6574         b.full = dfixed_mul(a, disp_clk);
6575
6576         a.full = dfixed_const(10);
6577         disp_clk_request_efficiency.full = dfixed_const(8);
6578         disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
6579
6580         bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
6581
6582         return dfixed_trunc(bandwidth);
6583 }
6584
6585 /**
6586  * dce8_available_bandwidth - get the min available bandwidth
6587  *
6588  * @wm: watermark calculation data
6589  *
6590  * Calculate the min available bandwidth used for display (CIK).
6591  * Used for display watermark bandwidth calculations
6592  * Returns the min available bandwidth in MBytes/s
6593  */
6594 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
6595 {
6596         /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
6597         u32 dram_bandwidth = dce8_dram_bandwidth(wm);
6598         u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
6599         u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
6600
6601         return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
6602 }
6603
6604 /**
6605  * dce8_average_bandwidth - get the average available bandwidth
6606  *
6607  * @wm: watermark calculation data
6608  *
6609  * Calculate the average available bandwidth used for display (CIK).
6610  * Used for display watermark bandwidth calculations
6611  * Returns the average available bandwidth in MBytes/s
6612  */
6613 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
6614 {
6615         /* Calculate the display mode Average Bandwidth
6616          * DisplayMode should contain the source and destination dimensions,
6617          * timing, etc.
6618          */
6619         fixed20_12 bpp;
6620         fixed20_12 line_time;
6621         fixed20_12 src_width;
6622         fixed20_12 bandwidth;
6623         fixed20_12 a;
6624
6625         a.full = dfixed_const(1000);
6626         line_time.full = dfixed_const(wm->active_time + wm->blank_time);
6627         line_time.full = dfixed_div(line_time, a);
6628         bpp.full = dfixed_const(wm->bytes_per_pixel);
6629         src_width.full = dfixed_const(wm->src_width);
6630         bandwidth.full = dfixed_mul(src_width, bpp);
6631         bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
6632         bandwidth.full = dfixed_div(bandwidth, line_time);
6633
6634         return dfixed_trunc(bandwidth);
6635 }
6636
6637 /**
6638  * dce8_latency_watermark - get the latency watermark
6639  *
6640  * @wm: watermark calculation data
6641  *
6642  * Calculate the latency watermark (CIK).
6643  * Used for display watermark bandwidth calculations
6644  * Returns the latency watermark in ns
6645  */
6646 static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
6647 {
6648         /* First calculate the latency in ns */
6649         u32 mc_latency = 2000; /* 2000 ns. */
6650         u32 available_bandwidth = dce8_available_bandwidth(wm);
6651         u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
6652         u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
6653         u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
6654         u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
6655                 (wm->num_heads * cursor_line_pair_return_time);
6656         u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
6657         u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
6658         u32 tmp, dmif_size = 12288;
6659         fixed20_12 a, b, c;
6660
6661         if (wm->num_heads == 0)
6662                 return 0;
6663
6664         a.full = dfixed_const(2);
6665         b.full = dfixed_const(1);
6666         if ((wm->vsc.full > a.full) ||
6667             ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
6668             (wm->vtaps >= 5) ||
6669             ((wm->vsc.full >= a.full) && wm->interlaced))
6670                 max_src_lines_per_dst_line = 4;
6671         else
6672                 max_src_lines_per_dst_line = 2;
6673
6674         a.full = dfixed_const(available_bandwidth);
6675         b.full = dfixed_const(wm->num_heads);
6676         a.full = dfixed_div(a, b);
6677
6678         b.full = dfixed_const(mc_latency + 512);
6679         c.full = dfixed_const(wm->disp_clk);
6680         b.full = dfixed_div(b, c);
6681
6682         c.full = dfixed_const(dmif_size);
6683         b.full = dfixed_div(c, b);
6684
6685         tmp = min(dfixed_trunc(a), dfixed_trunc(b));
6686
6687         b.full = dfixed_const(1000);
6688         c.full = dfixed_const(wm->disp_clk);
6689         b.full = dfixed_div(c, b);
6690         c.full = dfixed_const(wm->bytes_per_pixel);
6691         b.full = dfixed_mul(b, c);
6692
6693         lb_fill_bw = min(tmp, dfixed_trunc(b));
6694
6695         a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
6696         b.full = dfixed_const(1000);
6697         c.full = dfixed_const(lb_fill_bw);
6698         b.full = dfixed_div(c, b);
6699         a.full = dfixed_div(a, b);
6700         line_fill_time = dfixed_trunc(a);
6701
6702         if (line_fill_time < wm->active_time)
6703                 return latency;
6704         else
6705                 return latency + (line_fill_time - wm->active_time);
6706
6707 }
6708
6709 /**
6710  * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
6711  * average and available dram bandwidth
6712  *
6713  * @wm: watermark calculation data
6714  *
6715  * Check if the display average bandwidth fits in the display
6716  * dram bandwidth (CIK).
6717  * Used for display watermark bandwidth calculations
6718  * Returns true if the display fits, false if not.
6719  */
6720 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
6721 {
6722         if (dce8_average_bandwidth(wm) <=
6723             (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
6724                 return true;
6725         else
6726                 return false;
6727 }
6728
6729 /**
6730  * dce8_average_bandwidth_vs_available_bandwidth - check
6731  * average and available bandwidth
6732  *
6733  * @wm: watermark calculation data
6734  *
6735  * Check if the display average bandwidth fits in the display
6736  * available bandwidth (CIK).
6737  * Used for display watermark bandwidth calculations
6738  * Returns true if the display fits, false if not.
6739  */
6740 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
6741 {
6742         if (dce8_average_bandwidth(wm) <=
6743             (dce8_available_bandwidth(wm) / wm->num_heads))
6744                 return true;
6745         else
6746                 return false;
6747 }
6748
6749 /**
6750  * dce8_check_latency_hiding - check latency hiding
6751  *
6752  * @wm: watermark calculation data
6753  *
6754  * Check latency hiding (CIK).
6755  * Used for display watermark bandwidth calculations
6756  * Returns true if the display fits, false if not.
6757  */
6758 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
6759 {
6760         u32 lb_partitions = wm->lb_size / wm->src_width;
6761         u32 line_time = wm->active_time + wm->blank_time;
6762         u32 latency_tolerant_lines;
6763         u32 latency_hiding;
6764         fixed20_12 a;
6765
6766         a.full = dfixed_const(1);
6767         if (wm->vsc.full > a.full)
6768                 latency_tolerant_lines = 1;
6769         else {
6770                 if (lb_partitions <= (wm->vtaps + 1))
6771                         latency_tolerant_lines = 1;
6772                 else
6773                         latency_tolerant_lines = 2;
6774         }
6775
6776         latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
6777
6778         if (dce8_latency_watermark(wm) <= latency_hiding)
6779                 return true;
6780         else
6781                 return false;
6782 }
6783
6784 /**
6785  * dce8_program_watermarks - program display watermarks
6786  *
6787  * @rdev: radeon_device pointer
6788  * @radeon_crtc: the selected display controller
6789  * @lb_size: line buffer size
6790  * @num_heads: number of display controllers in use
6791  *
6792  * Calculate and program the display watermarks for the
6793  * selected display controller (CIK).
6794  */
6795 static void dce8_program_watermarks(struct radeon_device *rdev,
6796                                     struct radeon_crtc *radeon_crtc,
6797                                     u32 lb_size, u32 num_heads)
6798 {
6799         struct drm_display_mode *mode = &radeon_crtc->base.mode;
6800         struct dce8_wm_params wm;
6801         u32 pixel_period;
6802         u32 line_time = 0;
6803         u32 latency_watermark_a = 0, latency_watermark_b = 0;
6804         u32 tmp, wm_mask;
6805
6806         if (radeon_crtc->base.enabled && num_heads && mode) {
6807                 pixel_period = 1000000 / (u32)mode->clock;
6808                 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
6809
6810                 wm.yclk = rdev->pm.current_mclk * 10;
6811                 wm.sclk = rdev->pm.current_sclk * 10;
6812                 wm.disp_clk = mode->clock;
6813                 wm.src_width = mode->crtc_hdisplay;
6814                 wm.active_time = mode->crtc_hdisplay * pixel_period;
6815                 wm.blank_time = line_time - wm.active_time;
6816                 wm.interlaced = false;
6817                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
6818                         wm.interlaced = true;
6819                 wm.vsc = radeon_crtc->vsc;
6820                 wm.vtaps = 1;
6821                 if (radeon_crtc->rmx_type != RMX_OFF)
6822                         wm.vtaps = 2;
6823                 wm.bytes_per_pixel = 4; /* XXX: get this from fb config */
6824                 wm.lb_size = lb_size;
6825                 wm.dram_channels = cik_get_number_of_dram_channels(rdev);
6826                 wm.num_heads = num_heads;
6827
6828                 /* set for high clocks */
6829                 latency_watermark_a = min(dce8_latency_watermark(&wm), (u32)65535);
6830                 /* set for low clocks */
6831                 /* wm.yclk = low clk; wm.sclk = low clk */
6832                 latency_watermark_b = min(dce8_latency_watermark(&wm), (u32)65535);
6833
6834                 /* possibly force display priority to high */
6835                 /* should really do this at mode validation time... */
6836                 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm) ||
6837                     !dce8_average_bandwidth_vs_available_bandwidth(&wm) ||
6838                     !dce8_check_latency_hiding(&wm) ||
6839                     (rdev->disp_priority == 2)) {
6840                         DRM_DEBUG_KMS("force priority to high\n");
6841                 }
6842         }
6843
6844         /* select wm A */
6845         wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
6846         tmp = wm_mask;
6847         tmp &= ~LATENCY_WATERMARK_MASK(3);
6848         tmp |= LATENCY_WATERMARK_MASK(1);
6849         WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
6850         WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
6851                (LATENCY_LOW_WATERMARK(latency_watermark_a) |
6852                 LATENCY_HIGH_WATERMARK(line_time)));
6853         /* select wm B */
6854         tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
6855         tmp &= ~LATENCY_WATERMARK_MASK(3);
6856         tmp |= LATENCY_WATERMARK_MASK(2);
6857         WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
6858         WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
6859                (LATENCY_LOW_WATERMARK(latency_watermark_b) |
6860                 LATENCY_HIGH_WATERMARK(line_time)));
6861         /* restore original selection */
6862         WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
6863 }
6864
6865 /**
6866  * dce8_bandwidth_update - program display watermarks
6867  *
6868  * @rdev: radeon_device pointer
6869  *
6870  * Calculate and program the display watermarks and line
6871  * buffer allocation (CIK).
6872  */
6873 void dce8_bandwidth_update(struct radeon_device *rdev)
6874 {
6875         struct drm_display_mode *mode = NULL;
6876         u32 num_heads = 0, lb_size;
6877         int i;
6878
6879         radeon_update_display_priority(rdev);
6880
6881         for (i = 0; i < rdev->num_crtc; i++) {
6882                 if (rdev->mode_info.crtcs[i]->base.enabled)
6883                         num_heads++;
6884         }
6885         for (i = 0; i < rdev->num_crtc; i++) {
6886                 mode = &rdev->mode_info.crtcs[i]->base.mode;
6887                 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
6888                 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
6889         }
6890 }
6891
6892 /**
6893  * cik_get_gpu_clock_counter - return GPU clock counter snapshot
6894  *
6895  * @rdev: radeon_device pointer
6896  *
6897  * Fetches a GPU clock counter snapshot (SI).
6898  * Returns the 64 bit clock counter snapshot.
6899  */
6900 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
6901 {
6902         uint64_t clock;
6903
6904         mutex_lock(&rdev->gpu_clock_mutex);
6905         WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
6906         clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
6907                 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
6908         mutex_unlock(&rdev->gpu_clock_mutex);
6909         return clock;
6910 }
6911
6912 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
6913                               u32 cntl_reg, u32 status_reg)
6914 {
6915         int r, i;
6916         struct atom_clock_dividers dividers;
6917         uint32_t tmp;
6918
6919         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
6920                                            clock, false, &dividers);
6921         if (r)
6922                 return r;
6923
6924         tmp = RREG32_SMC(cntl_reg);
6925         tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
6926         tmp |= dividers.post_divider;
6927         WREG32_SMC(cntl_reg, tmp);
6928
6929         for (i = 0; i < 100; i++) {
6930                 if (RREG32_SMC(status_reg) & DCLK_STATUS)
6931                         break;
6932                 mdelay(10);
6933         }
6934         if (i == 100)
6935                 return -ETIMEDOUT;
6936
6937         return 0;
6938 }
6939
6940 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
6941 {
6942         int r = 0;
6943
6944         r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
6945         if (r)
6946                 return r;
6947
6948         r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
6949         return r;
6950 }
6951
6952 int cik_uvd_resume(struct radeon_device *rdev)
6953 {
6954         uint64_t addr;
6955         uint32_t size;
6956         int r;
6957
6958         r = radeon_uvd_resume(rdev);
6959         if (r)
6960                 return r;
6961
6962         /* programm the VCPU memory controller bits 0-27 */
6963         addr = rdev->uvd.gpu_addr >> 3;
6964         size = RADEON_GPU_PAGE_ALIGN(rdev->uvd_fw->size + 4) >> 3;
6965         WREG32(UVD_VCPU_CACHE_OFFSET0, addr);
6966         WREG32(UVD_VCPU_CACHE_SIZE0, size);
6967
6968         addr += size;
6969         size = RADEON_UVD_STACK_SIZE >> 3;
6970         WREG32(UVD_VCPU_CACHE_OFFSET1, addr);
6971         WREG32(UVD_VCPU_CACHE_SIZE1, size);
6972
6973         addr += size;
6974         size = RADEON_UVD_HEAP_SIZE >> 3;
6975         WREG32(UVD_VCPU_CACHE_OFFSET2, addr);
6976         WREG32(UVD_VCPU_CACHE_SIZE2, size);
6977
6978         /* bits 28-31 */
6979         addr = (rdev->uvd.gpu_addr >> 28) & 0xF;
6980         WREG32(UVD_LMI_ADDR_EXT, (addr << 12) | (addr << 0));
6981
6982         /* bits 32-39 */
6983         addr = (rdev->uvd.gpu_addr >> 32) & 0xFF;
6984         WREG32(UVD_LMI_EXT40_ADDR, addr | (0x9 << 16) | (0x1 << 31));
6985
6986         return 0;
6987 }