drivers/gpu/drm/radeon/ni.c

   1 /*
   2  * Copyright 2010 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/platform_device.h>
  26 #include <linux/slab.h>
  27 #include <linux/module.h>
  28 #include <drm/drmP.h>
  29 #include "radeon.h"
  30 #include "radeon_asic.h"
  31 #include <drm/radeon_drm.h>
  32 #include "nid.h"
  33 #include "atom.h"
  34 #include "ni_reg.h"
  35 #include "cayman_blit_shaders.h"
  36 #include "radeon_ucode.h"
  37 #include "clearstate_cayman.h"
  38
  39 static u32 tn_rlc_save_restore_register_list[] =
  40 {
  41         0x98fc,
  42         0x98f0,
  43         0x9834,
  44         0x9838,
  45         0x9870,
  46         0x9874,
  47         0x8a14,
  48         0x8b24,
  49         0x8bcc,
  50         0x8b10,
  51         0x8c30,
  52         0x8d00,
  53         0x8d04,
  54         0x8c00,
  55         0x8c04,
  56         0x8c10,
  57         0x8c14,
  58         0x8d8c,
  59         0x8cf0,
  60         0x8e38,
  61         0x9508,
  62         0x9688,
  63         0x9608,
  64         0x960c,
  65         0x9610,
  66         0x9614,
  67         0x88c4,
  68         0x8978,
  69         0x88d4,
  70         0x900c,
  71         0x9100,
  72         0x913c,
  73         0x90e8,
  74         0x9354,
  75         0xa008,
  76         0x98f8,
  77         0x9148,
  78         0x914c,
  79         0x3f94,
  80         0x98f4,
  81         0x9b7c,
  82         0x3f8c,
  83         0x8950,
  84         0x8954,
  85         0x8a18,
  86         0x8b28,
  87         0x9144,
  88         0x3f90,
  89         0x915c,
  90         0x9160,
  91         0x9178,
  92         0x917c,
  93         0x9180,
  94         0x918c,
  95         0x9190,
  96         0x9194,
  97         0x9198,
  98         0x919c,
  99         0x91a8,
 100         0x91ac,
 101         0x91b0,
 102         0x91b4,
 103         0x91b8,
 104         0x91c4,
 105         0x91c8,
 106         0x91cc,
 107         0x91d0,
 108         0x91d4,
 109         0x91e0,
 110         0x91e4,
 111         0x91ec,
 112         0x91f0,
 113         0x91f4,
 114         0x9200,
 115         0x9204,
 116         0x929c,
 117         0x8030,
 118         0x9150,
 119         0x9a60,
 120         0x920c,
 121         0x9210,
 122         0x9228,
 123         0x922c,
 124         0x9244,
 125         0x9248,
 126         0x91e8,
 127         0x9294,
 128         0x9208,
 129         0x9224,
 130         0x9240,
 131         0x9220,
 132         0x923c,
 133         0x9258,
 134         0x9744,
 135         0xa200,
 136         0xa204,
 137         0xa208,
 138         0xa20c,
 139         0x8d58,
 140         0x9030,
 141         0x9034,
 142         0x9038,
 143         0x903c,
 144         0x9040,
 145         0x9654,
 146         0x897c,
 147         0xa210,
 148         0xa214,
 149         0x9868,
 150         0xa02c,
 151         0x9664,
 152         0x9698,
 153         0x949c,
 154         0x8e10,
 155         0x8e18,
 156         0x8c50,
 157         0x8c58,
 158         0x8c60,
 159         0x8c68,
 160         0x89b4,
 161         0x9830,
 162         0x802c,
 163 };
 164 static u32 tn_rlc_save_restore_register_list_size = ARRAY_SIZE(tn_rlc_save_restore_register_list);
 165
 166 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
 167 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
 168 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
 169 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
 170 extern int evergreen_mc_wait_for_idle(struct radeon_device *rdev);
 171 extern void evergreen_mc_program(struct radeon_device *rdev);
 172 extern void evergreen_irq_suspend(struct radeon_device *rdev);
 173 extern int evergreen_mc_init(struct radeon_device *rdev);
 174 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
 175 extern void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
 176 extern void evergreen_program_aspm(struct radeon_device *rdev);
 177 extern void sumo_rlc_fini(struct radeon_device *rdev);
 178 extern int sumo_rlc_init(struct radeon_device *rdev);
 179
 180 /* Firmware Names */
 181 MODULE_FIRMWARE("radeon/BARTS_pfp.bin");
 182 MODULE_FIRMWARE("radeon/BARTS_me.bin");
 183 MODULE_FIRMWARE("radeon/BARTS_mc.bin");
 184 MODULE_FIRMWARE("radeon/BARTS_smc.bin");
 185 MODULE_FIRMWARE("radeon/BTC_rlc.bin");
 186 MODULE_FIRMWARE("radeon/TURKS_pfp.bin");
 187 MODULE_FIRMWARE("radeon/TURKS_me.bin");
 188 MODULE_FIRMWARE("radeon/TURKS_mc.bin");
 189 MODULE_FIRMWARE("radeon/TURKS_smc.bin");
 190 MODULE_FIRMWARE("radeon/CAICOS_pfp.bin");
 191 MODULE_FIRMWARE("radeon/CAICOS_me.bin");
 192 MODULE_FIRMWARE("radeon/CAICOS_mc.bin");
 193 MODULE_FIRMWARE("radeon/CAICOS_smc.bin");
 194 MODULE_FIRMWARE("radeon/CAYMAN_pfp.bin");
 195 MODULE_FIRMWARE("radeon/CAYMAN_me.bin");
 196 MODULE_FIRMWARE("radeon/CAYMAN_mc.bin");
 197 MODULE_FIRMWARE("radeon/CAYMAN_rlc.bin");
 198 MODULE_FIRMWARE("radeon/CAYMAN_smc.bin");
 199 MODULE_FIRMWARE("radeon/ARUBA_pfp.bin");
 200 MODULE_FIRMWARE("radeon/ARUBA_me.bin");
 201 MODULE_FIRMWARE("radeon/ARUBA_rlc.bin");
 202
 203
 204 static const u32 cayman_golden_registers2[] =
 205 {
 206         0x3e5c, 0xffffffff, 0x00000000,
 207         0x3e48, 0xffffffff, 0x00000000,
 208         0x3e4c, 0xffffffff, 0x00000000,
 209         0x3e64, 0xffffffff, 0x00000000,
 210         0x3e50, 0xffffffff, 0x00000000,
 211         0x3e60, 0xffffffff, 0x00000000
 212 };
 213
 214 static const u32 cayman_golden_registers[] =
 215 {
 216         0x5eb4, 0xffffffff, 0x00000002,
 217         0x5e78, 0x8f311ff1, 0x001000f0,
 218         0x3f90, 0xffff0000, 0xff000000,
 219         0x9148, 0xffff0000, 0xff000000,
 220         0x3f94, 0xffff0000, 0xff000000,
 221         0x914c, 0xffff0000, 0xff000000,
 222         0xc78, 0x00000080, 0x00000080,
 223         0xbd4, 0x70073777, 0x00011003,
 224         0xd02c, 0xbfffff1f, 0x08421000,
 225         0xd0b8, 0x73773777, 0x02011003,
 226         0x5bc0, 0x00200000, 0x50100000,
 227         0x98f8, 0x33773777, 0x02011003,
 228         0x98fc, 0xffffffff, 0x76541032,
 229         0x7030, 0x31000311, 0x00000011,
 230         0x2f48, 0x33773777, 0x42010001,
 231         0x6b28, 0x00000010, 0x00000012,
 232         0x7728, 0x00000010, 0x00000012,
 233         0x10328, 0x00000010, 0x00000012,
 234         0x10f28, 0x00000010, 0x00000012,
 235         0x11b28, 0x00000010, 0x00000012,
 236         0x12728, 0x00000010, 0x00000012,
 237         0x240c, 0x000007ff, 0x00000000,
 238         0x8a14, 0xf000001f, 0x00000007,
 239         0x8b24, 0x3fff3fff, 0x00ff0fff,
 240         0x8b10, 0x0000ff0f, 0x00000000,
 241         0x28a4c, 0x07ffffff, 0x06000000,
 242         0x10c, 0x00000001, 0x00010003,
 243         0xa02c, 0xffffffff, 0x0000009b,
 244         0x913c, 0x0000010f, 0x01000100,
 245         0x8c04, 0xf8ff00ff, 0x40600060,
 246         0x28350, 0x00000f01, 0x00000000,
 247         0x9508, 0x3700001f, 0x00000002,
 248         0x960c, 0xffffffff, 0x54763210,
 249         0x88c4, 0x001f3ae3, 0x00000082,
 250         0x88d0, 0xffffffff, 0x0f40df40,
 251         0x88d4, 0x0000001f, 0x00000010,
 252         0x8974, 0xffffffff, 0x00000000
 253 };
 254
 255 static const u32 dvst_golden_registers2[] =
 256 {
 257         0x8f8, 0xffffffff, 0,
 258         0x8fc, 0x00380000, 0,
 259         0x8f8, 0xffffffff, 1,
 260         0x8fc, 0x0e000000, 0
 261 };
 262
 263 static const u32 dvst_golden_registers[] =
 264 {
 265         0x690, 0x3fff3fff, 0x20c00033,
 266         0x918c, 0x0fff0fff, 0x00010006,
 267         0x91a8, 0x0fff0fff, 0x00010006,
 268         0x9150, 0xffffdfff, 0x6e944040,
 269         0x917c, 0x0fff0fff, 0x00030002,
 270         0x9198, 0x0fff0fff, 0x00030002,
 271         0x915c, 0x0fff0fff, 0x00010000,
 272         0x3f90, 0xffff0001, 0xff000000,
 273         0x9178, 0x0fff0fff, 0x00070000,
 274         0x9194, 0x0fff0fff, 0x00070000,
 275         0x9148, 0xffff0001, 0xff000000,
 276         0x9190, 0x0fff0fff, 0x00090008,
 277         0x91ac, 0x0fff0fff, 0x00090008,
 278         0x3f94, 0xffff0000, 0xff000000,
 279         0x914c, 0xffff0000, 0xff000000,
 280         0x929c, 0x00000fff, 0x00000001,
 281         0x55e4, 0xff607fff, 0xfc000100,
 282         0x8a18, 0xff000fff, 0x00000100,
 283         0x8b28, 0xff000fff, 0x00000100,
 284         0x9144, 0xfffc0fff, 0x00000100,
 285         0x6ed8, 0x00010101, 0x00010000,
 286         0x9830, 0xffffffff, 0x00000000,
 287         0x9834, 0xf00fffff, 0x00000400,
 288         0x9838, 0xfffffffe, 0x00000000,
 289         0xd0c0, 0xff000fff, 0x00000100,
 290         0xd02c, 0xbfffff1f, 0x08421000,
 291         0xd0b8, 0x73773777, 0x12010001,
 292         0x5bb0, 0x000000f0, 0x00000070,
 293         0x98f8, 0x73773777, 0x12010001,
 294         0x98fc, 0xffffffff, 0x00000010,
 295         0x9b7c, 0x00ff0000, 0x00fc0000,
 296         0x8030, 0x00001f0f, 0x0000100a,
 297         0x2f48, 0x73773777, 0x12010001,
 298         0x2408, 0x00030000, 0x000c007f,
 299         0x8a14, 0xf000003f, 0x00000007,
 300         0x8b24, 0x3fff3fff, 0x00ff0fff,
 301         0x8b10, 0x0000ff0f, 0x00000000,
 302         0x28a4c, 0x07ffffff, 0x06000000,
 303         0x4d8, 0x00000fff, 0x00000100,
 304         0xa008, 0xffffffff, 0x00010000,
 305         0x913c, 0xffff03ff, 0x01000100,
 306         0x8c00, 0x000000ff, 0x00000003,
 307         0x8c04, 0xf8ff00ff, 0x40600060,
 308         0x8cf0, 0x1fff1fff, 0x08e00410,
 309         0x28350, 0x00000f01, 0x00000000,
 310         0x9508, 0xf700071f, 0x00000002,
 311         0x960c, 0xffffffff, 0x54763210,
 312         0x20ef8, 0x01ff01ff, 0x00000002,
 313         0x20e98, 0xfffffbff, 0x00200000,
 314         0x2015c, 0xffffffff, 0x00000f40,
 315         0x88c4, 0x001f3ae3, 0x00000082,
 316         0x8978, 0x3fffffff, 0x04050140,
 317         0x88d4, 0x0000001f, 0x00000010,
 318         0x8974, 0xffffffff, 0x00000000
 319 };
 320
 321 static const u32 scrapper_golden_registers[] =
 322 {
 323         0x690, 0x3fff3fff, 0x20c00033,
 324         0x918c, 0x0fff0fff, 0x00010006,
 325         0x918c, 0x0fff0fff, 0x00010006,
 326         0x91a8, 0x0fff0fff, 0x00010006,
 327         0x91a8, 0x0fff0fff, 0x00010006,
 328         0x9150, 0xffffdfff, 0x6e944040,
 329         0x9150, 0xffffdfff, 0x6e944040,
 330         0x917c, 0x0fff0fff, 0x00030002,
 331         0x917c, 0x0fff0fff, 0x00030002,
 332         0x9198, 0x0fff0fff, 0x00030002,
 333         0x9198, 0x0fff0fff, 0x00030002,
 334         0x915c, 0x0fff0fff, 0x00010000,
 335         0x915c, 0x0fff0fff, 0x00010000,
 336         0x3f90, 0xffff0001, 0xff000000,
 337         0x3f90, 0xffff0001, 0xff000000,
 338         0x9178, 0x0fff0fff, 0x00070000,
 339         0x9178, 0x0fff0fff, 0x00070000,
 340         0x9194, 0x0fff0fff, 0x00070000,
 341         0x9194, 0x0fff0fff, 0x00070000,
 342         0x9148, 0xffff0001, 0xff000000,
 343         0x9148, 0xffff0001, 0xff000000,
 344         0x9190, 0x0fff0fff, 0x00090008,
 345         0x9190, 0x0fff0fff, 0x00090008,
 346         0x91ac, 0x0fff0fff, 0x00090008,
 347         0x91ac, 0x0fff0fff, 0x00090008,
 348         0x3f94, 0xffff0000, 0xff000000,
 349         0x3f94, 0xffff0000, 0xff000000,
 350         0x914c, 0xffff0000, 0xff000000,
 351         0x914c, 0xffff0000, 0xff000000,
 352         0x929c, 0x00000fff, 0x00000001,
 353         0x929c, 0x00000fff, 0x00000001,
 354         0x55e4, 0xff607fff, 0xfc000100,
 355         0x8a18, 0xff000fff, 0x00000100,
 356         0x8a18, 0xff000fff, 0x00000100,
 357         0x8b28, 0xff000fff, 0x00000100,
 358         0x8b28, 0xff000fff, 0x00000100,
 359         0x9144, 0xfffc0fff, 0x00000100,
 360         0x9144, 0xfffc0fff, 0x00000100,
 361         0x6ed8, 0x00010101, 0x00010000,
 362         0x9830, 0xffffffff, 0x00000000,
 363         0x9830, 0xffffffff, 0x00000000,
 364         0x9834, 0xf00fffff, 0x00000400,
 365         0x9834, 0xf00fffff, 0x00000400,
 366         0x9838, 0xfffffffe, 0x00000000,
 367         0x9838, 0xfffffffe, 0x00000000,
 368         0xd0c0, 0xff000fff, 0x00000100,
 369         0xd02c, 0xbfffff1f, 0x08421000,
 370         0xd02c, 0xbfffff1f, 0x08421000,
 371         0xd0b8, 0x73773777, 0x12010001,
 372         0xd0b8, 0x73773777, 0x12010001,
 373         0x5bb0, 0x000000f0, 0x00000070,
 374         0x98f8, 0x73773777, 0x12010001,
 375         0x98f8, 0x73773777, 0x12010001,
 376         0x98fc, 0xffffffff, 0x00000010,
 377         0x98fc, 0xffffffff, 0x00000010,
 378         0x9b7c, 0x00ff0000, 0x00fc0000,
 379         0x9b7c, 0x00ff0000, 0x00fc0000,
 380         0x8030, 0x00001f0f, 0x0000100a,
 381         0x8030, 0x00001f0f, 0x0000100a,
 382         0x2f48, 0x73773777, 0x12010001,
 383         0x2f48, 0x73773777, 0x12010001,
 384         0x2408, 0x00030000, 0x000c007f,
 385         0x8a14, 0xf000003f, 0x00000007,
 386         0x8a14, 0xf000003f, 0x00000007,
 387         0x8b24, 0x3fff3fff, 0x00ff0fff,
 388         0x8b24, 0x3fff3fff, 0x00ff0fff,
 389         0x8b10, 0x0000ff0f, 0x00000000,
 390         0x8b10, 0x0000ff0f, 0x00000000,
 391         0x28a4c, 0x07ffffff, 0x06000000,
 392         0x28a4c, 0x07ffffff, 0x06000000,
 393         0x4d8, 0x00000fff, 0x00000100,
 394         0x4d8, 0x00000fff, 0x00000100,
 395         0xa008, 0xffffffff, 0x00010000,
 396         0xa008, 0xffffffff, 0x00010000,
 397         0x913c, 0xffff03ff, 0x01000100,
 398         0x913c, 0xffff03ff, 0x01000100,
 399         0x90e8, 0x001fffff, 0x010400c0,
 400         0x8c00, 0x000000ff, 0x00000003,
 401         0x8c00, 0x000000ff, 0x00000003,
 402         0x8c04, 0xf8ff00ff, 0x40600060,
 403         0x8c04, 0xf8ff00ff, 0x40600060,
 404         0x8c30, 0x0000000f, 0x00040005,
 405         0x8cf0, 0x1fff1fff, 0x08e00410,
 406         0x8cf0, 0x1fff1fff, 0x08e00410,
 407         0x900c, 0x00ffffff, 0x0017071f,
 408         0x28350, 0x00000f01, 0x00000000,
 409         0x28350, 0x00000f01, 0x00000000,
 410         0x9508, 0xf700071f, 0x00000002,
 411         0x9508, 0xf700071f, 0x00000002,
 412         0x9688, 0x00300000, 0x0017000f,
 413         0x960c, 0xffffffff, 0x54763210,
 414         0x960c, 0xffffffff, 0x54763210,
 415         0x20ef8, 0x01ff01ff, 0x00000002,
 416         0x20e98, 0xfffffbff, 0x00200000,
 417         0x2015c, 0xffffffff, 0x00000f40,
 418         0x88c4, 0x001f3ae3, 0x00000082,
 419         0x88c4, 0x001f3ae3, 0x00000082,
 420         0x8978, 0x3fffffff, 0x04050140,
 421         0x8978, 0x3fffffff, 0x04050140,
 422         0x88d4, 0x0000001f, 0x00000010,
 423         0x88d4, 0x0000001f, 0x00000010,
 424         0x8974, 0xffffffff, 0x00000000,
 425         0x8974, 0xffffffff, 0x00000000
 426 };
 427
 428 static void ni_init_golden_registers(struct radeon_device *rdev)
 429 {
 430         switch (rdev->family) {
 431         case CHIP_CAYMAN:
 432                 radeon_program_register_sequence(rdev,
 433                                                  cayman_golden_registers,
 434                                                  (const u32)ARRAY_SIZE(cayman_golden_registers));
 435                 radeon_program_register_sequence(rdev,
 436                                                  cayman_golden_registers2,
 437                                                  (const u32)ARRAY_SIZE(cayman_golden_registers2));
 438                 break;
 439         case CHIP_ARUBA:
 440                 if ((rdev->pdev->device == 0x9900) ||
 441                     (rdev->pdev->device == 0x9901) ||
 442                     (rdev->pdev->device == 0x9903) ||
 443                     (rdev->pdev->device == 0x9904) ||
 444                     (rdev->pdev->device == 0x9905) ||
 445                     (rdev->pdev->device == 0x9906) ||
 446                     (rdev->pdev->device == 0x9907) ||
 447                     (rdev->pdev->device == 0x9908) ||
 448                     (rdev->pdev->device == 0x9909) ||
 449                     (rdev->pdev->device == 0x990A) ||
 450                     (rdev->pdev->device == 0x990B) ||
 451                     (rdev->pdev->device == 0x990C) ||
 452                     (rdev->pdev->device == 0x990D) ||
 453                     (rdev->pdev->device == 0x990E) ||
 454                     (rdev->pdev->device == 0x990F) ||
 455                     (rdev->pdev->device == 0x9910) ||
 456                     (rdev->pdev->device == 0x9913) ||
 457                     (rdev->pdev->device == 0x9917) ||
 458                     (rdev->pdev->device == 0x9918)) {
 459                         radeon_program_register_sequence(rdev,
 460                                                          dvst_golden_registers,
 461                                                          (const u32)ARRAY_SIZE(dvst_golden_registers));
 462                         radeon_program_register_sequence(rdev,
 463                                                          dvst_golden_registers2,
 464                                                          (const u32)ARRAY_SIZE(dvst_golden_registers2));
 465                 } else {
 466                         radeon_program_register_sequence(rdev,
 467                                                          scrapper_golden_registers,
 468                                                          (const u32)ARRAY_SIZE(scrapper_golden_registers));
 469                         radeon_program_register_sequence(rdev,
 470                                                          dvst_golden_registers2,
 471                                                          (const u32)ARRAY_SIZE(dvst_golden_registers2));
 472                 }
 473                 break;
 474         default:
 475                 break;
 476         }
 477 }
 478
 479 #define BTC_IO_MC_REGS_SIZE 29
 480
 481 static const u32 barts_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 482         {0x00000077, 0xff010100},
 483         {0x00000078, 0x00000000},
 484         {0x00000079, 0x00001434},
 485         {0x0000007a, 0xcc08ec08},
 486         {0x0000007b, 0x00040000},
 487         {0x0000007c, 0x000080c0},
 488         {0x0000007d, 0x09000000},
 489         {0x0000007e, 0x00210404},
 490         {0x00000081, 0x08a8e800},
 491         {0x00000082, 0x00030444},
 492         {0x00000083, 0x00000000},
 493         {0x00000085, 0x00000001},
 494         {0x00000086, 0x00000002},
 495         {0x00000087, 0x48490000},
 496         {0x00000088, 0x20244647},
 497         {0x00000089, 0x00000005},
 498         {0x0000008b, 0x66030000},
 499         {0x0000008c, 0x00006603},
 500         {0x0000008d, 0x00000100},
 501         {0x0000008f, 0x00001c0a},
 502         {0x00000090, 0xff000001},
 503         {0x00000094, 0x00101101},
 504         {0x00000095, 0x00000fff},
 505         {0x00000096, 0x00116fff},
 506         {0x00000097, 0x60010000},
 507         {0x00000098, 0x10010000},
 508         {0x00000099, 0x00006000},
 509         {0x0000009a, 0x00001000},
 510         {0x0000009f, 0x00946a00}
 511 };
 512
 513 static const u32 turks_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 514         {0x00000077, 0xff010100},
 515         {0x00000078, 0x00000000},
 516         {0x00000079, 0x00001434},
 517         {0x0000007a, 0xcc08ec08},
 518         {0x0000007b, 0x00040000},
 519         {0x0000007c, 0x000080c0},
 520         {0x0000007d, 0x09000000},
 521         {0x0000007e, 0x00210404},
 522         {0x00000081, 0x08a8e800},
 523         {0x00000082, 0x00030444},
 524         {0x00000083, 0x00000000},
 525         {0x00000085, 0x00000001},
 526         {0x00000086, 0x00000002},
 527         {0x00000087, 0x48490000},
 528         {0x00000088, 0x20244647},
 529         {0x00000089, 0x00000005},
 530         {0x0000008b, 0x66030000},
 531         {0x0000008c, 0x00006603},
 532         {0x0000008d, 0x00000100},
 533         {0x0000008f, 0x00001c0a},
 534         {0x00000090, 0xff000001},
 535         {0x00000094, 0x00101101},
 536         {0x00000095, 0x00000fff},
 537         {0x00000096, 0x00116fff},
 538         {0x00000097, 0x60010000},
 539         {0x00000098, 0x10010000},
 540         {0x00000099, 0x00006000},
 541         {0x0000009a, 0x00001000},
 542         {0x0000009f, 0x00936a00}
 543 };
 544
 545 static const u32 caicos_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 546         {0x00000077, 0xff010100},
 547         {0x00000078, 0x00000000},
 548         {0x00000079, 0x00001434},
 549         {0x0000007a, 0xcc08ec08},
 550         {0x0000007b, 0x00040000},
 551         {0x0000007c, 0x000080c0},
 552         {0x0000007d, 0x09000000},
 553         {0x0000007e, 0x00210404},
 554         {0x00000081, 0x08a8e800},
 555         {0x00000082, 0x00030444},
 556         {0x00000083, 0x00000000},
 557         {0x00000085, 0x00000001},
 558         {0x00000086, 0x00000002},
 559         {0x00000087, 0x48490000},
 560         {0x00000088, 0x20244647},
 561         {0x00000089, 0x00000005},
 562         {0x0000008b, 0x66030000},
 563         {0x0000008c, 0x00006603},
 564         {0x0000008d, 0x00000100},
 565         {0x0000008f, 0x00001c0a},
 566         {0x00000090, 0xff000001},
 567         {0x00000094, 0x00101101},
 568         {0x00000095, 0x00000fff},
 569         {0x00000096, 0x00116fff},
 570         {0x00000097, 0x60010000},
 571         {0x00000098, 0x10010000},
 572         {0x00000099, 0x00006000},
 573         {0x0000009a, 0x00001000},
 574         {0x0000009f, 0x00916a00}
 575 };
 576
 577 static const u32 cayman_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 578         {0x00000077, 0xff010100},
 579         {0x00000078, 0x00000000},
 580         {0x00000079, 0x00001434},
 581         {0x0000007a, 0xcc08ec08},
 582         {0x0000007b, 0x00040000},
 583         {0x0000007c, 0x000080c0},
 584         {0x0000007d, 0x09000000},
 585         {0x0000007e, 0x00210404},
 586         {0x00000081, 0x08a8e800},
 587         {0x00000082, 0x00030444},
 588         {0x00000083, 0x00000000},
 589         {0x00000085, 0x00000001},
 590         {0x00000086, 0x00000002},
 591         {0x00000087, 0x48490000},
 592         {0x00000088, 0x20244647},
 593         {0x00000089, 0x00000005},
 594         {0x0000008b, 0x66030000},
 595         {0x0000008c, 0x00006603},
 596         {0x0000008d, 0x00000100},
 597         {0x0000008f, 0x00001c0a},
 598         {0x00000090, 0xff000001},
 599         {0x00000094, 0x00101101},
 600         {0x00000095, 0x00000fff},
 601         {0x00000096, 0x00116fff},
 602         {0x00000097, 0x60010000},
 603         {0x00000098, 0x10010000},
 604         {0x00000099, 0x00006000},
 605         {0x0000009a, 0x00001000},
 606         {0x0000009f, 0x00976b00}
 607 };
 608
 609 int ni_mc_load_microcode(struct radeon_device *rdev)
 610 {
 611         const __be32 *fw_data;
 612         u32 mem_type, running, blackout = 0;
 613         u32 *io_mc_regs;
 614         int i, ucode_size, regs_size;
 615
 616         if (!rdev->mc_fw)
 617                 return -EINVAL;
 618
 619         switch (rdev->family) {
 620         case CHIP_BARTS:
 621                 io_mc_regs = (u32 *)&barts_io_mc_regs;
 622                 ucode_size = BTC_MC_UCODE_SIZE;
 623                 regs_size = BTC_IO_MC_REGS_SIZE;
 624                 break;
 625         case CHIP_TURKS:
 626                 io_mc_regs = (u32 *)&turks_io_mc_regs;
 627                 ucode_size = BTC_MC_UCODE_SIZE;
 628                 regs_size = BTC_IO_MC_REGS_SIZE;
 629                 break;
 630         case CHIP_CAICOS:
 631         default:
 632                 io_mc_regs = (u32 *)&caicos_io_mc_regs;
 633                 ucode_size = BTC_MC_UCODE_SIZE;
 634                 regs_size = BTC_IO_MC_REGS_SIZE;
 635                 break;
 636         case CHIP_CAYMAN:
 637                 io_mc_regs = (u32 *)&cayman_io_mc_regs;
 638                 ucode_size = CAYMAN_MC_UCODE_SIZE;
 639                 regs_size = BTC_IO_MC_REGS_SIZE;
 640                 break;
 641         }
 642
 643         mem_type = (RREG32(MC_SEQ_MISC0) & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT;
 644         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
 645
 646         if ((mem_type == MC_SEQ_MISC0_GDDR5_VALUE) && (running == 0)) {
 647                 if (running) {
 648                         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
 649                         WREG32(MC_SHARED_BLACKOUT_CNTL, 1);
 650                 }
 651
 652                 /* reset the engine and set to writable */
 653                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 654                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
 655
 656                 /* load mc io regs */
 657                 for (i = 0; i < regs_size; i++) {
 658                         WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
 659                         WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
 660                 }
 661                 /* load the MC ucode */
 662                 fw_data = (const __be32 *)rdev->mc_fw->data;
 663                 for (i = 0; i < ucode_size; i++)
 664                         WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
 665
 666                 /* put the engine back into the active state */
 667                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 668                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
 669                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
 670
 671                 /* wait for training to complete */
 672                 for (i = 0; i < rdev->usec_timeout; i++) {
 673                         if (RREG32(MC_IO_PAD_CNTL_D0) & MEM_FALL_OUT_CMD)
 674                                 break;
 675                         udelay(1);
 676                 }
 677
 678                 if (running)
 679                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
 680         }
 681
 682         return 0;
 683 }
 684
 685 int ni_init_microcode(struct radeon_device *rdev)
 686 {
 687         struct platform_device *pdev;
 688         const char *chip_name;
 689         const char *rlc_chip_name;
 690         size_t pfp_req_size, me_req_size, rlc_req_size, mc_req_size;
 691         size_t smc_req_size = 0;
 692         char fw_name[30];
 693         int err;
 694
 695         DRM_DEBUG("\n");
 696
 697         pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
 698         err = IS_ERR(pdev);
 699         if (err) {
 700                 printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
 701                 return -EINVAL;
 702         }
 703
 704         switch (rdev->family) {
 705         case CHIP_BARTS:
 706                 chip_name = "BARTS";
 707                 rlc_chip_name = "BTC";
 708                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 709                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 710                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 711                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 712                 smc_req_size = ALIGN(BARTS_SMC_UCODE_SIZE, 4);
 713                 break;
 714         case CHIP_TURKS:
 715                 chip_name = "TURKS";
 716                 rlc_chip_name = "BTC";
 717                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 718                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 719                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 720                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 721                 smc_req_size = ALIGN(TURKS_SMC_UCODE_SIZE, 4);
 722                 break;
 723         case CHIP_CAICOS:
 724                 chip_name = "CAICOS";
 725                 rlc_chip_name = "BTC";
 726                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 727                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 728                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 729                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 730                 smc_req_size = ALIGN(CAICOS_SMC_UCODE_SIZE, 4);
 731                 break;
 732         case CHIP_CAYMAN:
 733                 chip_name = "CAYMAN";
 734                 rlc_chip_name = "CAYMAN";
 735                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 736                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 737                 rlc_req_size = CAYMAN_RLC_UCODE_SIZE * 4;
 738                 mc_req_size = CAYMAN_MC_UCODE_SIZE * 4;
 739                 smc_req_size = ALIGN(CAYMAN_SMC_UCODE_SIZE, 4);
 740                 break;
 741         case CHIP_ARUBA:
 742                 chip_name = "ARUBA";
 743                 rlc_chip_name = "ARUBA";
 744                 /* pfp/me same size as CAYMAN */
 745                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 746                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 747                 rlc_req_size = ARUBA_RLC_UCODE_SIZE * 4;
 748                 mc_req_size = 0;
 749                 break;
 750         default: BUG();
 751         }
 752
 753         DRM_INFO("Loading %s Microcode\n", chip_name);
 754
 755         snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
 756         err = request_firmware(&rdev->pfp_fw, fw_name, &pdev->dev);
 757         if (err)
 758                 goto out;
 759         if (rdev->pfp_fw->size != pfp_req_size) {
 760                 printk(KERN_ERR
 761                        "ni_cp: Bogus length %zu in firmware \"%s\"\n",
 762                        rdev->pfp_fw->size, fw_name);
 763                 err = -EINVAL;
 764                 goto out;
 765         }
 766
 767         snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
 768         err = request_firmware(&rdev->me_fw, fw_name, &pdev->dev);
 769         if (err)
 770                 goto out;
 771         if (rdev->me_fw->size != me_req_size) {
 772                 printk(KERN_ERR
 773                        "ni_cp: Bogus length %zu in firmware \"%s\"\n",
 774                        rdev->me_fw->size, fw_name);
 775                 err = -EINVAL;
 776         }
 777
 778         snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
 779         err = request_firmware(&rdev->rlc_fw, fw_name, &pdev->dev);
 780         if (err)
 781                 goto out;
 782         if (rdev->rlc_fw->size != rlc_req_size) {
 783                 printk(KERN_ERR
 784                        "ni_rlc: Bogus length %zu in firmware \"%s\"\n",
 785                        rdev->rlc_fw->size, fw_name);
 786                 err = -EINVAL;
 787         }
 788
 789         /* no MC ucode on TN */
 790         if (!(rdev->flags & RADEON_IS_IGP)) {
 791                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
 792                 err = request_firmware(&rdev->mc_fw, fw_name, &pdev->dev);
 793                 if (err)
 794                         goto out;
 795                 if (rdev->mc_fw->size != mc_req_size) {
 796                         printk(KERN_ERR
 797                                "ni_mc: Bogus length %zu in firmware \"%s\"\n",
 798                                rdev->mc_fw->size, fw_name);
 799                         err = -EINVAL;
 800                 }
 801         }
 802
 803         if ((rdev->family >= CHIP_BARTS) && (rdev->family <= CHIP_CAYMAN)) {
 804                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
 805                 err = request_firmware(&rdev->smc_fw, fw_name, &pdev->dev);
 806                 if (err)
 807                         goto out;
 808                 if (rdev->smc_fw->size != smc_req_size) {
 809                         printk(KERN_ERR
 810                                "ni_mc: Bogus length %zu in firmware \"%s\"\n",
 811                                rdev->mc_fw->size, fw_name);
 812                         err = -EINVAL;
 813                 }
 814         }
 815
 816 out:
 817         platform_device_unregister(pdev);
 818
 819         if (err) {
 820                 if (err != -EINVAL)
 821                         printk(KERN_ERR
 822                                "ni_cp: Failed to load firmware \"%s\"\n",
 823                                fw_name);
 824                 release_firmware(rdev->pfp_fw);
 825                 rdev->pfp_fw = NULL;
 826                 release_firmware(rdev->me_fw);
 827                 rdev->me_fw = NULL;
 828                 release_firmware(rdev->rlc_fw);
 829                 rdev->rlc_fw = NULL;
 830                 release_firmware(rdev->mc_fw);
 831                 rdev->mc_fw = NULL;
 832         }
 833         return err;
 834 }
 835
 836 int tn_get_temp(struct radeon_device *rdev)
 837 {
 838         u32 temp = RREG32_SMC(TN_CURRENT_GNB_TEMP) & 0x7ff;
 839         int actual_temp = (temp / 8) - 49;
 840
 841         return actual_temp * 1000;
 842 }
 843
 844 /*
 845  * Core functions
 846  */
 847 static void cayman_gpu_init(struct radeon_device *rdev)
 848 {
 849         u32 gb_addr_config = 0;
 850         u32 mc_shared_chmap, mc_arb_ramcfg;
 851         u32 cgts_tcc_disable;
 852         u32 sx_debug_1;
 853         u32 smx_dc_ctl0;
 854         u32 cgts_sm_ctrl_reg;
 855         u32 hdp_host_path_cntl;
 856         u32 tmp;
 857         u32 disabled_rb_mask;
 858         int i, j;
 859
 860         switch (rdev->family) {
 861         case CHIP_CAYMAN:
 862                 rdev->config.cayman.max_shader_engines = 2;
 863                 rdev->config.cayman.max_pipes_per_simd = 4;
 864                 rdev->config.cayman.max_tile_pipes = 8;
 865                 rdev->config.cayman.max_simds_per_se = 12;
 866                 rdev->config.cayman.max_backends_per_se = 4;
 867                 rdev->config.cayman.max_texture_channel_caches = 8;
 868                 rdev->config.cayman.max_gprs = 256;
 869                 rdev->config.cayman.max_threads = 256;
 870                 rdev->config.cayman.max_gs_threads = 32;
 871                 rdev->config.cayman.max_stack_entries = 512;
 872                 rdev->config.cayman.sx_num_of_sets = 8;
 873                 rdev->config.cayman.sx_max_export_size = 256;
 874                 rdev->config.cayman.sx_max_export_pos_size = 64;
 875                 rdev->config.cayman.sx_max_export_smx_size = 192;
 876                 rdev->config.cayman.max_hw_contexts = 8;
 877                 rdev->config.cayman.sq_num_cf_insts = 2;
 878
 879                 rdev->config.cayman.sc_prim_fifo_size = 0x100;
 880                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 881                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 882                 gb_addr_config = CAYMAN_GB_ADDR_CONFIG_GOLDEN;
 883                 break;
 884         case CHIP_ARUBA:
 885         default:
 886                 rdev->config.cayman.max_shader_engines = 1;
 887                 rdev->config.cayman.max_pipes_per_simd = 4;
 888                 rdev->config.cayman.max_tile_pipes = 2;
 889                 if ((rdev->pdev->device == 0x9900) ||
 890                     (rdev->pdev->device == 0x9901) ||
 891                     (rdev->pdev->device == 0x9905) ||
 892                     (rdev->pdev->device == 0x9906) ||
 893                     (rdev->pdev->device == 0x9907) ||
 894                     (rdev->pdev->device == 0x9908) ||
 895                     (rdev->pdev->device == 0x9909) ||
 896                     (rdev->pdev->device == 0x990B) ||
 897                     (rdev->pdev->device == 0x990C) ||
 898                     (rdev->pdev->device == 0x990F) ||
 899                     (rdev->pdev->device == 0x9910) ||
 900                     (rdev->pdev->device == 0x9917) ||
 901                     (rdev->pdev->device == 0x9999) ||
 902                     (rdev->pdev->device == 0x999C)) {
 903                         rdev->config.cayman.max_simds_per_se = 6;
 904                         rdev->config.cayman.max_backends_per_se = 2;
 905                 } else if ((rdev->pdev->device == 0x9903) ||
 906                            (rdev->pdev->device == 0x9904) ||
 907                            (rdev->pdev->device == 0x990A) ||
 908                            (rdev->pdev->device == 0x990D) ||
 909                            (rdev->pdev->device == 0x990E) ||
 910                            (rdev->pdev->device == 0x9913) ||
 911                            (rdev->pdev->device == 0x9918) ||
 912                            (rdev->pdev->device == 0x999D)) {
 913                         rdev->config.cayman.max_simds_per_se = 4;
 914                         rdev->config.cayman.max_backends_per_se = 2;
 915                 } else if ((rdev->pdev->device == 0x9919) ||
 916                            (rdev->pdev->device == 0x9990) ||
 917                            (rdev->pdev->device == 0x9991) ||
 918                            (rdev->pdev->device == 0x9994) ||
 919                            (rdev->pdev->device == 0x9995) ||
 920                            (rdev->pdev->device == 0x9996) ||
 921                            (rdev->pdev->device == 0x999A) ||
 922                            (rdev->pdev->device == 0x99A0)) {
 923                         rdev->config.cayman.max_simds_per_se = 3;
 924                         rdev->config.cayman.max_backends_per_se = 1;
 925                 } else {
 926                         rdev->config.cayman.max_simds_per_se = 2;
 927                         rdev->config.cayman.max_backends_per_se = 1;
 928                 }
 929                 rdev->config.cayman.max_texture_channel_caches = 2;
 930                 rdev->config.cayman.max_gprs = 256;
 931                 rdev->config.cayman.max_threads = 256;
 932                 rdev->config.cayman.max_gs_threads = 32;
 933                 rdev->config.cayman.max_stack_entries = 512;
 934                 rdev->config.cayman.sx_num_of_sets = 8;
 935                 rdev->config.cayman.sx_max_export_size = 256;
 936                 rdev->config.cayman.sx_max_export_pos_size = 64;
 937                 rdev->config.cayman.sx_max_export_smx_size = 192;
 938                 rdev->config.cayman.max_hw_contexts = 8;
 939                 rdev->config.cayman.sq_num_cf_insts = 2;
 940
 941                 rdev->config.cayman.sc_prim_fifo_size = 0x40;
 942                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 943                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 944                 gb_addr_config = ARUBA_GB_ADDR_CONFIG_GOLDEN;
 945                 break;
 946         }
 947
 948         /* Initialize HDP */
 949         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
 950                 WREG32((0x2c14 + j), 0x00000000);
 951                 WREG32((0x2c18 + j), 0x00000000);
 952                 WREG32((0x2c1c + j), 0x00000000);
 953                 WREG32((0x2c20 + j), 0x00000000);
 954                 WREG32((0x2c24 + j), 0x00000000);
 955         }
 956
 957         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
 958
 959         evergreen_fix_pci_max_read_req_size(rdev);
 960
 961         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
 962         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
 963
 964         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
 965         rdev->config.cayman.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
 966         if (rdev->config.cayman.mem_row_size_in_kb > 4)
 967                 rdev->config.cayman.mem_row_size_in_kb = 4;
 968         /* XXX use MC settings? */
 969         rdev->config.cayman.shader_engine_tile_size = 32;
 970         rdev->config.cayman.num_gpus = 1;
 971         rdev->config.cayman.multi_gpu_tile_size = 64;
 972
 973         tmp = (gb_addr_config & NUM_PIPES_MASK) >> NUM_PIPES_SHIFT;
 974         rdev->config.cayman.num_tile_pipes = (1 << tmp);
 975         tmp = (gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT;
 976         rdev->config.cayman.mem_max_burst_length_bytes = (tmp + 1) * 256;
 977         tmp = (gb_addr_config & NUM_SHADER_ENGINES_MASK) >> NUM_SHADER_ENGINES_SHIFT;
 978         rdev->config.cayman.num_shader_engines = tmp + 1;
 979         tmp = (gb_addr_config & NUM_GPUS_MASK) >> NUM_GPUS_SHIFT;
 980         rdev->config.cayman.num_gpus = tmp + 1;
 981         tmp = (gb_addr_config & MULTI_GPU_TILE_SIZE_MASK) >> MULTI_GPU_TILE_SIZE_SHIFT;
 982         rdev->config.cayman.multi_gpu_tile_size = 1 << tmp;
 983         tmp = (gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT;
 984         rdev->config.cayman.mem_row_size_in_kb = 1 << tmp;
 985
 986
 987         /* setup tiling info dword.  gb_addr_config is not adequate since it does
 988          * not have bank info, so create a custom tiling dword.
 989          * bits 3:0   num_pipes
 990          * bits 7:4   num_banks
 991          * bits 11:8  group_size
 992          * bits 15:12 row_size
 993          */
 994         rdev->config.cayman.tile_config = 0;
 995         switch (rdev->config.cayman.num_tile_pipes) {
 996         case 1:
 997         default:
 998                 rdev->config.cayman.tile_config |= (0 << 0);
 999                 break;
1000         case 2:
1001                 rdev->config.cayman.tile_config |= (1 << 0);
1002                 break;
1003         case 4:
1004                 rdev->config.cayman.tile_config |= (2 << 0);
1005                 break;
1006         case 8:
1007                 rdev->config.cayman.tile_config |= (3 << 0);
1008                 break;
1009         }
1010
1011         /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
1012         if (rdev->flags & RADEON_IS_IGP)
1013                 rdev->config.cayman.tile_config |= 1 << 4;
1014         else {
1015                 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
1016                 case 0: /* four banks */
1017                         rdev->config.cayman.tile_config |= 0 << 4;
1018                         break;
1019                 case 1: /* eight banks */
1020                         rdev->config.cayman.tile_config |= 1 << 4;
1021                         break;
1022                 case 2: /* sixteen banks */
1023                 default:
1024                         rdev->config.cayman.tile_config |= 2 << 4;
1025                         break;
1026                 }
1027         }
1028         rdev->config.cayman.tile_config |=
1029                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
1030         rdev->config.cayman.tile_config |=
1031                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
1032
1033         tmp = 0;
1034         for (i = (rdev->config.cayman.max_shader_engines - 1); i >= 0; i--) {
1035                 u32 rb_disable_bitmap;
1036
1037                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1038                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1039                 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
1040                 tmp <<= 4;
1041                 tmp |= rb_disable_bitmap;
1042         }
1043         /* enabled rb are just the one not disabled :) */
1044         disabled_rb_mask = tmp;
1045         tmp = 0;
1046         for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1047                 tmp |= (1 << i);
1048         /* if all the backends are disabled, fix it up here */
1049         if ((disabled_rb_mask & tmp) == tmp) {
1050                 for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1051                         disabled_rb_mask &= ~(1 << i);
1052         }
1053
1054         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1055         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1056
1057         WREG32(GB_ADDR_CONFIG, gb_addr_config);
1058         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
1059         if (ASIC_IS_DCE6(rdev))
1060                 WREG32(DMIF_ADDR_CALC, gb_addr_config);
1061         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
1062         WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
1063         WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1064         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
1065         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
1066         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
1067
1068         if ((rdev->config.cayman.max_backends_per_se == 1) &&
1069             (rdev->flags & RADEON_IS_IGP)) {
1070                 if ((disabled_rb_mask & 3) == 1) {
1071                         /* RB0 disabled, RB1 enabled */
1072                         tmp = 0x11111111;
1073                 } else {
1074                         /* RB1 disabled, RB0 enabled */
1075                         tmp = 0x00000000;
1076                 }
1077         } else {
1078                 tmp = gb_addr_config & NUM_PIPES_MASK;
1079                 tmp = r6xx_remap_render_backend(rdev, tmp,
1080                                                 rdev->config.cayman.max_backends_per_se *
1081                                                 rdev->config.cayman.max_shader_engines,
1082                                                 CAYMAN_MAX_BACKENDS, disabled_rb_mask);
1083         }
1084         WREG32(GB_BACKEND_MAP, tmp);
1085
1086         cgts_tcc_disable = 0xffff0000;
1087         for (i = 0; i < rdev->config.cayman.max_texture_channel_caches; i++)
1088                 cgts_tcc_disable &= ~(1 << (16 + i));
1089         WREG32(CGTS_TCC_DISABLE, cgts_tcc_disable);
1090         WREG32(CGTS_SYS_TCC_DISABLE, cgts_tcc_disable);
1091         WREG32(CGTS_USER_SYS_TCC_DISABLE, cgts_tcc_disable);
1092         WREG32(CGTS_USER_TCC_DISABLE, cgts_tcc_disable);
1093
1094         /* reprogram the shader complex */
1095         cgts_sm_ctrl_reg = RREG32(CGTS_SM_CTRL_REG);
1096         for (i = 0; i < 16; i++)
1097                 WREG32(CGTS_SM_CTRL_REG, OVERRIDE);
1098         WREG32(CGTS_SM_CTRL_REG, cgts_sm_ctrl_reg);
1099
1100         /* set HW defaults for 3D engine */
1101         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
1102
1103         sx_debug_1 = RREG32(SX_DEBUG_1);
1104         sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
1105         WREG32(SX_DEBUG_1, sx_debug_1);
1106
1107         smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
1108         smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
1109         smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.cayman.sx_num_of_sets);
1110         WREG32(SMX_DC_CTL0, smx_dc_ctl0);
1111
1112         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4) | CRC_SIMD_ID_WADDR_DISABLE);
1113
1114         /* need to be explicitly zero-ed */
1115         WREG32(VGT_OFFCHIP_LDS_BASE, 0);
1116         WREG32(SQ_LSTMP_RING_BASE, 0);
1117         WREG32(SQ_HSTMP_RING_BASE, 0);
1118         WREG32(SQ_ESTMP_RING_BASE, 0);
1119         WREG32(SQ_GSTMP_RING_BASE, 0);
1120         WREG32(SQ_VSTMP_RING_BASE, 0);
1121         WREG32(SQ_PSTMP_RING_BASE, 0);
1122
1123         WREG32(TA_CNTL_AUX, DISABLE_CUBE_ANISO);
1124
1125         WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.cayman.sx_max_export_size / 4) - 1) |
1126                                         POSITION_BUFFER_SIZE((rdev->config.cayman.sx_max_export_pos_size / 4) - 1) |
1127                                         SMX_BUFFER_SIZE((rdev->config.cayman.sx_max_export_smx_size / 4) - 1)));
1128
1129         WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.cayman.sc_prim_fifo_size) |
1130                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_hiz_tile_fifo_size) |
1131                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_earlyz_tile_fifo_size)));
1132
1133
1134         WREG32(VGT_NUM_INSTANCES, 1);
1135
1136         WREG32(CP_PERFMON_CNTL, 0);
1137
1138         WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.cayman.sq_num_cf_insts) |
1139                                   FETCH_FIFO_HIWATER(0x4) |
1140                                   DONE_FIFO_HIWATER(0xe0) |
1141                                   ALU_UPDATE_FIFO_HIWATER(0x8)));
1142
1143         WREG32(SQ_GPR_RESOURCE_MGMT_1, NUM_CLAUSE_TEMP_GPRS(4));
1144         WREG32(SQ_CONFIG, (VC_ENABLE |
1145                            EXPORT_SRC_C |
1146                            GFX_PRIO(0) |
1147                            CS1_PRIO(0) |
1148                            CS2_PRIO(1)));
1149         WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, DYN_GPR_ENABLE);
1150
1151         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
1152                                           FORCE_EOV_MAX_REZ_CNT(255)));
1153
1154         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
1155                AUTO_INVLD_EN(ES_AND_GS_AUTO));
1156
1157         WREG32(VGT_GS_VERTEX_REUSE, 16);
1158         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1159
1160         WREG32(CB_PERF_CTR0_SEL_0, 0);
1161         WREG32(CB_PERF_CTR0_SEL_1, 0);
1162         WREG32(CB_PERF_CTR1_SEL_0, 0);
1163         WREG32(CB_PERF_CTR1_SEL_1, 0);
1164         WREG32(CB_PERF_CTR2_SEL_0, 0);
1165         WREG32(CB_PERF_CTR2_SEL_1, 0);
1166         WREG32(CB_PERF_CTR3_SEL_0, 0);
1167         WREG32(CB_PERF_CTR3_SEL_1, 0);
1168
1169         tmp = RREG32(HDP_MISC_CNTL);
1170         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
1171         WREG32(HDP_MISC_CNTL, tmp);
1172
1173         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
1174         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1175
1176         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
1177
1178         udelay(50);
1179
1180         /* set clockgating golden values on TN */
1181         if (rdev->family == CHIP_ARUBA) {
1182                 tmp = RREG32_CG(CG_CGTT_LOCAL_0);
1183                 tmp &= ~0x00380000;
1184                 WREG32_CG(CG_CGTT_LOCAL_0, tmp);
1185                 tmp = RREG32_CG(CG_CGTT_LOCAL_1);
1186                 tmp &= ~0x0e000000;
1187                 WREG32_CG(CG_CGTT_LOCAL_1, tmp);
1188         }
1189 }
1190
1191 /*
1192  * GART
1193  */
1194 void cayman_pcie_gart_tlb_flush(struct radeon_device *rdev)
1195 {
1196         /* flush hdp cache */
1197         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
1198
1199         /* bits 0-7 are the VM contexts0-7 */
1200         WREG32(VM_INVALIDATE_REQUEST, 1);
1201 }
1202
1203 static int cayman_pcie_gart_enable(struct radeon_device *rdev)
1204 {
1205         int i, r;
1206
1207         if (rdev->gart.robj == NULL) {
1208                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
1209                 return -EINVAL;
1210         }
1211         r = radeon_gart_table_vram_pin(rdev);
1212         if (r)
1213                 return r;
1214         radeon_gart_restore(rdev);
1215         /* Setup TLB control */
1216         WREG32(MC_VM_MX_L1_TLB_CNTL,
1217                (0xA << 7) |
1218                ENABLE_L1_TLB |
1219                ENABLE_L1_FRAGMENT_PROCESSING |
1220                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1221                ENABLE_ADVANCED_DRIVER_MODEL |
1222                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1223         /* Setup L2 cache */
1224         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
1225                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1226                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1227                EFFECTIVE_L2_QUEUE_SIZE(7) |
1228                CONTEXT1_IDENTITY_ACCESS_MODE(1));
1229         WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
1230         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1231                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1232         /* setup context0 */
1233         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
1234         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
1235         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
1236         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
1237                         (u32)(rdev->dummy_page.addr >> 12));
1238         WREG32(VM_CONTEXT0_CNTL2, 0);
1239         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
1240                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
1241
1242         WREG32(0x15D4, 0);
1243         WREG32(0x15D8, 0);
1244         WREG32(0x15DC, 0);
1245
1246         /* empty context1-7 */
1247         /* Assign the pt base to something valid for now; the pts used for
1248          * the VMs are determined by the application and setup and assigned
1249          * on the fly in the vm part of radeon_gart.c
1250          */
1251         for (i = 1; i < 8; i++) {
1252                 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (i << 2), 0);
1253                 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (i << 2), rdev->vm_manager.max_pfn);
1254                 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
1255                         rdev->gart.table_addr >> 12);
1256         }
1257
1258         /* enable context1-7 */
1259         WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
1260                (u32)(rdev->dummy_page.addr >> 12));
1261         WREG32(VM_CONTEXT1_CNTL2, 4);
1262         WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
1263                                 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1264                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1265                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1266                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1267                                 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
1268                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
1269                                 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
1270                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
1271                                 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
1272                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
1273                                 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1274                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1275
1276         cayman_pcie_gart_tlb_flush(rdev);
1277         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
1278                  (unsigned)(rdev->mc.gtt_size >> 20),
1279                  (unsigned long long)rdev->gart.table_addr);
1280         rdev->gart.ready = true;
1281         return 0;
1282 }
1283
1284 static void cayman_pcie_gart_disable(struct radeon_device *rdev)
1285 {
1286         /* Disable all tables */
1287         WREG32(VM_CONTEXT0_CNTL, 0);
1288         WREG32(VM_CONTEXT1_CNTL, 0);
1289         /* Setup TLB control */
1290         WREG32(MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING |
1291                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1292                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1293         /* Setup L2 cache */
1294         WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1295                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1296                EFFECTIVE_L2_QUEUE_SIZE(7) |
1297                CONTEXT1_IDENTITY_ACCESS_MODE(1));
1298         WREG32(VM_L2_CNTL2, 0);
1299         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1300                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1301         radeon_gart_table_vram_unpin(rdev);
1302 }
1303
1304 static void cayman_pcie_gart_fini(struct radeon_device *rdev)
1305 {
1306         cayman_pcie_gart_disable(rdev);
1307         radeon_gart_table_vram_free(rdev);
1308         radeon_gart_fini(rdev);
1309 }
1310
1311 void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
1312                               int ring, u32 cp_int_cntl)
1313 {
1314         u32 srbm_gfx_cntl = RREG32(SRBM_GFX_CNTL) & ~3;
1315
1316         WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl | (ring & 3));
1317         WREG32(CP_INT_CNTL, cp_int_cntl);
1318 }
1319
1320 /*
1321  * CP.
1322  */
1323 void cayman_fence_ring_emit(struct radeon_device *rdev,
1324                             struct radeon_fence *fence)
1325 {
1326         struct radeon_ring *ring = &rdev->ring[fence->ring];
1327         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
1328
1329         /* flush read cache over gart for this vmid */
1330         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1331         radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
1332         radeon_ring_write(ring, 0);
1333         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1334         radeon_ring_write(ring, PACKET3_TC_ACTION_ENA | PACKET3_SH_ACTION_ENA);
1335         radeon_ring_write(ring, 0xFFFFFFFF);
1336         radeon_ring_write(ring, 0);
1337         radeon_ring_write(ring, 10); /* poll interval */
1338         /* EVENT_WRITE_EOP - flush caches, send int */
1339         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
1340         radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
1341         radeon_ring_write(ring, addr & 0xffffffff);
1342         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
1343         radeon_ring_write(ring, fence->seq);
1344         radeon_ring_write(ring, 0);
1345 }
1346
1347 void cayman_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
1348 {
1349         struct radeon_ring *ring = &rdev->ring[ib->ring];
1350
1351         /* set to DX10/11 mode */
1352         radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
1353         radeon_ring_write(ring, 1);
1354
1355         if (ring->rptr_save_reg) {
1356                 uint32_t next_rptr = ring->wptr + 3 + 4 + 8;
1357                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1358                 radeon_ring_write(ring, ((ring->rptr_save_reg -
1359                                           PACKET3_SET_CONFIG_REG_START) >> 2));
1360                 radeon_ring_write(ring, next_rptr);
1361         }
1362
1363         radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
1364         radeon_ring_write(ring,
1365 #ifdef __BIG_ENDIAN
1366                           (2 << 0) |
1367 #endif
1368                           (ib->gpu_addr & 0xFFFFFFFC));
1369         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
1370         radeon_ring_write(ring, ib->length_dw |
1371                           (ib->vm ? (ib->vm->id << 24) : 0));
1372
1373         /* flush read cache over gart for this vmid */
1374         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1375         radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
1376         radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
1377         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1378         radeon_ring_write(ring, PACKET3_TC_ACTION_ENA | PACKET3_SH_ACTION_ENA);
1379         radeon_ring_write(ring, 0xFFFFFFFF);
1380         radeon_ring_write(ring, 0);
1381         radeon_ring_write(ring, 10); /* poll interval */
1382 }
1383
1384 void cayman_uvd_semaphore_emit(struct radeon_device *rdev,
1385                                struct radeon_ring *ring,
1386                                struct radeon_semaphore *semaphore,
1387                                bool emit_wait)
1388 {
1389         uint64_t addr = semaphore->gpu_addr;
1390
1391         radeon_ring_write(ring, PACKET0(UVD_SEMA_ADDR_LOW, 0));
1392         radeon_ring_write(ring, (addr >> 3) & 0x000FFFFF);
1393
1394         radeon_ring_write(ring, PACKET0(UVD_SEMA_ADDR_HIGH, 0));
1395         radeon_ring_write(ring, (addr >> 23) & 0x000FFFFF);
1396
1397         radeon_ring_write(ring, PACKET0(UVD_SEMA_CMD, 0));
1398         radeon_ring_write(ring, 0x80 | (emit_wait ? 1 : 0));
1399 }
1400
1401 static void cayman_cp_enable(struct radeon_device *rdev, bool enable)
1402 {
1403         if (enable)
1404                 WREG32(CP_ME_CNTL, 0);
1405         else {
1406                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1407                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
1408                 WREG32(SCRATCH_UMSK, 0);
1409                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1410         }
1411 }
1412
1413 static int cayman_cp_load_microcode(struct radeon_device *rdev)
1414 {
1415         const __be32 *fw_data;
1416         int i;
1417
1418         if (!rdev->me_fw || !rdev->pfp_fw)
1419                 return -EINVAL;
1420
1421         cayman_cp_enable(rdev, false);
1422
1423         fw_data = (const __be32 *)rdev->pfp_fw->data;
1424         WREG32(CP_PFP_UCODE_ADDR, 0);
1425         for (i = 0; i < CAYMAN_PFP_UCODE_SIZE; i++)
1426                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1427         WREG32(CP_PFP_UCODE_ADDR, 0);
1428
1429         fw_data = (const __be32 *)rdev->me_fw->data;
1430         WREG32(CP_ME_RAM_WADDR, 0);
1431         for (i = 0; i < CAYMAN_PM4_UCODE_SIZE; i++)
1432                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1433
1434         WREG32(CP_PFP_UCODE_ADDR, 0);
1435         WREG32(CP_ME_RAM_WADDR, 0);
1436         WREG32(CP_ME_RAM_RADDR, 0);
1437         return 0;
1438 }
1439
1440 static int cayman_cp_start(struct radeon_device *rdev)
1441 {
1442         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1443         int r, i;
1444
1445         r = radeon_ring_lock(rdev, ring, 7);
1446         if (r) {
1447                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1448                 return r;
1449         }
1450         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
1451         radeon_ring_write(ring, 0x1);
1452         radeon_ring_write(ring, 0x0);
1453         radeon_ring_write(ring, rdev->config.cayman.max_hw_contexts - 1);
1454         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1455         radeon_ring_write(ring, 0);
1456         radeon_ring_write(ring, 0);
1457         radeon_ring_unlock_commit(rdev, ring);
1458
1459         cayman_cp_enable(rdev, true);
1460
1461         r = radeon_ring_lock(rdev, ring, cayman_default_size + 19);
1462         if (r) {
1463                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1464                 return r;
1465         }
1466
1467         /* setup clear context state */
1468         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1469         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1470
1471         for (i = 0; i < cayman_default_size; i++)
1472                 radeon_ring_write(ring, cayman_default_state[i]);
1473
1474         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1475         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
1476
1477         /* set clear context state */
1478         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
1479         radeon_ring_write(ring, 0);
1480
1481         /* SQ_VTX_BASE_VTX_LOC */
1482         radeon_ring_write(ring, 0xc0026f00);
1483         radeon_ring_write(ring, 0x00000000);
1484         radeon_ring_write(ring, 0x00000000);
1485         radeon_ring_write(ring, 0x00000000);
1486
1487         /* Clear consts */
1488         radeon_ring_write(ring, 0xc0036f00);
1489         radeon_ring_write(ring, 0x00000bc4);
1490         radeon_ring_write(ring, 0xffffffff);
1491         radeon_ring_write(ring, 0xffffffff);
1492         radeon_ring_write(ring, 0xffffffff);
1493
1494         radeon_ring_write(ring, 0xc0026900);
1495         radeon_ring_write(ring, 0x00000316);
1496         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
1497         radeon_ring_write(ring, 0x00000010); /*  */
1498
1499         radeon_ring_unlock_commit(rdev, ring);
1500
1501         /* XXX init other rings */
1502
1503         return 0;
1504 }
1505
1506 static void cayman_cp_fini(struct radeon_device *rdev)
1507 {
1508         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1509         cayman_cp_enable(rdev, false);
1510         radeon_ring_fini(rdev, ring);
1511         radeon_scratch_free(rdev, ring->rptr_save_reg);
1512 }
1513
1514 static int cayman_cp_resume(struct radeon_device *rdev)
1515 {
1516         static const int ridx[] = {
1517                 RADEON_RING_TYPE_GFX_INDEX,
1518                 CAYMAN_RING_TYPE_CP1_INDEX,
1519                 CAYMAN_RING_TYPE_CP2_INDEX
1520         };
1521         static const unsigned cp_rb_cntl[] = {
1522                 CP_RB0_CNTL,
1523                 CP_RB1_CNTL,
1524                 CP_RB2_CNTL,
1525         };
1526         static const unsigned cp_rb_rptr_addr[] = {
1527                 CP_RB0_RPTR_ADDR,
1528                 CP_RB1_RPTR_ADDR,
1529                 CP_RB2_RPTR_ADDR
1530         };
1531         static const unsigned cp_rb_rptr_addr_hi[] = {
1532                 CP_RB0_RPTR_ADDR_HI,
1533                 CP_RB1_RPTR_ADDR_HI,
1534                 CP_RB2_RPTR_ADDR_HI
1535         };
1536         static const unsigned cp_rb_base[] = {
1537                 CP_RB0_BASE,
1538                 CP_RB1_BASE,
1539                 CP_RB2_BASE
1540         };
1541         struct radeon_ring *ring;
1542         int i, r;
1543
1544         /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
1545         WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
1546                                  SOFT_RESET_PA |
1547                                  SOFT_RESET_SH |
1548                                  SOFT_RESET_VGT |
1549                                  SOFT_RESET_SPI |
1550                                  SOFT_RESET_SX));
1551         RREG32(GRBM_SOFT_RESET);
1552         mdelay(15);
1553         WREG32(GRBM_SOFT_RESET, 0);
1554         RREG32(GRBM_SOFT_RESET);
1555
1556         WREG32(CP_SEM_WAIT_TIMER, 0x0);
1557         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
1558
1559         /* Set the write pointer delay */
1560         WREG32(CP_RB_WPTR_DELAY, 0);
1561
1562         WREG32(CP_DEBUG, (1 << 27));
1563
1564         /* set the wb address whether it's enabled or not */
1565         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
1566         WREG32(SCRATCH_UMSK, 0xff);
1567
1568         for (i = 0; i < 3; ++i) {
1569                 uint32_t rb_cntl;
1570                 uint64_t addr;
1571
1572                 /* Set ring buffer size */
1573                 ring = &rdev->ring[ridx[i]];
1574                 rb_cntl = drm_order(ring->ring_size / 8);
1575                 rb_cntl |= drm_order(RADEON_GPU_PAGE_SIZE/8) << 8;
1576 #ifdef __BIG_ENDIAN
1577                 rb_cntl |= BUF_SWAP_32BIT;
1578 #endif
1579                 WREG32(cp_rb_cntl[i], rb_cntl);
1580
1581                 /* set the wb address whether it's enabled or not */
1582                 addr = rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET;
1583                 WREG32(cp_rb_rptr_addr[i], addr & 0xFFFFFFFC);
1584                 WREG32(cp_rb_rptr_addr_hi[i], upper_32_bits(addr) & 0xFF);
1585         }
1586
1587         /* set the rb base addr, this causes an internal reset of ALL rings */
1588         for (i = 0; i < 3; ++i) {
1589                 ring = &rdev->ring[ridx[i]];
1590                 WREG32(cp_rb_base[i], ring->gpu_addr >> 8);
1591         }
1592
1593         for (i = 0; i < 3; ++i) {
1594                 /* Initialize the ring buffer's read and write pointers */
1595                 ring = &rdev->ring[ridx[i]];
1596                 WREG32_P(cp_rb_cntl[i], RB_RPTR_WR_ENA, ~RB_RPTR_WR_ENA);
1597
1598                 ring->rptr = ring->wptr = 0;
1599                 WREG32(ring->rptr_reg, ring->rptr);
1600                 WREG32(ring->wptr_reg, ring->wptr);
1601
1602                 mdelay(1);
1603                 WREG32_P(cp_rb_cntl[i], 0, ~RB_RPTR_WR_ENA);
1604         }
1605
1606         /* start the rings */
1607         cayman_cp_start(rdev);
1608         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
1609         rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1610         rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1611         /* this only test cp0 */
1612         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
1613         if (r) {
1614                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1615                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1616                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1617                 return r;
1618         }
1619
1620         return 0;
1621 }
1622
1623 /*
1624  * DMA
1625  * Starting with R600, the GPU has an asynchronous
1626  * DMA engine.  The programming model is very similar
1627  * to the 3D engine (ring buffer, IBs, etc.), but the
1628  * DMA controller has it's own packet format that is
1629  * different form the PM4 format used by the 3D engine.
1630  * It supports copying data, writing embedded data,
1631  * solid fills, and a number of other things.  It also
1632  * has support for tiling/detiling of buffers.
1633  * Cayman and newer support two asynchronous DMA engines.
1634  */
1635 /**
1636  * cayman_dma_ring_ib_execute - Schedule an IB on the DMA engine
1637  *
1638  * @rdev: radeon_device pointer
1639  * @ib: IB object to schedule
1640  *
1641  * Schedule an IB in the DMA ring (cayman-SI).
1642  */
1643 void cayman_dma_ring_ib_execute(struct radeon_device *rdev,
1644                                 struct radeon_ib *ib)
1645 {
1646         struct radeon_ring *ring = &rdev->ring[ib->ring];
1647
1648         if (rdev->wb.enabled) {
1649                 u32 next_rptr = ring->wptr + 4;
1650                 while ((next_rptr & 7) != 5)
1651                         next_rptr++;
1652                 next_rptr += 3;
1653                 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1));
1654                 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
1655                 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff);
1656                 radeon_ring_write(ring, next_rptr);
1657         }
1658
1659         /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring.
1660          * Pad as necessary with NOPs.
1661          */
1662         while ((ring->wptr & 7) != 5)
1663                 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
1664         radeon_ring_write(ring, DMA_IB_PACKET(DMA_PACKET_INDIRECT_BUFFER, ib->vm ? ib->vm->id : 0, 0));
1665         radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0));
1666         radeon_ring_write(ring, (ib->length_dw << 12) | (upper_32_bits(ib->gpu_addr) & 0xFF));
1667
1668 }
1669
1670 /**
1671  * cayman_dma_stop - stop the async dma engines
1672  *
1673  * @rdev: radeon_device pointer
1674  *
1675  * Stop the async dma engines (cayman-SI).
1676  */
1677 void cayman_dma_stop(struct radeon_device *rdev)
1678 {
1679         u32 rb_cntl;
1680
1681         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1682
1683         /* dma0 */
1684         rb_cntl = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
1685         rb_cntl &= ~DMA_RB_ENABLE;
1686         WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, rb_cntl);
1687
1688         /* dma1 */
1689         rb_cntl = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
1690         rb_cntl &= ~DMA_RB_ENABLE;
1691         WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, rb_cntl);
1692
1693         rdev->ring[R600_RING_TYPE_DMA_INDEX].ready = false;
1694         rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX].ready = false;
1695 }
1696
1697 /**
1698  * cayman_dma_resume - setup and start the async dma engines
1699  *
1700  * @rdev: radeon_device pointer
1701  *
1702  * Set up the DMA ring buffers and enable them. (cayman-SI).
1703  * Returns 0 for success, error for failure.
1704  */
1705 int cayman_dma_resume(struct radeon_device *rdev)
1706 {
1707         struct radeon_ring *ring;
1708         u32 rb_cntl, dma_cntl, ib_cntl;
1709         u32 rb_bufsz;
1710         u32 reg_offset, wb_offset;
1711         int i, r;
1712
1713         /* Reset dma */
1714         WREG32(SRBM_SOFT_RESET, SOFT_RESET_DMA | SOFT_RESET_DMA1);
1715         RREG32(SRBM_SOFT_RESET);
1716         udelay(50);
1717         WREG32(SRBM_SOFT_RESET, 0);
1718
1719         for (i = 0; i < 2; i++) {
1720                 if (i == 0) {
1721                         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
1722                         reg_offset = DMA0_REGISTER_OFFSET;
1723                         wb_offset = R600_WB_DMA_RPTR_OFFSET;
1724                 } else {
1725                         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
1726                         reg_offset = DMA1_REGISTER_OFFSET;
1727                         wb_offset = CAYMAN_WB_DMA1_RPTR_OFFSET;
1728                 }
1729
1730                 WREG32(DMA_SEM_INCOMPLETE_TIMER_CNTL + reg_offset, 0);
1731                 WREG32(DMA_SEM_WAIT_FAIL_TIMER_CNTL + reg_offset, 0);
1732
1733                 /* Set ring buffer size in dwords */
1734                 rb_bufsz = drm_order(ring->ring_size / 4);
1735                 rb_cntl = rb_bufsz << 1;
1736 #ifdef __BIG_ENDIAN
1737                 rb_cntl |= DMA_RB_SWAP_ENABLE | DMA_RPTR_WRITEBACK_SWAP_ENABLE;
1738 #endif
1739                 WREG32(DMA_RB_CNTL + reg_offset, rb_cntl);
1740
1741                 /* Initialize the ring buffer's read and write pointers */
1742                 WREG32(DMA_RB_RPTR + reg_offset, 0);
1743                 WREG32(DMA_RB_WPTR + reg_offset, 0);
1744
1745                 /* set the wb address whether it's enabled or not */
1746                 WREG32(DMA_RB_RPTR_ADDR_HI + reg_offset,
1747                        upper_32_bits(rdev->wb.gpu_addr + wb_offset) & 0xFF);
1748                 WREG32(DMA_RB_RPTR_ADDR_LO + reg_offset,
1749                        ((rdev->wb.gpu_addr + wb_offset) & 0xFFFFFFFC));
1750
1751                 if (rdev->wb.enabled)
1752                         rb_cntl |= DMA_RPTR_WRITEBACK_ENABLE;
1753
1754                 WREG32(DMA_RB_BASE + reg_offset, ring->gpu_addr >> 8);
1755
1756                 /* enable DMA IBs */
1757                 ib_cntl = DMA_IB_ENABLE | CMD_VMID_FORCE;
1758 #ifdef __BIG_ENDIAN
1759                 ib_cntl |= DMA_IB_SWAP_ENABLE;
1760 #endif
1761                 WREG32(DMA_IB_CNTL + reg_offset, ib_cntl);
1762
1763                 dma_cntl = RREG32(DMA_CNTL + reg_offset);
1764                 dma_cntl &= ~CTXEMPTY_INT_ENABLE;
1765                 WREG32(DMA_CNTL + reg_offset, dma_cntl);
1766
1767                 ring->wptr = 0;
1768                 WREG32(DMA_RB_WPTR + reg_offset, ring->wptr << 2);
1769
1770                 ring->rptr = RREG32(DMA_RB_RPTR + reg_offset) >> 2;
1771
1772                 WREG32(DMA_RB_CNTL + reg_offset, rb_cntl | DMA_RB_ENABLE);
1773
1774                 ring->ready = true;
1775
1776                 r = radeon_ring_test(rdev, ring->idx, ring);
1777                 if (r) {
1778                         ring->ready = false;
1779                         return r;
1780                 }
1781         }
1782
1783         radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
1784
1785         return 0;
1786 }
1787
1788 /**
1789  * cayman_dma_fini - tear down the async dma engines
1790  *
1791  * @rdev: radeon_device pointer
1792  *
1793  * Stop the async dma engines and free the rings (cayman-SI).
1794  */
1795 void cayman_dma_fini(struct radeon_device *rdev)
1796 {
1797         cayman_dma_stop(rdev);
1798         radeon_ring_fini(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX]);
1799         radeon_ring_fini(rdev, &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]);
1800 }
1801
1802 static u32 cayman_gpu_check_soft_reset(struct radeon_device *rdev)
1803 {
1804         u32 reset_mask = 0;
1805         u32 tmp;
1806
1807         /* GRBM_STATUS */
1808         tmp = RREG32(GRBM_STATUS);
1809         if (tmp & (PA_BUSY | SC_BUSY |
1810                    SH_BUSY | SX_BUSY |
1811                    TA_BUSY | VGT_BUSY |
1812                    DB_BUSY | CB_BUSY |
1813                    GDS_BUSY | SPI_BUSY |
1814                    IA_BUSY | IA_BUSY_NO_DMA))
1815                 reset_mask |= RADEON_RESET_GFX;
1816
1817         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
1818                    CP_BUSY | CP_COHERENCY_BUSY))
1819                 reset_mask |= RADEON_RESET_CP;
1820
1821         if (tmp & GRBM_EE_BUSY)
1822                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
1823
1824         /* DMA_STATUS_REG 0 */
1825         tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
1826         if (!(tmp & DMA_IDLE))
1827                 reset_mask |= RADEON_RESET_DMA;
1828
1829         /* DMA_STATUS_REG 1 */
1830         tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
1831         if (!(tmp & DMA_IDLE))
1832                 reset_mask |= RADEON_RESET_DMA1;
1833
1834         /* SRBM_STATUS2 */
1835         tmp = RREG32(SRBM_STATUS2);
1836         if (tmp & DMA_BUSY)
1837                 reset_mask |= RADEON_RESET_DMA;
1838
1839         if (tmp & DMA1_BUSY)
1840                 reset_mask |= RADEON_RESET_DMA1;
1841
1842         /* SRBM_STATUS */
1843         tmp = RREG32(SRBM_STATUS);
1844         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
1845                 reset_mask |= RADEON_RESET_RLC;
1846
1847         if (tmp & IH_BUSY)
1848                 reset_mask |= RADEON_RESET_IH;
1849
1850         if (tmp & SEM_BUSY)
1851                 reset_mask |= RADEON_RESET_SEM;
1852
1853         if (tmp & GRBM_RQ_PENDING)
1854                 reset_mask |= RADEON_RESET_GRBM;
1855
1856         if (tmp & VMC_BUSY)
1857                 reset_mask |= RADEON_RESET_VMC;
1858
1859         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
1860                    MCC_BUSY | MCD_BUSY))
1861                 reset_mask |= RADEON_RESET_MC;
1862
1863         if (evergreen_is_display_hung(rdev))
1864                 reset_mask |= RADEON_RESET_DISPLAY;
1865
1866         /* VM_L2_STATUS */
1867         tmp = RREG32(VM_L2_STATUS);
1868         if (tmp & L2_BUSY)
1869                 reset_mask |= RADEON_RESET_VMC;
1870
1871         /* Skip MC reset as it's mostly likely not hung, just busy */
1872         if (reset_mask & RADEON_RESET_MC) {
1873                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
1874                 reset_mask &= ~RADEON_RESET_MC;
1875         }
1876
1877         return reset_mask;
1878 }
1879
1880 static void cayman_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
1881 {
1882         struct evergreen_mc_save save;
1883         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
1884         u32 tmp;
1885
1886         if (reset_mask == 0)
1887                 return;
1888
1889         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
1890
1891         evergreen_print_gpu_status_regs(rdev);
1892         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_ADDR   0x%08X\n",
1893                  RREG32(0x14F8));
1894         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_STATUS 0x%08X\n",
1895                  RREG32(0x14D8));
1896         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
1897                  RREG32(0x14FC));
1898         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1899                  RREG32(0x14DC));
1900
1901         /* Disable CP parsing/prefetching */
1902         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
1903
1904         if (reset_mask & RADEON_RESET_DMA) {
1905                 /* dma0 */
1906                 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
1907                 tmp &= ~DMA_RB_ENABLE;
1908                 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
1909         }
1910
1911         if (reset_mask & RADEON_RESET_DMA1) {
1912                 /* dma1 */
1913                 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
1914                 tmp &= ~DMA_RB_ENABLE;
1915                 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
1916         }
1917
1918         udelay(50);
1919
1920         evergreen_mc_stop(rdev, &save);
1921         if (evergreen_mc_wait_for_idle(rdev)) {
1922                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1923         }
1924
1925         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
1926                 grbm_soft_reset = SOFT_RESET_CB |
1927                         SOFT_RESET_DB |
1928                         SOFT_RESET_GDS |
1929                         SOFT_RESET_PA |
1930                         SOFT_RESET_SC |
1931                         SOFT_RESET_SPI |
1932                         SOFT_RESET_SH |
1933                         SOFT_RESET_SX |
1934                         SOFT_RESET_TC |
1935                         SOFT_RESET_TA |
1936                         SOFT_RESET_VGT |
1937                         SOFT_RESET_IA;
1938         }
1939
1940         if (reset_mask & RADEON_RESET_CP) {
1941                 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
1942
1943                 srbm_soft_reset |= SOFT_RESET_GRBM;
1944         }
1945
1946         if (reset_mask & RADEON_RESET_DMA)
1947                 srbm_soft_reset |= SOFT_RESET_DMA;
1948
1949         if (reset_mask & RADEON_RESET_DMA1)
1950                 srbm_soft_reset |= SOFT_RESET_DMA1;
1951
1952         if (reset_mask & RADEON_RESET_DISPLAY)
1953                 srbm_soft_reset |= SOFT_RESET_DC;
1954
1955         if (reset_mask & RADEON_RESET_RLC)
1956                 srbm_soft_reset |= SOFT_RESET_RLC;
1957
1958         if (reset_mask & RADEON_RESET_SEM)
1959                 srbm_soft_reset |= SOFT_RESET_SEM;
1960
1961         if (reset_mask & RADEON_RESET_IH)
1962                 srbm_soft_reset |= SOFT_RESET_IH;
1963
1964         if (reset_mask & RADEON_RESET_GRBM)
1965                 srbm_soft_reset |= SOFT_RESET_GRBM;
1966
1967         if (reset_mask & RADEON_RESET_VMC)
1968                 srbm_soft_reset |= SOFT_RESET_VMC;
1969
1970         if (!(rdev->flags & RADEON_IS_IGP)) {
1971                 if (reset_mask & RADEON_RESET_MC)
1972                         srbm_soft_reset |= SOFT_RESET_MC;
1973         }
1974
1975         if (grbm_soft_reset) {
1976                 tmp = RREG32(GRBM_SOFT_RESET);
1977                 tmp |= grbm_soft_reset;
1978                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
1979                 WREG32(GRBM_SOFT_RESET, tmp);
1980                 tmp = RREG32(GRBM_SOFT_RESET);
1981
1982                 udelay(50);
1983
1984                 tmp &= ~grbm_soft_reset;
1985                 WREG32(GRBM_SOFT_RESET, tmp);
1986                 tmp = RREG32(GRBM_SOFT_RESET);
1987         }
1988
1989         if (srbm_soft_reset) {
1990                 tmp = RREG32(SRBM_SOFT_RESET);
1991                 tmp |= srbm_soft_reset;
1992                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1993                 WREG32(SRBM_SOFT_RESET, tmp);
1994                 tmp = RREG32(SRBM_SOFT_RESET);
1995
1996                 udelay(50);
1997
1998                 tmp &= ~srbm_soft_reset;
1999                 WREG32(SRBM_SOFT_RESET, tmp);
2000                 tmp = RREG32(SRBM_SOFT_RESET);
2001         }
2002
2003         /* Wait a little for things to settle down */
2004         udelay(50);
2005
2006         evergreen_mc_resume(rdev, &save);
2007         udelay(50);
2008
2009         evergreen_print_gpu_status_regs(rdev);
2010 }
2011
2012 int cayman_asic_reset(struct radeon_device *rdev)
2013 {
2014         u32 reset_mask;
2015
2016         reset_mask = cayman_gpu_check_soft_reset(rdev);
2017
2018         if (reset_mask)
2019                 r600_set_bios_scratch_engine_hung(rdev, true);
2020
2021         cayman_gpu_soft_reset(rdev, reset_mask);
2022
2023         reset_mask = cayman_gpu_check_soft_reset(rdev);
2024
2025         if (!reset_mask)
2026                 r600_set_bios_scratch_engine_hung(rdev, false);
2027
2028         return 0;
2029 }
2030
2031 /**
2032  * cayman_gfx_is_lockup - Check if the GFX engine is locked up
2033  *
2034  * @rdev: radeon_device pointer
2035  * @ring: radeon_ring structure holding ring information
2036  *
2037  * Check if the GFX engine is locked up.
2038  * Returns true if the engine appears to be locked up, false if not.
2039  */
2040 bool cayman_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
2041 {
2042         u32 reset_mask = cayman_gpu_check_soft_reset(rdev);
2043
2044         if (!(reset_mask & (RADEON_RESET_GFX |
2045                             RADEON_RESET_COMPUTE |
2046                             RADEON_RESET_CP))) {
2047                 radeon_ring_lockup_update(ring);
2048                 return false;
2049         }
2050         /* force CP activities */
2051         radeon_ring_force_activity(rdev, ring);
2052         return radeon_ring_test_lockup(rdev, ring);
2053 }
2054
2055 /**
2056  * cayman_dma_is_lockup - Check if the DMA engine is locked up
2057  *
2058  * @rdev: radeon_device pointer
2059  * @ring: radeon_ring structure holding ring information
2060  *
2061  * Check if the async DMA engine is locked up.
2062  * Returns true if the engine appears to be locked up, false if not.
2063  */
2064 bool cayman_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
2065 {
2066         u32 reset_mask = cayman_gpu_check_soft_reset(rdev);
2067         u32 mask;
2068
2069         if (ring->idx == R600_RING_TYPE_DMA_INDEX)
2070                 mask = RADEON_RESET_DMA;
2071         else
2072                 mask = RADEON_RESET_DMA1;
2073
2074         if (!(reset_mask & mask)) {
2075                 radeon_ring_lockup_update(ring);
2076                 return false;
2077         }
2078         /* force ring activities */
2079         radeon_ring_force_activity(rdev, ring);
2080         return radeon_ring_test_lockup(rdev, ring);
2081 }
2082
2083 static int cayman_startup(struct radeon_device *rdev)
2084 {
2085         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2086         int r;
2087
2088         /* enable pcie gen2 link */
2089         evergreen_pcie_gen2_enable(rdev);
2090         /* enable aspm */
2091         evergreen_program_aspm(rdev);
2092
2093         if (rdev->flags & RADEON_IS_IGP) {
2094                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
2095                         r = ni_init_microcode(rdev);
2096                         if (r) {
2097                                 DRM_ERROR("Failed to load firmware!\n");
2098                                 return r;
2099                         }
2100                 }
2101         } else {
2102                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
2103                         r = ni_init_microcode(rdev);
2104                         if (r) {
2105                                 DRM_ERROR("Failed to load firmware!\n");
2106                                 return r;
2107                         }
2108                 }
2109
2110                 r = ni_mc_load_microcode(rdev);
2111                 if (r) {
2112                         DRM_ERROR("Failed to load MC firmware!\n");
2113                         return r;
2114                 }
2115         }
2116
2117         r = r600_vram_scratch_init(rdev);
2118         if (r)
2119                 return r;
2120
2121         evergreen_mc_program(rdev);
2122         r = cayman_pcie_gart_enable(rdev);
2123         if (r)
2124                 return r;
2125         cayman_gpu_init(rdev);
2126
2127         r = evergreen_blit_init(rdev);
2128         if (r) {
2129                 r600_blit_fini(rdev);
2130                 rdev->asic->copy.copy = NULL;
2131                 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
2132         }
2133
2134         /* allocate rlc buffers */
2135         if (rdev->flags & RADEON_IS_IGP) {
2136                 rdev->rlc.reg_list = tn_rlc_save_restore_register_list;
2137                 rdev->rlc.reg_list_size = tn_rlc_save_restore_register_list_size;
2138                 rdev->rlc.cs_data = cayman_cs_data;
2139                 r = sumo_rlc_init(rdev);
2140                 if (r) {
2141                         DRM_ERROR("Failed to init rlc BOs!\n");
2142                         return r;
2143                 }
2144         }
2145
2146         /* allocate wb buffer */
2147         r = radeon_wb_init(rdev);
2148         if (r)
2149                 return r;
2150
2151         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
2152         if (r) {
2153                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2154                 return r;
2155         }
2156
2157         r = rv770_uvd_resume(rdev);
2158         if (!r) {
2159                 r = radeon_fence_driver_start_ring(rdev,
2160                                                    R600_RING_TYPE_UVD_INDEX);
2161                 if (r)
2162                         dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
2163         }
2164         if (r)
2165                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
2166
2167         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
2168         if (r) {
2169                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2170                 return r;
2171         }
2172
2173         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
2174         if (r) {
2175                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2176                 return r;
2177         }
2178
2179         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
2180         if (r) {
2181                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2182                 return r;
2183         }
2184
2185         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
2186         if (r) {
2187                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2188                 return r;
2189         }
2190
2191         /* Enable IRQ */
2192         if (!rdev->irq.installed) {
2193                 r = radeon_irq_kms_init(rdev);
2194                 if (r)
2195                         return r;
2196         }
2197
2198         r = r600_irq_init(rdev);
2199         if (r) {
2200                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
2201                 radeon_irq_kms_fini(rdev);
2202                 return r;
2203         }
2204         evergreen_irq_set(rdev);
2205
2206         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2207                              CP_RB0_RPTR, CP_RB0_WPTR,
2208                              0, 0xfffff, RADEON_CP_PACKET2);
2209         if (r)
2210                 return r;
2211
2212         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2213         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2214                              DMA_RB_RPTR + DMA0_REGISTER_OFFSET,
2215                              DMA_RB_WPTR + DMA0_REGISTER_OFFSET,
2216                              2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2217         if (r)
2218                 return r;
2219
2220         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2221         r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2222                              DMA_RB_RPTR + DMA1_REGISTER_OFFSET,
2223                              DMA_RB_WPTR + DMA1_REGISTER_OFFSET,
2224                              2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2225         if (r)
2226                 return r;
2227
2228         r = cayman_cp_load_microcode(rdev);
2229         if (r)
2230                 return r;
2231         r = cayman_cp_resume(rdev);
2232         if (r)
2233                 return r;
2234
2235         r = cayman_dma_resume(rdev);
2236         if (r)
2237                 return r;
2238
2239         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2240         if (ring->ring_size) {
2241                 r = radeon_ring_init(rdev, ring, ring->ring_size,
2242                                      R600_WB_UVD_RPTR_OFFSET,
2243                                      UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
2244                                      0, 0xfffff, RADEON_CP_PACKET2);
2245                 if (!r)
2246                         r = r600_uvd_init(rdev);
2247                 if (r)
2248                         DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
2249         }
2250
2251         r = radeon_ib_pool_init(rdev);
2252         if (r) {
2253                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
2254                 return r;
2255         }
2256
2257         r = radeon_vm_manager_init(rdev);
2258         if (r) {
2259                 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
2260                 return r;
2261         }
2262
2263         r = r600_audio_init(rdev);
2264         if (r)
2265                 return r;
2266
2267         return 0;
2268 }
2269
2270 int cayman_resume(struct radeon_device *rdev)
2271 {
2272         int r;
2273
2274         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
2275          * posting will perform necessary task to bring back GPU into good
2276          * shape.
2277          */
2278         /* post card */
2279         atom_asic_init(rdev->mode_info.atom_context);
2280
2281         /* init golden registers */
2282         ni_init_golden_registers(rdev);
2283
2284         rdev->accel_working = true;
2285         r = cayman_startup(rdev);
2286         if (r) {
2287                 DRM_ERROR("cayman startup failed on resume\n");
2288                 rdev->accel_working = false;
2289                 return r;
2290         }
2291         return r;
2292 }
2293
2294 int cayman_suspend(struct radeon_device *rdev)
2295 {
2296         r600_audio_fini(rdev);
2297         radeon_vm_manager_fini(rdev);
2298         cayman_cp_enable(rdev, false);
2299         cayman_dma_stop(rdev);
2300         r600_uvd_rbc_stop(rdev);
2301         radeon_uvd_suspend(rdev);
2302         evergreen_irq_suspend(rdev);
2303         radeon_wb_disable(rdev);
2304         cayman_pcie_gart_disable(rdev);
2305         return 0;
2306 }
2307
2308 /* Plan is to move initialization in that function and use
2309  * helper function so that radeon_device_init pretty much
2310  * do nothing more than calling asic specific function. This
2311  * should also allow to remove a bunch of callback function
2312  * like vram_info.
2313  */
2314 int cayman_init(struct radeon_device *rdev)
2315 {
2316         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2317         int r;
2318
2319         /* Read BIOS */
2320         if (!radeon_get_bios(rdev)) {
2321                 if (ASIC_IS_AVIVO(rdev))
2322                         return -EINVAL;
2323         }
2324         /* Must be an ATOMBIOS */
2325         if (!rdev->is_atom_bios) {
2326                 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
2327                 return -EINVAL;
2328         }
2329         r = radeon_atombios_init(rdev);
2330         if (r)
2331                 return r;
2332
2333         /* Post card if necessary */
2334         if (!radeon_card_posted(rdev)) {
2335                 if (!rdev->bios) {
2336                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
2337                         return -EINVAL;
2338                 }
2339                 DRM_INFO("GPU not posted. posting now...\n");
2340                 atom_asic_init(rdev->mode_info.atom_context);
2341         }
2342         /* init golden registers */
2343         ni_init_golden_registers(rdev);
2344         /* Initialize scratch registers */
2345         r600_scratch_init(rdev);
2346         /* Initialize surface registers */
2347         radeon_surface_init(rdev);
2348         /* Initialize clocks */
2349         radeon_get_clock_info(rdev->ddev);
2350         /* Fence driver */
2351         r = radeon_fence_driver_init(rdev);
2352         if (r)
2353                 return r;
2354         /* initialize memory controller */
2355         r = evergreen_mc_init(rdev);
2356         if (r)
2357                 return r;
2358         /* Memory manager */
2359         r = radeon_bo_init(rdev);
2360         if (r)
2361                 return r;
2362
2363         ring->ring_obj = NULL;
2364         r600_ring_init(rdev, ring, 1024 * 1024);
2365
2366         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2367         ring->ring_obj = NULL;
2368         r600_ring_init(rdev, ring, 64 * 1024);
2369
2370         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2371         ring->ring_obj = NULL;
2372         r600_ring_init(rdev, ring, 64 * 1024);
2373
2374         r = radeon_uvd_init(rdev);
2375         if (!r) {
2376                 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2377                 ring->ring_obj = NULL;
2378                 r600_ring_init(rdev, ring, 4096);
2379         }
2380
2381         rdev->ih.ring_obj = NULL;
2382         r600_ih_ring_init(rdev, 64 * 1024);
2383
2384         r = r600_pcie_gart_init(rdev);
2385         if (r)
2386                 return r;
2387
2388         rdev->accel_working = true;
2389         r = cayman_startup(rdev);
2390         if (r) {
2391                 dev_err(rdev->dev, "disabling GPU acceleration\n");
2392                 cayman_cp_fini(rdev);
2393                 cayman_dma_fini(rdev);
2394                 r600_irq_fini(rdev);
2395                 if (rdev->flags & RADEON_IS_IGP)
2396                         sumo_rlc_fini(rdev);
2397                 radeon_wb_fini(rdev);
2398                 radeon_ib_pool_fini(rdev);
2399                 radeon_vm_manager_fini(rdev);
2400                 radeon_irq_kms_fini(rdev);
2401                 cayman_pcie_gart_fini(rdev);
2402                 rdev->accel_working = false;
2403         }
2404
2405         /* Don't start up if the MC ucode is missing.
2406          * The default clocks and voltages before the MC ucode
2407          * is loaded are not suffient for advanced operations.
2408          *
2409          * We can skip this check for TN, because there is no MC
2410          * ucode.
2411          */
2412         if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
2413                 DRM_ERROR("radeon: MC ucode required for NI+.\n");
2414                 return -EINVAL;
2415         }
2416
2417         return 0;
2418 }
2419
2420 void cayman_fini(struct radeon_device *rdev)
2421 {
2422         r600_blit_fini(rdev);
2423         cayman_cp_fini(rdev);
2424         cayman_dma_fini(rdev);
2425         r600_irq_fini(rdev);
2426         if (rdev->flags & RADEON_IS_IGP)
2427                 sumo_rlc_fini(rdev);
2428         radeon_wb_fini(rdev);
2429         radeon_vm_manager_fini(rdev);
2430         radeon_ib_pool_fini(rdev);
2431         radeon_irq_kms_fini(rdev);
2432         radeon_uvd_fini(rdev);
2433         cayman_pcie_gart_fini(rdev);
2434         r600_vram_scratch_fini(rdev);
2435         radeon_gem_fini(rdev);
2436         radeon_fence_driver_fini(rdev);
2437         radeon_bo_fini(rdev);
2438         radeon_atombios_fini(rdev);
2439         kfree(rdev->bios);
2440         rdev->bios = NULL;
2441 }
2442
2443 /*
2444  * vm
2445  */
2446 int cayman_vm_init(struct radeon_device *rdev)
2447 {
2448         /* number of VMs */
2449         rdev->vm_manager.nvm = 8;
2450         /* base offset of vram pages */
2451         if (rdev->flags & RADEON_IS_IGP) {
2452                 u64 tmp = RREG32(FUS_MC_VM_FB_OFFSET);
2453                 tmp <<= 22;
2454                 rdev->vm_manager.vram_base_offset = tmp;
2455         } else
2456                 rdev->vm_manager.vram_base_offset = 0;
2457         return 0;
2458 }
2459
2460 void cayman_vm_fini(struct radeon_device *rdev)
2461 {
2462 }
2463
2464 #define R600_ENTRY_VALID   (1 << 0)
2465 #define R600_PTE_SYSTEM    (1 << 1)
2466 #define R600_PTE_SNOOPED   (1 << 2)
2467 #define R600_PTE_READABLE  (1 << 5)
2468 #define R600_PTE_WRITEABLE (1 << 6)
2469
2470 uint32_t cayman_vm_page_flags(struct radeon_device *rdev, uint32_t flags)
2471 {
2472         uint32_t r600_flags = 0;
2473         r600_flags |= (flags & RADEON_VM_PAGE_VALID) ? R600_ENTRY_VALID : 0;
2474         r600_flags |= (flags & RADEON_VM_PAGE_READABLE) ? R600_PTE_READABLE : 0;
2475         r600_flags |= (flags & RADEON_VM_PAGE_WRITEABLE) ? R600_PTE_WRITEABLE : 0;
2476         if (flags & RADEON_VM_PAGE_SYSTEM) {
2477                 r600_flags |= R600_PTE_SYSTEM;
2478                 r600_flags |= (flags & RADEON_VM_PAGE_SNOOPED) ? R600_PTE_SNOOPED : 0;
2479         }
2480         return r600_flags;
2481 }
2482
2483 /**
2484  * cayman_vm_set_page - update the page tables using the CP
2485  *
2486  * @rdev: radeon_device pointer
2487  * @ib: indirect buffer to fill with commands
2488  * @pe: addr of the page entry
2489  * @addr: dst addr to write into pe
2490  * @count: number of page entries to update
2491  * @incr: increase next addr by incr bytes
2492  * @flags: access flags
2493  *
2494  * Update the page tables using the CP (cayman/TN).
2495  */
2496 void cayman_vm_set_page(struct radeon_device *rdev,
2497                         struct radeon_ib *ib,
2498                         uint64_t pe,
2499                         uint64_t addr, unsigned count,
2500                         uint32_t incr, uint32_t flags)
2501 {
2502         uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
2503         uint64_t value;
2504         unsigned ndw;
2505
2506         if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) {
2507                 while (count) {
2508                         ndw = 1 + count * 2;
2509                         if (ndw > 0x3FFF)
2510                                 ndw = 0x3FFF;
2511
2512                         ib->ptr[ib->length_dw++] = PACKET3(PACKET3_ME_WRITE, ndw);
2513                         ib->ptr[ib->length_dw++] = pe;
2514                         ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
2515                         for (; ndw > 1; ndw -= 2, --count, pe += 8) {
2516                                 if (flags & RADEON_VM_PAGE_SYSTEM) {
2517                                         value = radeon_vm_map_gart(rdev, addr);
2518                                         value &= 0xFFFFFFFFFFFFF000ULL;
2519                                 } else if (flags & RADEON_VM_PAGE_VALID) {
2520                                         value = addr;
2521                                 } else {
2522                                         value = 0;
2523                                 }
2524                                 addr += incr;
2525                                 value |= r600_flags;
2526                                 ib->ptr[ib->length_dw++] = value;
2527                                 ib->ptr[ib->length_dw++] = upper_32_bits(value);
2528                         }
2529                 }
2530         } else {
2531                 if ((flags & RADEON_VM_PAGE_SYSTEM) ||
2532                     (count == 1)) {
2533                         while (count) {
2534                                 ndw = count * 2;
2535                                 if (ndw > 0xFFFFE)
2536                                         ndw = 0xFFFFE;
2537
2538                                 /* for non-physically contiguous pages (system) */
2539                                 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, ndw);
2540                                 ib->ptr[ib->length_dw++] = pe;
2541                                 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
2542                                 for (; ndw > 0; ndw -= 2, --count, pe += 8) {
2543                                         if (flags & RADEON_VM_PAGE_SYSTEM) {
2544                                                 value = radeon_vm_map_gart(rdev, addr);
2545                                                 value &= 0xFFFFFFFFFFFFF000ULL;
2546                                         } else if (flags & RADEON_VM_PAGE_VALID) {
2547                                                 value = addr;
2548                                         } else {
2549                                                 value = 0;
2550                                         }
2551                                         addr += incr;
2552                                         value |= r600_flags;
2553                                         ib->ptr[ib->length_dw++] = value;
2554                                         ib->ptr[ib->length_dw++] = upper_32_bits(value);
2555                                 }
2556                         }
2557                         while (ib->length_dw & 0x7)
2558                                 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0);
2559                 } else {
2560                         while (count) {
2561                                 ndw = count * 2;
2562                                 if (ndw > 0xFFFFE)
2563                                         ndw = 0xFFFFE;
2564
2565                                 if (flags & RADEON_VM_PAGE_VALID)
2566                                         value = addr;
2567                                 else
2568                                         value = 0;
2569                                 /* for physically contiguous pages (vram) */
2570                                 ib->ptr[ib->length_dw++] = DMA_PTE_PDE_PACKET(ndw);
2571                                 ib->ptr[ib->length_dw++] = pe; /* dst addr */
2572                                 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
2573                                 ib->ptr[ib->length_dw++] = r600_flags; /* mask */
2574                                 ib->ptr[ib->length_dw++] = 0;
2575                                 ib->ptr[ib->length_dw++] = value; /* value */
2576                                 ib->ptr[ib->length_dw++] = upper_32_bits(value);
2577                                 ib->ptr[ib->length_dw++] = incr; /* increment size */
2578                                 ib->ptr[ib->length_dw++] = 0;
2579                                 pe += ndw * 4;
2580                                 addr += (ndw / 2) * incr;
2581                                 count -= ndw / 2;
2582                         }
2583                 }
2584                 while (ib->length_dw & 0x7)
2585                         ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0);
2586         }
2587 }
2588
2589 /**
2590  * cayman_vm_flush - vm flush using the CP
2591  *
2592  * @rdev: radeon_device pointer
2593  *
2594  * Update the page table base and flush the VM TLB
2595  * using the CP (cayman-si).
2596  */
2597 void cayman_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
2598 {
2599         struct radeon_ring *ring = &rdev->ring[ridx];
2600
2601         if (vm == NULL)
2602                 return;
2603
2604         radeon_ring_write(ring, PACKET0(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2), 0));
2605         radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
2606
2607         /* flush hdp cache */
2608         radeon_ring_write(ring, PACKET0(HDP_MEM_COHERENCY_FLUSH_CNTL, 0));
2609         radeon_ring_write(ring, 0x1);
2610
2611         /* bits 0-7 are the VM contexts0-7 */
2612         radeon_ring_write(ring, PACKET0(VM_INVALIDATE_REQUEST, 0));
2613         radeon_ring_write(ring, 1 << vm->id);
2614
2615         /* sync PFP to ME, otherwise we might get invalid PFP reads */
2616         radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
2617         radeon_ring_write(ring, 0x0);
2618 }
2619
2620 void cayman_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
2621 {
2622         struct radeon_ring *ring = &rdev->ring[ridx];
2623
2624         if (vm == NULL)
2625                 return;
2626
2627         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0));
2628         radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2));
2629         radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
2630
2631         /* flush hdp cache */
2632         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0));
2633         radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
2634         radeon_ring_write(ring, 1);
2635
2636         /* bits 0-7 are the VM contexts0-7 */
2637         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0));
2638         radeon_ring_write(ring, (0xf << 16) | (VM_INVALIDATE_REQUEST >> 2));
2639         radeon_ring_write(ring, 1 << vm->id);
2640 }
2641