drivers/gpu/drm/radeon/r600.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <linux/slab.h>
  29 #include <linux/seq_file.h>
  30 #include <linux/firmware.h>
  31 #include <linux/platform_device.h>
  32 #include <linux/module.h>
  33 #include <drm/drmP.h>
  34 #include <drm/radeon_drm.h>
  35 #include "radeon.h"
  36 #include "radeon_asic.h"
  37 #include "radeon_mode.h"
  38 #include "r600d.h"
  39 #include "atom.h"
  40 #include "avivod.h"
  41 #include "radeon_ucode.h"
  42
  43 /* Firmware Names */
  44 MODULE_FIRMWARE("radeon/R600_pfp.bin");
  45 MODULE_FIRMWARE("radeon/R600_me.bin");
  46 MODULE_FIRMWARE("radeon/RV610_pfp.bin");
  47 MODULE_FIRMWARE("radeon/RV610_me.bin");
  48 MODULE_FIRMWARE("radeon/RV630_pfp.bin");
  49 MODULE_FIRMWARE("radeon/RV630_me.bin");
  50 MODULE_FIRMWARE("radeon/RV620_pfp.bin");
  51 MODULE_FIRMWARE("radeon/RV620_me.bin");
  52 MODULE_FIRMWARE("radeon/RV635_pfp.bin");
  53 MODULE_FIRMWARE("radeon/RV635_me.bin");
  54 MODULE_FIRMWARE("radeon/RV670_pfp.bin");
  55 MODULE_FIRMWARE("radeon/RV670_me.bin");
  56 MODULE_FIRMWARE("radeon/RS780_pfp.bin");
  57 MODULE_FIRMWARE("radeon/RS780_me.bin");
  58 MODULE_FIRMWARE("radeon/RV770_pfp.bin");
  59 MODULE_FIRMWARE("radeon/RV770_me.bin");
  60 MODULE_FIRMWARE("radeon/RV730_pfp.bin");
  61 MODULE_FIRMWARE("radeon/RV730_me.bin");
  62 MODULE_FIRMWARE("radeon/RV710_pfp.bin");
  63 MODULE_FIRMWARE("radeon/RV710_me.bin");
  64 MODULE_FIRMWARE("radeon/R600_rlc.bin");
  65 MODULE_FIRMWARE("radeon/R700_rlc.bin");
  66 MODULE_FIRMWARE("radeon/CEDAR_pfp.bin");
  67 MODULE_FIRMWARE("radeon/CEDAR_me.bin");
  68 MODULE_FIRMWARE("radeon/CEDAR_rlc.bin");
  69 MODULE_FIRMWARE("radeon/REDWOOD_pfp.bin");
  70 MODULE_FIRMWARE("radeon/REDWOOD_me.bin");
  71 MODULE_FIRMWARE("radeon/REDWOOD_rlc.bin");
  72 MODULE_FIRMWARE("radeon/JUNIPER_pfp.bin");
  73 MODULE_FIRMWARE("radeon/JUNIPER_me.bin");
  74 MODULE_FIRMWARE("radeon/JUNIPER_rlc.bin");
  75 MODULE_FIRMWARE("radeon/CYPRESS_pfp.bin");
  76 MODULE_FIRMWARE("radeon/CYPRESS_me.bin");
  77 MODULE_FIRMWARE("radeon/CYPRESS_rlc.bin");
  78 MODULE_FIRMWARE("radeon/PALM_pfp.bin");
  79 MODULE_FIRMWARE("radeon/PALM_me.bin");
  80 MODULE_FIRMWARE("radeon/SUMO_rlc.bin");
  81 MODULE_FIRMWARE("radeon/SUMO_pfp.bin");
  82 MODULE_FIRMWARE("radeon/SUMO_me.bin");
  83 MODULE_FIRMWARE("radeon/SUMO2_pfp.bin");
  84 MODULE_FIRMWARE("radeon/SUMO2_me.bin");
  85
  86 static const u32 crtc_offsets[2] =
  87 {
  88         0,
  89         AVIVO_D2CRTC_H_TOTAL - AVIVO_D1CRTC_H_TOTAL
  90 };
  91
  92 int r600_debugfs_mc_info_init(struct radeon_device *rdev);
  93
  94 /* r600,rv610,rv630,rv620,rv635,rv670 */
  95 int r600_mc_wait_for_idle(struct radeon_device *rdev);
  96 static void r600_gpu_init(struct radeon_device *rdev);
  97 void r600_fini(struct radeon_device *rdev);
  98 void r600_irq_disable(struct radeon_device *rdev);
  99 static void r600_pcie_gen2_enable(struct radeon_device *rdev);
 100 extern int evergreen_rlc_resume(struct radeon_device *rdev);
 101
 102 /**
 103  * r600_get_xclk - get the xclk
 104  *
 105  * @rdev: radeon_device pointer
 106  *
 107  * Returns the reference clock used by the gfx engine
 108  * (r6xx, IGPs, APUs).
 109  */
 110 u32 r600_get_xclk(struct radeon_device *rdev)
 111 {
 112         return rdev->clock.spll.reference_freq;
 113 }
 114
 115 /* get temperature in millidegrees */
 116 int rv6xx_get_temp(struct radeon_device *rdev)
 117 {
 118         u32 temp = (RREG32(CG_THERMAL_STATUS) & ASIC_T_MASK) >>
 119                 ASIC_T_SHIFT;
 120         int actual_temp = temp & 0xff;
 121
 122         if (temp & 0x100)
 123                 actual_temp -= 256;
 124
 125         return actual_temp * 1000;
 126 }
 127
 128 void r600_pm_get_dynpm_state(struct radeon_device *rdev)
 129 {
 130         int i;
 131
 132         rdev->pm.dynpm_can_upclock = true;
 133         rdev->pm.dynpm_can_downclock = true;
 134
 135         /* power state array is low to high, default is first */
 136         if ((rdev->flags & RADEON_IS_IGP) || (rdev->family == CHIP_R600)) {
 137                 int min_power_state_index = 0;
 138
 139                 if (rdev->pm.num_power_states > 2)
 140                         min_power_state_index = 1;
 141
 142                 switch (rdev->pm.dynpm_planned_action) {
 143                 case DYNPM_ACTION_MINIMUM:
 144                         rdev->pm.requested_power_state_index = min_power_state_index;
 145                         rdev->pm.requested_clock_mode_index = 0;
 146                         rdev->pm.dynpm_can_downclock = false;
 147                         break;
 148                 case DYNPM_ACTION_DOWNCLOCK:
 149                         if (rdev->pm.current_power_state_index == min_power_state_index) {
 150                                 rdev->pm.requested_power_state_index = rdev->pm.current_power_state_index;
 151                                 rdev->pm.dynpm_can_downclock = false;
 152                         } else {
 153                                 if (rdev->pm.active_crtc_count > 1) {
 154                                         for (i = 0; i < rdev->pm.num_power_states; i++) {
 155                                                 if (rdev->pm.power_state[i].flags & RADEON_PM_STATE_SINGLE_DISPLAY_ONLY)
 156                                                         continue;
 157                                                 else if (i >= rdev->pm.current_power_state_index) {
 158                                                         rdev->pm.requested_power_state_index =
 159                                                                 rdev->pm.current_power_state_index;
 160                                                         break;
 161                                                 } else {
 162                                                         rdev->pm.requested_power_state_index = i;
 163                                                         break;
 164                                                 }
 165                                         }
 166                                 } else {
 167                                         if (rdev->pm.current_power_state_index == 0)
 168                                                 rdev->pm.requested_power_state_index =
 169                                                         rdev->pm.num_power_states - 1;
 170                                         else
 171                                                 rdev->pm.requested_power_state_index =
 172                                                         rdev->pm.current_power_state_index - 1;
 173                                 }
 174                         }
 175                         rdev->pm.requested_clock_mode_index = 0;
 176                         /* don't use the power state if crtcs are active and no display flag is set */
 177                         if ((rdev->pm.active_crtc_count > 0) &&
 178                             (rdev->pm.power_state[rdev->pm.requested_power_state_index].
 179                              clock_info[rdev->pm.requested_clock_mode_index].flags &
 180                              RADEON_PM_MODE_NO_DISPLAY)) {
 181                                 rdev->pm.requested_power_state_index++;
 182                         }
 183                         break;
 184                 case DYNPM_ACTION_UPCLOCK:
 185                         if (rdev->pm.current_power_state_index == (rdev->pm.num_power_states - 1)) {
 186                                 rdev->pm.requested_power_state_index = rdev->pm.current_power_state_index;
 187                                 rdev->pm.dynpm_can_upclock = false;
 188                         } else {
 189                                 if (rdev->pm.active_crtc_count > 1) {
 190                                         for (i = (rdev->pm.num_power_states - 1); i >= 0; i--) {
 191                                                 if (rdev->pm.power_state[i].flags & RADEON_PM_STATE_SINGLE_DISPLAY_ONLY)
 192                                                         continue;
 193                                                 else if (i <= rdev->pm.current_power_state_index) {
 194                                                         rdev->pm.requested_power_state_index =
 195                                                                 rdev->pm.current_power_state_index;
 196                                                         break;
 197                                                 } else {
 198                                                         rdev->pm.requested_power_state_index = i;
 199                                                         break;
 200                                                 }
 201                                         }
 202                                 } else
 203                                         rdev->pm.requested_power_state_index =
 204                                                 rdev->pm.current_power_state_index + 1;
 205                         }
 206                         rdev->pm.requested_clock_mode_index = 0;
 207                         break;
 208                 case DYNPM_ACTION_DEFAULT:
 209                         rdev->pm.requested_power_state_index = rdev->pm.default_power_state_index;
 210                         rdev->pm.requested_clock_mode_index = 0;
 211                         rdev->pm.dynpm_can_upclock = false;
 212                         break;
 213                 case DYNPM_ACTION_NONE:
 214                 default:
 215                         DRM_ERROR("Requested mode for not defined action\n");
 216                         return;
 217                 }
 218         } else {
 219                 /* XXX select a power state based on AC/DC, single/dualhead, etc. */
 220                 /* for now just select the first power state and switch between clock modes */
 221                 /* power state array is low to high, default is first (0) */
 222                 if (rdev->pm.active_crtc_count > 1) {
 223                         rdev->pm.requested_power_state_index = -1;
 224                         /* start at 1 as we don't want the default mode */
 225                         for (i = 1; i < rdev->pm.num_power_states; i++) {
 226                                 if (rdev->pm.power_state[i].flags & RADEON_PM_STATE_SINGLE_DISPLAY_ONLY)
 227                                         continue;
 228                                 else if ((rdev->pm.power_state[i].type == POWER_STATE_TYPE_PERFORMANCE) ||
 229                                          (rdev->pm.power_state[i].type == POWER_STATE_TYPE_BATTERY)) {
 230                                         rdev->pm.requested_power_state_index = i;
 231                                         break;
 232                                 }
 233                         }
 234                         /* if nothing selected, grab the default state. */
 235                         if (rdev->pm.requested_power_state_index == -1)
 236                                 rdev->pm.requested_power_state_index = 0;
 237                 } else
 238                         rdev->pm.requested_power_state_index = 1;
 239
 240                 switch (rdev->pm.dynpm_planned_action) {
 241                 case DYNPM_ACTION_MINIMUM:
 242                         rdev->pm.requested_clock_mode_index = 0;
 243                         rdev->pm.dynpm_can_downclock = false;
 244                         break;
 245                 case DYNPM_ACTION_DOWNCLOCK:
 246                         if (rdev->pm.requested_power_state_index == rdev->pm.current_power_state_index) {
 247                                 if (rdev->pm.current_clock_mode_index == 0) {
 248                                         rdev->pm.requested_clock_mode_index = 0;
 249                                         rdev->pm.dynpm_can_downclock = false;
 250                                 } else
 251                                         rdev->pm.requested_clock_mode_index =
 252                                                 rdev->pm.current_clock_mode_index - 1;
 253                         } else {
 254                                 rdev->pm.requested_clock_mode_index = 0;
 255                                 rdev->pm.dynpm_can_downclock = false;
 256                         }
 257                         /* don't use the power state if crtcs are active and no display flag is set */
 258                         if ((rdev->pm.active_crtc_count > 0) &&
 259                             (rdev->pm.power_state[rdev->pm.requested_power_state_index].
 260                              clock_info[rdev->pm.requested_clock_mode_index].flags &
 261                              RADEON_PM_MODE_NO_DISPLAY)) {
 262                                 rdev->pm.requested_clock_mode_index++;
 263                         }
 264                         break;
 265                 case DYNPM_ACTION_UPCLOCK:
 266                         if (rdev->pm.requested_power_state_index == rdev->pm.current_power_state_index) {
 267                                 if (rdev->pm.current_clock_mode_index ==
 268                                     (rdev->pm.power_state[rdev->pm.requested_power_state_index].num_clock_modes - 1)) {
 269                                         rdev->pm.requested_clock_mode_index = rdev->pm.current_clock_mode_index;
 270                                         rdev->pm.dynpm_can_upclock = false;
 271                                 } else
 272                                         rdev->pm.requested_clock_mode_index =
 273                                                 rdev->pm.current_clock_mode_index + 1;
 274                         } else {
 275                                 rdev->pm.requested_clock_mode_index =
 276                                         rdev->pm.power_state[rdev->pm.requested_power_state_index].num_clock_modes - 1;
 277                                 rdev->pm.dynpm_can_upclock = false;
 278                         }
 279                         break;
 280                 case DYNPM_ACTION_DEFAULT:
 281                         rdev->pm.requested_power_state_index = rdev->pm.default_power_state_index;
 282                         rdev->pm.requested_clock_mode_index = 0;
 283                         rdev->pm.dynpm_can_upclock = false;
 284                         break;
 285                 case DYNPM_ACTION_NONE:
 286                 default:
 287                         DRM_ERROR("Requested mode for not defined action\n");
 288                         return;
 289                 }
 290         }
 291
 292         DRM_DEBUG_DRIVER("Requested: e: %d m: %d p: %d\n",
 293                   rdev->pm.power_state[rdev->pm.requested_power_state_index].
 294                   clock_info[rdev->pm.requested_clock_mode_index].sclk,
 295                   rdev->pm.power_state[rdev->pm.requested_power_state_index].
 296                   clock_info[rdev->pm.requested_clock_mode_index].mclk,
 297                   rdev->pm.power_state[rdev->pm.requested_power_state_index].
 298                   pcie_lanes);
 299 }
 300
 301 void rs780_pm_init_profile(struct radeon_device *rdev)
 302 {
 303         if (rdev->pm.num_power_states == 2) {
 304                 /* default */
 305                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 306                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 307                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
 308                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
 309                 /* low sh */
 310                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 0;
 311                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 0;
 312                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
 313                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
 314                 /* mid sh */
 315                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 0;
 316                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 0;
 317                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
 318                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
 319                 /* high sh */
 320                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 0;
 321                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 1;
 322                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
 323                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
 324                 /* low mh */
 325                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 0;
 326                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 0;
 327                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
 328                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
 329                 /* mid mh */
 330                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 0;
 331                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 0;
 332                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
 333                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
 334                 /* high mh */
 335                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 0;
 336                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 1;
 337                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
 338                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
 339         } else if (rdev->pm.num_power_states == 3) {
 340                 /* default */
 341                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 342                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 343                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
 344                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
 345                 /* low sh */
 346                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 1;
 347                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 1;
 348                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
 349                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
 350                 /* mid sh */
 351                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 1;
 352                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 1;
 353                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
 354                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
 355                 /* high sh */
 356                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 1;
 357                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 2;
 358                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
 359                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
 360                 /* low mh */
 361                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 1;
 362                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 1;
 363                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
 364                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
 365                 /* mid mh */
 366                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 1;
 367                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 1;
 368                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
 369                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
 370                 /* high mh */
 371                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 1;
 372                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 2;
 373                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
 374                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
 375         } else {
 376                 /* default */
 377                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 378                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 379                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
 380                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
 381                 /* low sh */
 382                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 2;
 383                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 2;
 384                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
 385                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
 386                 /* mid sh */
 387                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 2;
 388                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 2;
 389                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
 390                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
 391                 /* high sh */
 392                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 2;
 393                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 3;
 394                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
 395                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
 396                 /* low mh */
 397                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 2;
 398                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 0;
 399                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
 400                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
 401                 /* mid mh */
 402                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 2;
 403                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 0;
 404                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
 405                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
 406                 /* high mh */
 407                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 2;
 408                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 3;
 409                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
 410                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
 411         }
 412 }
 413
 414 void r600_pm_init_profile(struct radeon_device *rdev)
 415 {
 416         int idx;
 417
 418         if (rdev->family == CHIP_R600) {
 419                 /* XXX */
 420                 /* default */
 421                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 422                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 423                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
 424                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
 425                 /* low sh */
 426                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 427                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 428                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
 429                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
 430                 /* mid sh */
 431                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 432                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 433                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
 434                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
 435                 /* high sh */
 436                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 437                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 438                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
 439                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
 440                 /* low mh */
 441                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 442                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 443                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
 444                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
 445                 /* mid mh */
 446                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 447                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 448                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
 449                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
 450                 /* high mh */
 451                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 452                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 453                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
 454                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
 455         } else {
 456                 if (rdev->pm.num_power_states < 4) {
 457                         /* default */
 458                         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 459                         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 460                         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
 461                         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
 462                         /* low sh */
 463                         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 1;
 464                         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 1;
 465                         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
 466                         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
 467                         /* mid sh */
 468                         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 1;
 469                         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 1;
 470                         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
 471                         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
 472                         /* high sh */
 473                         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 1;
 474                         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 1;
 475                         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
 476                         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
 477                         /* low mh */
 478                         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 2;
 479                         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 2;
 480                         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
 481                         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
 482                         /* low mh */
 483                         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 2;
 484                         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 2;
 485                         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
 486                         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
 487                         /* high mh */
 488                         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 2;
 489                         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 2;
 490                         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
 491                         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
 492                 } else {
 493                         /* default */
 494                         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 495                         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 496                         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
 497                         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
 498                         /* low sh */
 499                         if (rdev->flags & RADEON_IS_MOBILITY)
 500                                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
 501                         else
 502                                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
 503                         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
 504                         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
 505                         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
 506                         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
 507                         /* mid sh */
 508                         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
 509                         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
 510                         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
 511                         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
 512                         /* high sh */
 513                         idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
 514                         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
 515                         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
 516                         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
 517                         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
 518                         /* low mh */
 519                         if (rdev->flags & RADEON_IS_MOBILITY)
 520                                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 1);
 521                         else
 522                                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 1);
 523                         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
 524                         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
 525                         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
 526                         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
 527                         /* mid mh */
 528                         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
 529                         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
 530                         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
 531                         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
 532                         /* high mh */
 533                         idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 1);
 534                         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
 535                         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
 536                         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
 537                         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
 538                 }
 539         }
 540 }
 541
 542 void r600_pm_misc(struct radeon_device *rdev)
 543 {
 544         int req_ps_idx = rdev->pm.requested_power_state_index;
 545         int req_cm_idx = rdev->pm.requested_clock_mode_index;
 546         struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
 547         struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
 548
 549         if ((voltage->type == VOLTAGE_SW) && voltage->voltage) {
 550                 /* 0xff01 is a flag rather then an actual voltage */
 551                 if (voltage->voltage == 0xff01)
 552                         return;
 553                 if (voltage->voltage != rdev->pm.current_vddc) {
 554                         radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
 555                         rdev->pm.current_vddc = voltage->voltage;
 556                         DRM_DEBUG_DRIVER("Setting: v: %d\n", voltage->voltage);
 557                 }
 558         }
 559 }
 560
 561 bool r600_gui_idle(struct radeon_device *rdev)
 562 {
 563         if (RREG32(GRBM_STATUS) & GUI_ACTIVE)
 564                 return false;
 565         else
 566                 return true;
 567 }
 568
 569 /* hpd for digital panel detect/disconnect */
 570 bool r600_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
 571 {
 572         bool connected = false;
 573
 574         if (ASIC_IS_DCE3(rdev)) {
 575                 switch (hpd) {
 576                 case RADEON_HPD_1:
 577                         if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
 578                                 connected = true;
 579                         break;
 580                 case RADEON_HPD_2:
 581                         if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
 582                                 connected = true;
 583                         break;
 584                 case RADEON_HPD_3:
 585                         if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
 586                                 connected = true;
 587                         break;
 588                 case RADEON_HPD_4:
 589                         if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
 590                                 connected = true;
 591                         break;
 592                         /* DCE 3.2 */
 593                 case RADEON_HPD_5:
 594                         if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
 595                                 connected = true;
 596                         break;
 597                 case RADEON_HPD_6:
 598                         if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
 599                                 connected = true;
 600                         break;
 601                 default:
 602                         break;
 603                 }
 604         } else {
 605                 switch (hpd) {
 606                 case RADEON_HPD_1:
 607                         if (RREG32(DC_HOT_PLUG_DETECT1_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
 608                                 connected = true;
 609                         break;
 610                 case RADEON_HPD_2:
 611                         if (RREG32(DC_HOT_PLUG_DETECT2_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
 612                                 connected = true;
 613                         break;
 614                 case RADEON_HPD_3:
 615                         if (RREG32(DC_HOT_PLUG_DETECT3_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
 616                                 connected = true;
 617                         break;
 618                 default:
 619                         break;
 620                 }
 621         }
 622         return connected;
 623 }
 624
 625 void r600_hpd_set_polarity(struct radeon_device *rdev,
 626                            enum radeon_hpd_id hpd)
 627 {
 628         u32 tmp;
 629         bool connected = r600_hpd_sense(rdev, hpd);
 630
 631         if (ASIC_IS_DCE3(rdev)) {
 632                 switch (hpd) {
 633                 case RADEON_HPD_1:
 634                         tmp = RREG32(DC_HPD1_INT_CONTROL);
 635                         if (connected)
 636                                 tmp &= ~DC_HPDx_INT_POLARITY;
 637                         else
 638                                 tmp |= DC_HPDx_INT_POLARITY;
 639                         WREG32(DC_HPD1_INT_CONTROL, tmp);
 640                         break;
 641                 case RADEON_HPD_2:
 642                         tmp = RREG32(DC_HPD2_INT_CONTROL);
 643                         if (connected)
 644                                 tmp &= ~DC_HPDx_INT_POLARITY;
 645                         else
 646                                 tmp |= DC_HPDx_INT_POLARITY;
 647                         WREG32(DC_HPD2_INT_CONTROL, tmp);
 648                         break;
 649                 case RADEON_HPD_3:
 650                         tmp = RREG32(DC_HPD3_INT_CONTROL);
 651                         if (connected)
 652                                 tmp &= ~DC_HPDx_INT_POLARITY;
 653                         else
 654                                 tmp |= DC_HPDx_INT_POLARITY;
 655                         WREG32(DC_HPD3_INT_CONTROL, tmp);
 656                         break;
 657                 case RADEON_HPD_4:
 658                         tmp = RREG32(DC_HPD4_INT_CONTROL);
 659                         if (connected)
 660                                 tmp &= ~DC_HPDx_INT_POLARITY;
 661                         else
 662                                 tmp |= DC_HPDx_INT_POLARITY;
 663                         WREG32(DC_HPD4_INT_CONTROL, tmp);
 664                         break;
 665                 case RADEON_HPD_5:
 666                         tmp = RREG32(DC_HPD5_INT_CONTROL);
 667                         if (connected)
 668                                 tmp &= ~DC_HPDx_INT_POLARITY;
 669                         else
 670                                 tmp |= DC_HPDx_INT_POLARITY;
 671                         WREG32(DC_HPD5_INT_CONTROL, tmp);
 672                         break;
 673                         /* DCE 3.2 */
 674                 case RADEON_HPD_6:
 675                         tmp = RREG32(DC_HPD6_INT_CONTROL);
 676                         if (connected)
 677                                 tmp &= ~DC_HPDx_INT_POLARITY;
 678                         else
 679                                 tmp |= DC_HPDx_INT_POLARITY;
 680                         WREG32(DC_HPD6_INT_CONTROL, tmp);
 681                         break;
 682                 default:
 683                         break;
 684                 }
 685         } else {
 686                 switch (hpd) {
 687                 case RADEON_HPD_1:
 688                         tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
 689                         if (connected)
 690                                 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
 691                         else
 692                                 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
 693                         WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
 694                         break;
 695                 case RADEON_HPD_2:
 696                         tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
 697                         if (connected)
 698                                 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
 699                         else
 700                                 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
 701                         WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
 702                         break;
 703                 case RADEON_HPD_3:
 704                         tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
 705                         if (connected)
 706                                 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
 707                         else
 708                                 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
 709                         WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
 710                         break;
 711                 default:
 712                         break;
 713                 }
 714         }
 715 }
 716
 717 void r600_hpd_init(struct radeon_device *rdev)
 718 {
 719         struct drm_device *dev = rdev->ddev;
 720         struct drm_connector *connector;
 721         unsigned enable = 0;
 722
 723         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 724                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
 725
 726                 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
 727                     connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
 728                         /* don't try to enable hpd on eDP or LVDS avoid breaking the
 729                          * aux dp channel on imac and help (but not completely fix)
 730                          * https://bugzilla.redhat.com/show_bug.cgi?id=726143
 731                          */
 732                         continue;
 733                 }
 734                 if (ASIC_IS_DCE3(rdev)) {
 735                         u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) | DC_HPDx_RX_INT_TIMER(0xfa);
 736                         if (ASIC_IS_DCE32(rdev))
 737                                 tmp |= DC_HPDx_EN;
 738
 739                         switch (radeon_connector->hpd.hpd) {
 740                         case RADEON_HPD_1:
 741                                 WREG32(DC_HPD1_CONTROL, tmp);
 742                                 break;
 743                         case RADEON_HPD_2:
 744                                 WREG32(DC_HPD2_CONTROL, tmp);
 745                                 break;
 746                         case RADEON_HPD_3:
 747                                 WREG32(DC_HPD3_CONTROL, tmp);
 748                                 break;
 749                         case RADEON_HPD_4:
 750                                 WREG32(DC_HPD4_CONTROL, tmp);
 751                                 break;
 752                                 /* DCE 3.2 */
 753                         case RADEON_HPD_5:
 754                                 WREG32(DC_HPD5_CONTROL, tmp);
 755                                 break;
 756                         case RADEON_HPD_6:
 757                                 WREG32(DC_HPD6_CONTROL, tmp);
 758                                 break;
 759                         default:
 760                                 break;
 761                         }
 762                 } else {
 763                         switch (radeon_connector->hpd.hpd) {
 764                         case RADEON_HPD_1:
 765                                 WREG32(DC_HOT_PLUG_DETECT1_CONTROL, DC_HOT_PLUG_DETECTx_EN);
 766                                 break;
 767                         case RADEON_HPD_2:
 768                                 WREG32(DC_HOT_PLUG_DETECT2_CONTROL, DC_HOT_PLUG_DETECTx_EN);
 769                                 break;
 770                         case RADEON_HPD_3:
 771                                 WREG32(DC_HOT_PLUG_DETECT3_CONTROL, DC_HOT_PLUG_DETECTx_EN);
 772                                 break;
 773                         default:
 774                                 break;
 775                         }
 776                 }
 777                 enable |= 1 << radeon_connector->hpd.hpd;
 778                 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
 779         }
 780         radeon_irq_kms_enable_hpd(rdev, enable);
 781 }
 782
 783 void r600_hpd_fini(struct radeon_device *rdev)
 784 {
 785         struct drm_device *dev = rdev->ddev;
 786         struct drm_connector *connector;
 787         unsigned disable = 0;
 788
 789         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 790                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
 791                 if (ASIC_IS_DCE3(rdev)) {
 792                         switch (radeon_connector->hpd.hpd) {
 793                         case RADEON_HPD_1:
 794                                 WREG32(DC_HPD1_CONTROL, 0);
 795                                 break;
 796                         case RADEON_HPD_2:
 797                                 WREG32(DC_HPD2_CONTROL, 0);
 798                                 break;
 799                         case RADEON_HPD_3:
 800                                 WREG32(DC_HPD3_CONTROL, 0);
 801                                 break;
 802                         case RADEON_HPD_4:
 803                                 WREG32(DC_HPD4_CONTROL, 0);
 804                                 break;
 805                                 /* DCE 3.2 */
 806                         case RADEON_HPD_5:
 807                                 WREG32(DC_HPD5_CONTROL, 0);
 808                                 break;
 809                         case RADEON_HPD_6:
 810                                 WREG32(DC_HPD6_CONTROL, 0);
 811                                 break;
 812                         default:
 813                                 break;
 814                         }
 815                 } else {
 816                         switch (radeon_connector->hpd.hpd) {
 817                         case RADEON_HPD_1:
 818                                 WREG32(DC_HOT_PLUG_DETECT1_CONTROL, 0);
 819                                 break;
 820                         case RADEON_HPD_2:
 821                                 WREG32(DC_HOT_PLUG_DETECT2_CONTROL, 0);
 822                                 break;
 823                         case RADEON_HPD_3:
 824                                 WREG32(DC_HOT_PLUG_DETECT3_CONTROL, 0);
 825                                 break;
 826                         default:
 827                                 break;
 828                         }
 829                 }
 830                 disable |= 1 << radeon_connector->hpd.hpd;
 831         }
 832         radeon_irq_kms_disable_hpd(rdev, disable);
 833 }
 834
 835 /*
 836  * R600 PCIE GART
 837  */
 838 void r600_pcie_gart_tlb_flush(struct radeon_device *rdev)
 839 {
 840         unsigned i;
 841         u32 tmp;
 842
 843         /* flush hdp cache so updates hit vram */
 844         if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&
 845             !(rdev->flags & RADEON_IS_AGP)) {
 846                 void __iomem *ptr = (void *)rdev->gart.ptr;
 847                 u32 tmp;
 848
 849                 /* r7xx hw bug.  write to HDP_DEBUG1 followed by fb read
 850                  * rather than write to HDP_REG_COHERENCY_FLUSH_CNTL
 851                  * This seems to cause problems on some AGP cards. Just use the old
 852                  * method for them.
 853                  */
 854                 WREG32(HDP_DEBUG1, 0);
 855                 tmp = readl((void __iomem *)ptr);
 856         } else
 857                 WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
 858
 859         WREG32(VM_CONTEXT0_INVALIDATION_LOW_ADDR, rdev->mc.gtt_start >> 12);
 860         WREG32(VM_CONTEXT0_INVALIDATION_HIGH_ADDR, (rdev->mc.gtt_end - 1) >> 12);
 861         WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
 862         for (i = 0; i < rdev->usec_timeout; i++) {
 863                 /* read MC_STATUS */
 864                 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
 865                 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
 866                 if (tmp == 2) {
 867                         printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
 868                         return;
 869                 }
 870                 if (tmp) {
 871                         return;
 872                 }
 873                 udelay(1);
 874         }
 875 }
 876
 877 int r600_pcie_gart_init(struct radeon_device *rdev)
 878 {
 879         int r;
 880
 881         if (rdev->gart.robj) {
 882                 WARN(1, "R600 PCIE GART already initialized\n");
 883                 return 0;
 884         }
 885         /* Initialize common gart structure */
 886         r = radeon_gart_init(rdev);
 887         if (r)
 888                 return r;
 889         rdev->gart.table_size = rdev->gart.num_gpu_pages * 8;
 890         return radeon_gart_table_vram_alloc(rdev);
 891 }
 892
 893 static int r600_pcie_gart_enable(struct radeon_device *rdev)
 894 {
 895         u32 tmp;
 896         int r, i;
 897
 898         if (rdev->gart.robj == NULL) {
 899                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
 900                 return -EINVAL;
 901         }
 902         r = radeon_gart_table_vram_pin(rdev);
 903         if (r)
 904                 return r;
 905         radeon_gart_restore(rdev);
 906
 907         /* Setup L2 cache */
 908         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
 909                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
 910                                 EFFECTIVE_L2_QUEUE_SIZE(7));
 911         WREG32(VM_L2_CNTL2, 0);
 912         WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
 913         /* Setup TLB control */
 914         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
 915                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
 916                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
 917                 ENABLE_WAIT_L2_QUERY;
 918         WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
 919         WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
 920         WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp | ENABLE_L1_STRICT_ORDERING);
 921         WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
 922         WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
 923         WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
 924         WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
 925         WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
 926         WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
 927         WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
 928         WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
 929         WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
 930         WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
 931         WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
 932         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
 933         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
 934         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
 935         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
 936                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
 937         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
 938                         (u32)(rdev->dummy_page.addr >> 12));
 939         for (i = 1; i < 7; i++)
 940                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 941
 942         r600_pcie_gart_tlb_flush(rdev);
 943         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
 944                  (unsigned)(rdev->mc.gtt_size >> 20),
 945                  (unsigned long long)rdev->gart.table_addr);
 946         rdev->gart.ready = true;
 947         return 0;
 948 }
 949
 950 static void r600_pcie_gart_disable(struct radeon_device *rdev)
 951 {
 952         u32 tmp;
 953         int i;
 954
 955         /* Disable all tables */
 956         for (i = 0; i < 7; i++)
 957                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 958
 959         /* Disable L2 cache */
 960         WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
 961                                 EFFECTIVE_L2_QUEUE_SIZE(7));
 962         WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
 963         /* Setup L1 TLB control */
 964         tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
 965                 ENABLE_WAIT_L2_QUERY;
 966         WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
 967         WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
 968         WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
 969         WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
 970         WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
 971         WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
 972         WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
 973         WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
 974         WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp);
 975         WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp);
 976         WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
 977         WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
 978         WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp);
 979         WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
 980         radeon_gart_table_vram_unpin(rdev);
 981 }
 982
 983 static void r600_pcie_gart_fini(struct radeon_device *rdev)
 984 {
 985         radeon_gart_fini(rdev);
 986         r600_pcie_gart_disable(rdev);
 987         radeon_gart_table_vram_free(rdev);
 988 }
 989
 990 static void r600_agp_enable(struct radeon_device *rdev)
 991 {
 992         u32 tmp;
 993         int i;
 994
 995         /* Setup L2 cache */
 996         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
 997                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
 998                                 EFFECTIVE_L2_QUEUE_SIZE(7));
 999         WREG32(VM_L2_CNTL2, 0);
1000         WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
1001         /* Setup TLB control */
1002         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
1003                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1004                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
1005                 ENABLE_WAIT_L2_QUERY;
1006         WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
1007         WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
1008         WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp | ENABLE_L1_STRICT_ORDERING);
1009         WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
1010         WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
1011         WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
1012         WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
1013         WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
1014         WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
1015         WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
1016         WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
1017         WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
1018         WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
1019         WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
1020         for (i = 0; i < 7; i++)
1021                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
1022 }
1023
1024 int r600_mc_wait_for_idle(struct radeon_device *rdev)
1025 {
1026         unsigned i;
1027         u32 tmp;
1028
1029         for (i = 0; i < rdev->usec_timeout; i++) {
1030                 /* read MC_STATUS */
1031                 tmp = RREG32(R_000E50_SRBM_STATUS) & 0x3F00;
1032                 if (!tmp)
1033                         return 0;
1034                 udelay(1);
1035         }
1036         return -1;
1037 }
1038
1039 uint32_t rs780_mc_rreg(struct radeon_device *rdev, uint32_t reg)
1040 {
1041         uint32_t r;
1042
1043         WREG32(R_0028F8_MC_INDEX, S_0028F8_MC_IND_ADDR(reg));
1044         r = RREG32(R_0028FC_MC_DATA);
1045         WREG32(R_0028F8_MC_INDEX, ~C_0028F8_MC_IND_ADDR);
1046         return r;
1047 }
1048
1049 void rs780_mc_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v)
1050 {
1051         WREG32(R_0028F8_MC_INDEX, S_0028F8_MC_IND_ADDR(reg) |
1052                 S_0028F8_MC_IND_WR_EN(1));
1053         WREG32(R_0028FC_MC_DATA, v);
1054         WREG32(R_0028F8_MC_INDEX, 0x7F);
1055 }
1056
1057 static void r600_mc_program(struct radeon_device *rdev)
1058 {
1059         struct rv515_mc_save save;
1060         u32 tmp;
1061         int i, j;
1062
1063         /* Initialize HDP */
1064         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1065                 WREG32((0x2c14 + j), 0x00000000);
1066                 WREG32((0x2c18 + j), 0x00000000);
1067                 WREG32((0x2c1c + j), 0x00000000);
1068                 WREG32((0x2c20 + j), 0x00000000);
1069                 WREG32((0x2c24 + j), 0x00000000);
1070         }
1071         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
1072
1073         rv515_mc_stop(rdev, &save);
1074         if (r600_mc_wait_for_idle(rdev)) {
1075                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1076         }
1077         /* Lockout access through VGA aperture (doesn't exist before R600) */
1078         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
1079         /* Update configuration */
1080         if (rdev->flags & RADEON_IS_AGP) {
1081                 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
1082                         /* VRAM before AGP */
1083                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1084                                 rdev->mc.vram_start >> 12);
1085                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1086                                 rdev->mc.gtt_end >> 12);
1087                 } else {
1088                         /* VRAM after AGP */
1089                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1090                                 rdev->mc.gtt_start >> 12);
1091                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1092                                 rdev->mc.vram_end >> 12);
1093                 }
1094         } else {
1095                 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, rdev->mc.vram_start >> 12);
1096                 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, rdev->mc.vram_end >> 12);
1097         }
1098         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
1099         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
1100         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
1101         WREG32(MC_VM_FB_LOCATION, tmp);
1102         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
1103         WREG32(HDP_NONSURFACE_INFO, (2 << 7));
1104         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1105         if (rdev->flags & RADEON_IS_AGP) {
1106                 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 22);
1107                 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 22);
1108                 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
1109         } else {
1110                 WREG32(MC_VM_AGP_BASE, 0);
1111                 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
1112                 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
1113         }
1114         if (r600_mc_wait_for_idle(rdev)) {
1115                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1116         }
1117         rv515_mc_resume(rdev, &save);
1118         /* we need to own VRAM, so turn off the VGA renderer here
1119          * to stop it overwriting our objects */
1120         rv515_vga_render_disable(rdev);
1121 }
1122
1123 /**
1124  * r600_vram_gtt_location - try to find VRAM & GTT location
1125  * @rdev: radeon device structure holding all necessary informations
1126  * @mc: memory controller structure holding memory informations
1127  *
1128  * Function will place try to place VRAM at same place as in CPU (PCI)
1129  * address space as some GPU seems to have issue when we reprogram at
1130  * different address space.
1131  *
1132  * If there is not enough space to fit the unvisible VRAM after the
1133  * aperture then we limit the VRAM size to the aperture.
1134  *
1135  * If we are using AGP then place VRAM adjacent to AGP aperture are we need
1136  * them to be in one from GPU point of view so that we can program GPU to
1137  * catch access outside them (weird GPU policy see ??).
1138  *
1139  * This function will never fails, worst case are limiting VRAM or GTT.
1140  *
1141  * Note: GTT start, end, size should be initialized before calling this
1142  * function on AGP platform.
1143  */
1144 static void r600_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
1145 {
1146         u64 size_bf, size_af;
1147
1148         if (mc->mc_vram_size > 0xE0000000) {
1149                 /* leave room for at least 512M GTT */
1150                 dev_warn(rdev->dev, "limiting VRAM\n");
1151                 mc->real_vram_size = 0xE0000000;
1152                 mc->mc_vram_size = 0xE0000000;
1153         }
1154         if (rdev->flags & RADEON_IS_AGP) {
1155                 size_bf = mc->gtt_start;
1156                 size_af = mc->mc_mask - mc->gtt_end;
1157                 if (size_bf > size_af) {
1158                         if (mc->mc_vram_size > size_bf) {
1159                                 dev_warn(rdev->dev, "limiting VRAM\n");
1160                                 mc->real_vram_size = size_bf;
1161                                 mc->mc_vram_size = size_bf;
1162                         }
1163                         mc->vram_start = mc->gtt_start - mc->mc_vram_size;
1164                 } else {
1165                         if (mc->mc_vram_size > size_af) {
1166                                 dev_warn(rdev->dev, "limiting VRAM\n");
1167                                 mc->real_vram_size = size_af;
1168                                 mc->mc_vram_size = size_af;
1169                         }
1170                         mc->vram_start = mc->gtt_end + 1;
1171                 }
1172                 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
1173                 dev_info(rdev->dev, "VRAM: %lluM 0x%08llX - 0x%08llX (%lluM used)\n",
1174                                 mc->mc_vram_size >> 20, mc->vram_start,
1175                                 mc->vram_end, mc->real_vram_size >> 20);
1176         } else {
1177                 u64 base = 0;
1178                 if (rdev->flags & RADEON_IS_IGP) {
1179                         base = RREG32(MC_VM_FB_LOCATION) & 0xFFFF;
1180                         base <<= 24;
1181                 }
1182                 radeon_vram_location(rdev, &rdev->mc, base);
1183                 rdev->mc.gtt_base_align = 0;
1184                 radeon_gtt_location(rdev, mc);
1185         }
1186 }
1187
1188 static int r600_mc_init(struct radeon_device *rdev)
1189 {
1190         u32 tmp;
1191         int chansize, numchan;
1192         uint32_t h_addr, l_addr;
1193         unsigned long long k8_addr;
1194
1195         /* Get VRAM informations */
1196         rdev->mc.vram_is_ddr = true;
1197         tmp = RREG32(RAMCFG);
1198         if (tmp & CHANSIZE_OVERRIDE) {
1199                 chansize = 16;
1200         } else if (tmp & CHANSIZE_MASK) {
1201                 chansize = 64;
1202         } else {
1203                 chansize = 32;
1204         }
1205         tmp = RREG32(CHMAP);
1206         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1207         case 0:
1208         default:
1209                 numchan = 1;
1210                 break;
1211         case 1:
1212                 numchan = 2;
1213                 break;
1214         case 2:
1215                 numchan = 4;
1216                 break;
1217         case 3:
1218                 numchan = 8;
1219                 break;
1220         }
1221         rdev->mc.vram_width = numchan * chansize;
1222         /* Could aper size report 0 ? */
1223         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
1224         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
1225         /* Setup GPU memory space */
1226         rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
1227         rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
1228         rdev->mc.visible_vram_size = rdev->mc.aper_size;
1229         r600_vram_gtt_location(rdev, &rdev->mc);
1230
1231         if (rdev->flags & RADEON_IS_IGP) {
1232                 rs690_pm_info(rdev);
1233                 rdev->mc.igp_sideport_enabled = radeon_atombios_sideport_present(rdev);
1234
1235                 if (rdev->family == CHIP_RS780 || rdev->family == CHIP_RS880) {
1236                         /* Use K8 direct mapping for fast fb access. */
1237                         rdev->fastfb_working = false;
1238                         h_addr = G_000012_K8_ADDR_EXT(RREG32_MC(R_000012_MC_MISC_UMA_CNTL));
1239                         l_addr = RREG32_MC(R_000011_K8_FB_LOCATION);
1240                         k8_addr = ((unsigned long long)h_addr) << 32 | l_addr;
1241 #if defined(CONFIG_X86_32) && !defined(CONFIG_X86_PAE)
1242                         if (k8_addr + rdev->mc.visible_vram_size < 0x100000000ULL)
1243 #endif
1244                         {
1245                                 /* FastFB shall be used with UMA memory. Here it is simply disabled when sideport
1246                                 * memory is present.
1247                                 */
1248                                 if (rdev->mc.igp_sideport_enabled == false && radeon_fastfb == 1) {
1249                                         DRM_INFO("Direct mapping: aper base at 0x%llx, replaced by direct mapping base 0x%llx.\n",
1250                                                 (unsigned long long)rdev->mc.aper_base, k8_addr);
1251                                         rdev->mc.aper_base = (resource_size_t)k8_addr;
1252                                         rdev->fastfb_working = true;
1253                                 }
1254                         }
1255                 }
1256         }
1257
1258         radeon_update_bandwidth_info(rdev);
1259         return 0;
1260 }
1261
1262 int r600_vram_scratch_init(struct radeon_device *rdev)
1263 {
1264         int r;
1265
1266         if (rdev->vram_scratch.robj == NULL) {
1267                 r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE,
1268                                      PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM,
1269                                      NULL, &rdev->vram_scratch.robj);
1270                 if (r) {
1271                         return r;
1272                 }
1273         }
1274
1275         r = radeon_bo_reserve(rdev->vram_scratch.robj, false);
1276         if (unlikely(r != 0))
1277                 return r;
1278         r = radeon_bo_pin(rdev->vram_scratch.robj,
1279                           RADEON_GEM_DOMAIN_VRAM, &rdev->vram_scratch.gpu_addr);
1280         if (r) {
1281                 radeon_bo_unreserve(rdev->vram_scratch.robj);
1282                 return r;
1283         }
1284         r = radeon_bo_kmap(rdev->vram_scratch.robj,
1285                                 (void **)&rdev->vram_scratch.ptr);
1286         if (r)
1287                 radeon_bo_unpin(rdev->vram_scratch.robj);
1288         radeon_bo_unreserve(rdev->vram_scratch.robj);
1289
1290         return r;
1291 }
1292
1293 void r600_vram_scratch_fini(struct radeon_device *rdev)
1294 {
1295         int r;
1296
1297         if (rdev->vram_scratch.robj == NULL) {
1298                 return;
1299         }
1300         r = radeon_bo_reserve(rdev->vram_scratch.robj, false);
1301         if (likely(r == 0)) {
1302                 radeon_bo_kunmap(rdev->vram_scratch.robj);
1303                 radeon_bo_unpin(rdev->vram_scratch.robj);
1304                 radeon_bo_unreserve(rdev->vram_scratch.robj);
1305         }
1306         radeon_bo_unref(&rdev->vram_scratch.robj);
1307 }
1308
1309 void r600_set_bios_scratch_engine_hung(struct radeon_device *rdev, bool hung)
1310 {
1311         u32 tmp = RREG32(R600_BIOS_3_SCRATCH);
1312
1313         if (hung)
1314                 tmp |= ATOM_S3_ASIC_GUI_ENGINE_HUNG;
1315         else
1316                 tmp &= ~ATOM_S3_ASIC_GUI_ENGINE_HUNG;
1317
1318         WREG32(R600_BIOS_3_SCRATCH, tmp);
1319 }
1320
1321 static void r600_print_gpu_status_regs(struct radeon_device *rdev)
1322 {
1323         dev_info(rdev->dev, "  R_008010_GRBM_STATUS      = 0x%08X\n",
1324                  RREG32(R_008010_GRBM_STATUS));
1325         dev_info(rdev->dev, "  R_008014_GRBM_STATUS2     = 0x%08X\n",
1326                  RREG32(R_008014_GRBM_STATUS2));
1327         dev_info(rdev->dev, "  R_000E50_SRBM_STATUS      = 0x%08X\n",
1328                  RREG32(R_000E50_SRBM_STATUS));
1329         dev_info(rdev->dev, "  R_008674_CP_STALLED_STAT1 = 0x%08X\n",
1330                  RREG32(CP_STALLED_STAT1));
1331         dev_info(rdev->dev, "  R_008678_CP_STALLED_STAT2 = 0x%08X\n",
1332                  RREG32(CP_STALLED_STAT2));
1333         dev_info(rdev->dev, "  R_00867C_CP_BUSY_STAT     = 0x%08X\n",
1334                  RREG32(CP_BUSY_STAT));
1335         dev_info(rdev->dev, "  R_008680_CP_STAT          = 0x%08X\n",
1336                  RREG32(CP_STAT));
1337         dev_info(rdev->dev, "  R_00D034_DMA_STATUS_REG   = 0x%08X\n",
1338                 RREG32(DMA_STATUS_REG));
1339 }
1340
1341 static bool r600_is_display_hung(struct radeon_device *rdev)
1342 {
1343         u32 crtc_hung = 0;
1344         u32 crtc_status[2];
1345         u32 i, j, tmp;
1346
1347         for (i = 0; i < rdev->num_crtc; i++) {
1348                 if (RREG32(AVIVO_D1CRTC_CONTROL + crtc_offsets[i]) & AVIVO_CRTC_EN) {
1349                         crtc_status[i] = RREG32(AVIVO_D1CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
1350                         crtc_hung |= (1 << i);
1351                 }
1352         }
1353
1354         for (j = 0; j < 10; j++) {
1355                 for (i = 0; i < rdev->num_crtc; i++) {
1356                         if (crtc_hung & (1 << i)) {
1357                                 tmp = RREG32(AVIVO_D1CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
1358                                 if (tmp != crtc_status[i])
1359                                         crtc_hung &= ~(1 << i);
1360                         }
1361                 }
1362                 if (crtc_hung == 0)
1363                         return false;
1364                 udelay(100);
1365         }
1366
1367         return true;
1368 }
1369
1370 static u32 r600_gpu_check_soft_reset(struct radeon_device *rdev)
1371 {
1372         u32 reset_mask = 0;
1373         u32 tmp;
1374
1375         /* GRBM_STATUS */
1376         tmp = RREG32(R_008010_GRBM_STATUS);
1377         if (rdev->family >= CHIP_RV770) {
1378                 if (G_008010_PA_BUSY(tmp) | G_008010_SC_BUSY(tmp) |
1379                     G_008010_SH_BUSY(tmp) | G_008010_SX_BUSY(tmp) |
1380                     G_008010_TA_BUSY(tmp) | G_008010_VGT_BUSY(tmp) |
1381                     G_008010_DB03_BUSY(tmp) | G_008010_CB03_BUSY(tmp) |
1382                     G_008010_SPI03_BUSY(tmp) | G_008010_VGT_BUSY_NO_DMA(tmp))
1383                         reset_mask |= RADEON_RESET_GFX;
1384         } else {
1385                 if (G_008010_PA_BUSY(tmp) | G_008010_SC_BUSY(tmp) |
1386                     G_008010_SH_BUSY(tmp) | G_008010_SX_BUSY(tmp) |
1387                     G_008010_TA03_BUSY(tmp) | G_008010_VGT_BUSY(tmp) |
1388                     G_008010_DB03_BUSY(tmp) | G_008010_CB03_BUSY(tmp) |
1389                     G_008010_SPI03_BUSY(tmp) | G_008010_VGT_BUSY_NO_DMA(tmp))
1390                         reset_mask |= RADEON_RESET_GFX;
1391         }
1392
1393         if (G_008010_CF_RQ_PENDING(tmp) | G_008010_PF_RQ_PENDING(tmp) |
1394             G_008010_CP_BUSY(tmp) | G_008010_CP_COHERENCY_BUSY(tmp))
1395                 reset_mask |= RADEON_RESET_CP;
1396
1397         if (G_008010_GRBM_EE_BUSY(tmp))
1398                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
1399
1400         /* DMA_STATUS_REG */
1401         tmp = RREG32(DMA_STATUS_REG);
1402         if (!(tmp & DMA_IDLE))
1403                 reset_mask |= RADEON_RESET_DMA;
1404
1405         /* SRBM_STATUS */
1406         tmp = RREG32(R_000E50_SRBM_STATUS);
1407         if (G_000E50_RLC_RQ_PENDING(tmp) | G_000E50_RLC_BUSY(tmp))
1408                 reset_mask |= RADEON_RESET_RLC;
1409
1410         if (G_000E50_IH_BUSY(tmp))
1411                 reset_mask |= RADEON_RESET_IH;
1412
1413         if (G_000E50_SEM_BUSY(tmp))
1414                 reset_mask |= RADEON_RESET_SEM;
1415
1416         if (G_000E50_GRBM_RQ_PENDING(tmp))
1417                 reset_mask |= RADEON_RESET_GRBM;
1418
1419         if (G_000E50_VMC_BUSY(tmp))
1420                 reset_mask |= RADEON_RESET_VMC;
1421
1422         if (G_000E50_MCB_BUSY(tmp) | G_000E50_MCDZ_BUSY(tmp) |
1423             G_000E50_MCDY_BUSY(tmp) | G_000E50_MCDX_BUSY(tmp) |
1424             G_000E50_MCDW_BUSY(tmp))
1425                 reset_mask |= RADEON_RESET_MC;
1426
1427         if (r600_is_display_hung(rdev))
1428                 reset_mask |= RADEON_RESET_DISPLAY;
1429
1430         /* Skip MC reset as it's mostly likely not hung, just busy */
1431         if (reset_mask & RADEON_RESET_MC) {
1432                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
1433                 reset_mask &= ~RADEON_RESET_MC;
1434         }
1435
1436         return reset_mask;
1437 }
1438
1439 static void r600_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
1440 {
1441         struct rv515_mc_save save;
1442         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
1443         u32 tmp;
1444
1445         if (reset_mask == 0)
1446                 return;
1447
1448         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
1449
1450         r600_print_gpu_status_regs(rdev);
1451
1452         /* Disable CP parsing/prefetching */
1453         if (rdev->family >= CHIP_RV770)
1454                 WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1) | S_0086D8_CP_PFP_HALT(1));
1455         else
1456                 WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
1457
1458         /* disable the RLC */
1459         WREG32(RLC_CNTL, 0);
1460
1461         if (reset_mask & RADEON_RESET_DMA) {
1462                 /* Disable DMA */
1463                 tmp = RREG32(DMA_RB_CNTL);
1464                 tmp &= ~DMA_RB_ENABLE;
1465                 WREG32(DMA_RB_CNTL, tmp);
1466         }
1467
1468         mdelay(50);
1469
1470         rv515_mc_stop(rdev, &save);
1471         if (r600_mc_wait_for_idle(rdev)) {
1472                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1473         }
1474
1475         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
1476                 if (rdev->family >= CHIP_RV770)
1477                         grbm_soft_reset |= S_008020_SOFT_RESET_DB(1) |
1478                                 S_008020_SOFT_RESET_CB(1) |
1479                                 S_008020_SOFT_RESET_PA(1) |
1480                                 S_008020_SOFT_RESET_SC(1) |
1481                                 S_008020_SOFT_RESET_SPI(1) |
1482                                 S_008020_SOFT_RESET_SX(1) |
1483                                 S_008020_SOFT_RESET_SH(1) |
1484                                 S_008020_SOFT_RESET_TC(1) |
1485                                 S_008020_SOFT_RESET_TA(1) |
1486                                 S_008020_SOFT_RESET_VC(1) |
1487                                 S_008020_SOFT_RESET_VGT(1);
1488                 else
1489                         grbm_soft_reset |= S_008020_SOFT_RESET_CR(1) |
1490                                 S_008020_SOFT_RESET_DB(1) |
1491                                 S_008020_SOFT_RESET_CB(1) |
1492                                 S_008020_SOFT_RESET_PA(1) |
1493                                 S_008020_SOFT_RESET_SC(1) |
1494                                 S_008020_SOFT_RESET_SMX(1) |
1495                                 S_008020_SOFT_RESET_SPI(1) |
1496                                 S_008020_SOFT_RESET_SX(1) |
1497                                 S_008020_SOFT_RESET_SH(1) |
1498                                 S_008020_SOFT_RESET_TC(1) |
1499                                 S_008020_SOFT_RESET_TA(1) |
1500                                 S_008020_SOFT_RESET_VC(1) |
1501                                 S_008020_SOFT_RESET_VGT(1);
1502         }
1503
1504         if (reset_mask & RADEON_RESET_CP) {
1505                 grbm_soft_reset |= S_008020_SOFT_RESET_CP(1) |
1506                         S_008020_SOFT_RESET_VGT(1);
1507
1508                 srbm_soft_reset |= S_000E60_SOFT_RESET_GRBM(1);
1509         }
1510
1511         if (reset_mask & RADEON_RESET_DMA) {
1512                 if (rdev->family >= CHIP_RV770)
1513                         srbm_soft_reset |= RV770_SOFT_RESET_DMA;
1514                 else
1515                         srbm_soft_reset |= SOFT_RESET_DMA;
1516         }
1517
1518         if (reset_mask & RADEON_RESET_RLC)
1519                 srbm_soft_reset |= S_000E60_SOFT_RESET_RLC(1);
1520
1521         if (reset_mask & RADEON_RESET_SEM)
1522                 srbm_soft_reset |= S_000E60_SOFT_RESET_SEM(1);
1523
1524         if (reset_mask & RADEON_RESET_IH)
1525                 srbm_soft_reset |= S_000E60_SOFT_RESET_IH(1);
1526
1527         if (reset_mask & RADEON_RESET_GRBM)
1528                 srbm_soft_reset |= S_000E60_SOFT_RESET_GRBM(1);
1529
1530         if (!(rdev->flags & RADEON_IS_IGP)) {
1531                 if (reset_mask & RADEON_RESET_MC)
1532                         srbm_soft_reset |= S_000E60_SOFT_RESET_MC(1);
1533         }
1534
1535         if (reset_mask & RADEON_RESET_VMC)
1536                 srbm_soft_reset |= S_000E60_SOFT_RESET_VMC(1);
1537
1538         if (grbm_soft_reset) {
1539                 tmp = RREG32(R_008020_GRBM_SOFT_RESET);
1540                 tmp |= grbm_soft_reset;
1541                 dev_info(rdev->dev, "R_008020_GRBM_SOFT_RESET=0x%08X\n", tmp);
1542                 WREG32(R_008020_GRBM_SOFT_RESET, tmp);
1543                 tmp = RREG32(R_008020_GRBM_SOFT_RESET);
1544
1545                 udelay(50);
1546
1547                 tmp &= ~grbm_soft_reset;
1548                 WREG32(R_008020_GRBM_SOFT_RESET, tmp);
1549                 tmp = RREG32(R_008020_GRBM_SOFT_RESET);
1550         }
1551
1552         if (srbm_soft_reset) {
1553                 tmp = RREG32(SRBM_SOFT_RESET);
1554                 tmp |= srbm_soft_reset;
1555                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1556                 WREG32(SRBM_SOFT_RESET, tmp);
1557                 tmp = RREG32(SRBM_SOFT_RESET);
1558
1559                 udelay(50);
1560
1561                 tmp &= ~srbm_soft_reset;
1562                 WREG32(SRBM_SOFT_RESET, tmp);
1563                 tmp = RREG32(SRBM_SOFT_RESET);
1564         }
1565
1566         /* Wait a little for things to settle down */
1567         mdelay(1);
1568
1569         rv515_mc_resume(rdev, &save);
1570         udelay(50);
1571
1572         r600_print_gpu_status_regs(rdev);
1573 }
1574
1575 int r600_asic_reset(struct radeon_device *rdev)
1576 {
1577         u32 reset_mask;
1578
1579         reset_mask = r600_gpu_check_soft_reset(rdev);
1580
1581         if (reset_mask)
1582                 r600_set_bios_scratch_engine_hung(rdev, true);
1583
1584         r600_gpu_soft_reset(rdev, reset_mask);
1585
1586         reset_mask = r600_gpu_check_soft_reset(rdev);
1587
1588         if (!reset_mask)
1589                 r600_set_bios_scratch_engine_hung(rdev, false);
1590
1591         return 0;
1592 }
1593
1594 /**
1595  * r600_gfx_is_lockup - Check if the GFX engine is locked up
1596  *
1597  * @rdev: radeon_device pointer
1598  * @ring: radeon_ring structure holding ring information
1599  *
1600  * Check if the GFX engine is locked up.
1601  * Returns true if the engine appears to be locked up, false if not.
1602  */
1603 bool r600_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
1604 {
1605         u32 reset_mask = r600_gpu_check_soft_reset(rdev);
1606
1607         if (!(reset_mask & (RADEON_RESET_GFX |
1608                             RADEON_RESET_COMPUTE |
1609                             RADEON_RESET_CP))) {
1610                 radeon_ring_lockup_update(ring);
1611                 return false;
1612         }
1613         /* force CP activities */
1614         radeon_ring_force_activity(rdev, ring);
1615         return radeon_ring_test_lockup(rdev, ring);
1616 }
1617
1618 /**
1619  * r600_dma_is_lockup - Check if the DMA engine is locked up
1620  *
1621  * @rdev: radeon_device pointer
1622  * @ring: radeon_ring structure holding ring information
1623  *
1624  * Check if the async DMA engine is locked up.
1625  * Returns true if the engine appears to be locked up, false if not.
1626  */
1627 bool r600_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
1628 {
1629         u32 reset_mask = r600_gpu_check_soft_reset(rdev);
1630
1631         if (!(reset_mask & RADEON_RESET_DMA)) {
1632                 radeon_ring_lockup_update(ring);
1633                 return false;
1634         }
1635         /* force ring activities */
1636         radeon_ring_force_activity(rdev, ring);
1637         return radeon_ring_test_lockup(rdev, ring);
1638 }
1639
1640 u32 r6xx_remap_render_backend(struct radeon_device *rdev,
1641                               u32 tiling_pipe_num,
1642                               u32 max_rb_num,
1643                               u32 total_max_rb_num,
1644                               u32 disabled_rb_mask)
1645 {
1646         u32 rendering_pipe_num, rb_num_width, req_rb_num;
1647         u32 pipe_rb_ratio, pipe_rb_remain, tmp;
1648         u32 data = 0, mask = 1 << (max_rb_num - 1);
1649         unsigned i, j;
1650
1651         /* mask out the RBs that don't exist on that asic */
1652         tmp = disabled_rb_mask | ((0xff << max_rb_num) & 0xff);
1653         /* make sure at least one RB is available */
1654         if ((tmp & 0xff) != 0xff)
1655                 disabled_rb_mask = tmp;
1656
1657         rendering_pipe_num = 1 << tiling_pipe_num;
1658         req_rb_num = total_max_rb_num - r600_count_pipe_bits(disabled_rb_mask);
1659         BUG_ON(rendering_pipe_num < req_rb_num);
1660
1661         pipe_rb_ratio = rendering_pipe_num / req_rb_num;
1662         pipe_rb_remain = rendering_pipe_num - pipe_rb_ratio * req_rb_num;
1663
1664         if (rdev->family <= CHIP_RV740) {
1665                 /* r6xx/r7xx */
1666                 rb_num_width = 2;
1667         } else {
1668                 /* eg+ */
1669                 rb_num_width = 4;
1670         }
1671
1672         for (i = 0; i < max_rb_num; i++) {
1673                 if (!(mask & disabled_rb_mask)) {
1674                         for (j = 0; j < pipe_rb_ratio; j++) {
1675                                 data <<= rb_num_width;
1676                                 data |= max_rb_num - i - 1;
1677                         }
1678                         if (pipe_rb_remain) {
1679                                 data <<= rb_num_width;
1680                                 data |= max_rb_num - i - 1;
1681                                 pipe_rb_remain--;
1682                         }
1683                 }
1684                 mask >>= 1;
1685         }
1686
1687         return data;
1688 }
1689
1690 int r600_count_pipe_bits(uint32_t val)
1691 {
1692         return hweight32(val);
1693 }
1694
1695 static void r600_gpu_init(struct radeon_device *rdev)
1696 {
1697         u32 tiling_config;
1698         u32 ramcfg;
1699         u32 cc_rb_backend_disable;
1700         u32 cc_gc_shader_pipe_config;
1701         u32 tmp;
1702         int i, j;
1703         u32 sq_config;
1704         u32 sq_gpr_resource_mgmt_1 = 0;
1705         u32 sq_gpr_resource_mgmt_2 = 0;
1706         u32 sq_thread_resource_mgmt = 0;
1707         u32 sq_stack_resource_mgmt_1 = 0;
1708         u32 sq_stack_resource_mgmt_2 = 0;
1709         u32 disabled_rb_mask;
1710
1711         rdev->config.r600.tiling_group_size = 256;
1712         switch (rdev->family) {
1713         case CHIP_R600:
1714                 rdev->config.r600.max_pipes = 4;
1715                 rdev->config.r600.max_tile_pipes = 8;
1716                 rdev->config.r600.max_simds = 4;
1717                 rdev->config.r600.max_backends = 4;
1718                 rdev->config.r600.max_gprs = 256;
1719                 rdev->config.r600.max_threads = 192;
1720                 rdev->config.r600.max_stack_entries = 256;
1721                 rdev->config.r600.max_hw_contexts = 8;
1722                 rdev->config.r600.max_gs_threads = 16;
1723                 rdev->config.r600.sx_max_export_size = 128;
1724                 rdev->config.r600.sx_max_export_pos_size = 16;
1725                 rdev->config.r600.sx_max_export_smx_size = 128;
1726                 rdev->config.r600.sq_num_cf_insts = 2;
1727                 break;
1728         case CHIP_RV630:
1729         case CHIP_RV635:
1730                 rdev->config.r600.max_pipes = 2;
1731                 rdev->config.r600.max_tile_pipes = 2;
1732                 rdev->config.r600.max_simds = 3;
1733                 rdev->config.r600.max_backends = 1;
1734                 rdev->config.r600.max_gprs = 128;
1735                 rdev->config.r600.max_threads = 192;
1736                 rdev->config.r600.max_stack_entries = 128;
1737                 rdev->config.r600.max_hw_contexts = 8;
1738                 rdev->config.r600.max_gs_threads = 4;
1739                 rdev->config.r600.sx_max_export_size = 128;
1740                 rdev->config.r600.sx_max_export_pos_size = 16;
1741                 rdev->config.r600.sx_max_export_smx_size = 128;
1742                 rdev->config.r600.sq_num_cf_insts = 2;
1743                 break;
1744         case CHIP_RV610:
1745         case CHIP_RV620:
1746         case CHIP_RS780:
1747         case CHIP_RS880:
1748                 rdev->config.r600.max_pipes = 1;
1749                 rdev->config.r600.max_tile_pipes = 1;
1750                 rdev->config.r600.max_simds = 2;
1751                 rdev->config.r600.max_backends = 1;
1752                 rdev->config.r600.max_gprs = 128;
1753                 rdev->config.r600.max_threads = 192;
1754                 rdev->config.r600.max_stack_entries = 128;
1755                 rdev->config.r600.max_hw_contexts = 4;
1756                 rdev->config.r600.max_gs_threads = 4;
1757                 rdev->config.r600.sx_max_export_size = 128;
1758                 rdev->config.r600.sx_max_export_pos_size = 16;
1759                 rdev->config.r600.sx_max_export_smx_size = 128;
1760                 rdev->config.r600.sq_num_cf_insts = 1;
1761                 break;
1762         case CHIP_RV670:
1763                 rdev->config.r600.max_pipes = 4;
1764                 rdev->config.r600.max_tile_pipes = 4;
1765                 rdev->config.r600.max_simds = 4;
1766                 rdev->config.r600.max_backends = 4;
1767                 rdev->config.r600.max_gprs = 192;
1768                 rdev->config.r600.max_threads = 192;
1769                 rdev->config.r600.max_stack_entries = 256;
1770                 rdev->config.r600.max_hw_contexts = 8;
1771                 rdev->config.r600.max_gs_threads = 16;
1772                 rdev->config.r600.sx_max_export_size = 128;
1773                 rdev->config.r600.sx_max_export_pos_size = 16;
1774                 rdev->config.r600.sx_max_export_smx_size = 128;
1775                 rdev->config.r600.sq_num_cf_insts = 2;
1776                 break;
1777         default:
1778                 break;
1779         }
1780
1781         /* Initialize HDP */
1782         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1783                 WREG32((0x2c14 + j), 0x00000000);
1784                 WREG32((0x2c18 + j), 0x00000000);
1785                 WREG32((0x2c1c + j), 0x00000000);
1786                 WREG32((0x2c20 + j), 0x00000000);
1787                 WREG32((0x2c24 + j), 0x00000000);
1788         }
1789
1790         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
1791
1792         /* Setup tiling */
1793         tiling_config = 0;
1794         ramcfg = RREG32(RAMCFG);
1795         switch (rdev->config.r600.max_tile_pipes) {
1796         case 1:
1797                 tiling_config |= PIPE_TILING(0);
1798                 break;
1799         case 2:
1800                 tiling_config |= PIPE_TILING(1);
1801                 break;
1802         case 4:
1803                 tiling_config |= PIPE_TILING(2);
1804                 break;
1805         case 8:
1806                 tiling_config |= PIPE_TILING(3);
1807                 break;
1808         default:
1809                 break;
1810         }
1811         rdev->config.r600.tiling_npipes = rdev->config.r600.max_tile_pipes;
1812         rdev->config.r600.tiling_nbanks = 4 << ((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
1813         tiling_config |= BANK_TILING((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
1814         tiling_config |= GROUP_SIZE((ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
1815
1816         tmp = (ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT;
1817         if (tmp > 3) {
1818                 tiling_config |= ROW_TILING(3);
1819                 tiling_config |= SAMPLE_SPLIT(3);
1820         } else {
1821                 tiling_config |= ROW_TILING(tmp);
1822                 tiling_config |= SAMPLE_SPLIT(tmp);
1823         }
1824         tiling_config |= BANK_SWAPS(1);
1825
1826         cc_rb_backend_disable = RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000;
1827         tmp = R6XX_MAX_BACKENDS -
1828                 r600_count_pipe_bits((cc_rb_backend_disable >> 16) & R6XX_MAX_BACKENDS_MASK);
1829         if (tmp < rdev->config.r600.max_backends) {
1830                 rdev->config.r600.max_backends = tmp;
1831         }
1832
1833         cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0x00ffff00;
1834         tmp = R6XX_MAX_PIPES -
1835                 r600_count_pipe_bits((cc_gc_shader_pipe_config >> 8) & R6XX_MAX_PIPES_MASK);
1836         if (tmp < rdev->config.r600.max_pipes) {
1837                 rdev->config.r600.max_pipes = tmp;
1838         }
1839         tmp = R6XX_MAX_SIMDS -
1840                 r600_count_pipe_bits((cc_gc_shader_pipe_config >> 16) & R6XX_MAX_SIMDS_MASK);
1841         if (tmp < rdev->config.r600.max_simds) {
1842                 rdev->config.r600.max_simds = tmp;
1843         }
1844
1845         disabled_rb_mask = (RREG32(CC_RB_BACKEND_DISABLE) >> 16) & R6XX_MAX_BACKENDS_MASK;
1846         tmp = (tiling_config & PIPE_TILING__MASK) >> PIPE_TILING__SHIFT;
1847         tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.r600.max_backends,
1848                                         R6XX_MAX_BACKENDS, disabled_rb_mask);
1849         tiling_config |= tmp << 16;
1850         rdev->config.r600.backend_map = tmp;
1851
1852         rdev->config.r600.tile_config = tiling_config;
1853         WREG32(GB_TILING_CONFIG, tiling_config);
1854         WREG32(DCP_TILING_CONFIG, tiling_config & 0xffff);
1855         WREG32(HDP_TILING_CONFIG, tiling_config & 0xffff);
1856         WREG32(DMA_TILING_CONFIG, tiling_config & 0xffff);
1857
1858         tmp = R6XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> 8);
1859         WREG32(VGT_OUT_DEALLOC_CNTL, (tmp * 4) & DEALLOC_DIST_MASK);
1860         WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, ((tmp * 4) - 2) & VTX_REUSE_DEPTH_MASK);
1861
1862         /* Setup some CP states */
1863         WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) | ROQ_IB2_START(0x2b)));
1864         WREG32(CP_MEQ_THRESHOLDS, (MEQ_END(0x40) | ROQ_END(0x40)));
1865
1866         WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO | SYNC_GRADIENT |
1867                              SYNC_WALKER | SYNC_ALIGNER));
1868         /* Setup various GPU states */
1869         if (rdev->family == CHIP_RV670)
1870                 WREG32(ARB_GDEC_RD_CNTL, 0x00000021);
1871
1872         tmp = RREG32(SX_DEBUG_1);
1873         tmp |= SMX_EVENT_RELEASE;
1874         if ((rdev->family > CHIP_R600))
1875                 tmp |= ENABLE_NEW_SMX_ADDRESS;
1876         WREG32(SX_DEBUG_1, tmp);
1877
1878         if (((rdev->family) == CHIP_R600) ||
1879             ((rdev->family) == CHIP_RV630) ||
1880             ((rdev->family) == CHIP_RV610) ||
1881             ((rdev->family) == CHIP_RV620) ||
1882             ((rdev->family) == CHIP_RS780) ||
1883             ((rdev->family) == CHIP_RS880)) {
1884                 WREG32(DB_DEBUG, PREZ_MUST_WAIT_FOR_POSTZ_DONE);
1885         } else {
1886                 WREG32(DB_DEBUG, 0);
1887         }
1888         WREG32(DB_WATERMARKS, (DEPTH_FREE(4) | DEPTH_CACHELINE_FREE(16) |
1889                                DEPTH_FLUSH(16) | DEPTH_PENDING_FREE(4)));
1890
1891         WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1892         WREG32(VGT_NUM_INSTANCES, 0);
1893
1894         WREG32(SPI_CONFIG_CNTL, GPR_WRITE_PRIORITY(0));
1895         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(0));
1896
1897         tmp = RREG32(SQ_MS_FIFO_SIZES);
1898         if (((rdev->family) == CHIP_RV610) ||
1899             ((rdev->family) == CHIP_RV620) ||
1900             ((rdev->family) == CHIP_RS780) ||
1901             ((rdev->family) == CHIP_RS880)) {
1902                 tmp = (CACHE_FIFO_SIZE(0xa) |
1903                        FETCH_FIFO_HIWATER(0xa) |
1904                        DONE_FIFO_HIWATER(0xe0) |
1905                        ALU_UPDATE_FIFO_HIWATER(0x8));
1906         } else if (((rdev->family) == CHIP_R600) ||
1907                    ((rdev->family) == CHIP_RV630)) {
1908                 tmp &= ~DONE_FIFO_HIWATER(0xff);
1909                 tmp |= DONE_FIFO_HIWATER(0x4);
1910         }
1911         WREG32(SQ_MS_FIFO_SIZES, tmp);
1912
1913         /* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
1914          * should be adjusted as needed by the 2D/3D drivers.  This just sets default values
1915          */
1916         sq_config = RREG32(SQ_CONFIG);
1917         sq_config &= ~(PS_PRIO(3) |
1918                        VS_PRIO(3) |
1919                        GS_PRIO(3) |
1920                        ES_PRIO(3));
1921         sq_config |= (DX9_CONSTS |
1922                       VC_ENABLE |
1923                       PS_PRIO(0) |
1924                       VS_PRIO(1) |
1925                       GS_PRIO(2) |
1926                       ES_PRIO(3));
1927
1928         if ((rdev->family) == CHIP_R600) {
1929                 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(124) |
1930                                           NUM_VS_GPRS(124) |
1931                                           NUM_CLAUSE_TEMP_GPRS(4));
1932                 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(0) |
1933                                           NUM_ES_GPRS(0));
1934                 sq_thread_resource_mgmt = (NUM_PS_THREADS(136) |
1935                                            NUM_VS_THREADS(48) |
1936                                            NUM_GS_THREADS(4) |
1937                                            NUM_ES_THREADS(4));
1938                 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(128) |
1939                                             NUM_VS_STACK_ENTRIES(128));
1940                 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(0) |
1941                                             NUM_ES_STACK_ENTRIES(0));
1942         } else if (((rdev->family) == CHIP_RV610) ||
1943                    ((rdev->family) == CHIP_RV620) ||
1944                    ((rdev->family) == CHIP_RS780) ||
1945                    ((rdev->family) == CHIP_RS880)) {
1946                 /* no vertex cache */
1947                 sq_config &= ~VC_ENABLE;
1948
1949                 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
1950                                           NUM_VS_GPRS(44) |
1951                                           NUM_CLAUSE_TEMP_GPRS(2));
1952                 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) |
1953                                           NUM_ES_GPRS(17));
1954                 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
1955                                            NUM_VS_THREADS(78) |
1956                                            NUM_GS_THREADS(4) |
1957                                            NUM_ES_THREADS(31));
1958                 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) |
1959                                             NUM_VS_STACK_ENTRIES(40));
1960                 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) |
1961                                             NUM_ES_STACK_ENTRIES(16));
1962         } else if (((rdev->family) == CHIP_RV630) ||
1963                    ((rdev->family) == CHIP_RV635)) {
1964                 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
1965                                           NUM_VS_GPRS(44) |
1966                                           NUM_CLAUSE_TEMP_GPRS(2));
1967                 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(18) |
1968                                           NUM_ES_GPRS(18));
1969                 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
1970                                            NUM_VS_THREADS(78) |
1971                                            NUM_GS_THREADS(4) |
1972                                            NUM_ES_THREADS(31));
1973                 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) |
1974                                             NUM_VS_STACK_ENTRIES(40));
1975                 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) |
1976                                             NUM_ES_STACK_ENTRIES(16));
1977         } else if ((rdev->family) == CHIP_RV670) {
1978                 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
1979                                           NUM_VS_GPRS(44) |
1980                                           NUM_CLAUSE_TEMP_GPRS(2));
1981                 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) |
1982                                           NUM_ES_GPRS(17));
1983                 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
1984                                            NUM_VS_THREADS(78) |
1985                                            NUM_GS_THREADS(4) |
1986                                            NUM_ES_THREADS(31));
1987                 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(64) |
1988                                             NUM_VS_STACK_ENTRIES(64));
1989                 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(64) |
1990                                             NUM_ES_STACK_ENTRIES(64));
1991         }
1992
1993         WREG32(SQ_CONFIG, sq_config);
1994         WREG32(SQ_GPR_RESOURCE_MGMT_1,  sq_gpr_resource_mgmt_1);
1995         WREG32(SQ_GPR_RESOURCE_MGMT_2,  sq_gpr_resource_mgmt_2);
1996         WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
1997         WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
1998         WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
1999
2000         if (((rdev->family) == CHIP_RV610) ||
2001             ((rdev->family) == CHIP_RV620) ||
2002             ((rdev->family) == CHIP_RS780) ||
2003             ((rdev->family) == CHIP_RS880)) {
2004                 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(TC_ONLY));
2005         } else {
2006                 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC));
2007         }
2008
2009         /* More default values. 2D/3D driver should adjust as needed */
2010         WREG32(PA_SC_AA_SAMPLE_LOCS_2S, (S0_X(0xc) | S0_Y(0x4) |
2011                                          S1_X(0x4) | S1_Y(0xc)));
2012         WREG32(PA_SC_AA_SAMPLE_LOCS_4S, (S0_X(0xe) | S0_Y(0xe) |
2013                                          S1_X(0x2) | S1_Y(0x2) |
2014                                          S2_X(0xa) | S2_Y(0x6) |
2015                                          S3_X(0x6) | S3_Y(0xa)));
2016         WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD0, (S0_X(0xe) | S0_Y(0xb) |
2017                                              S1_X(0x4) | S1_Y(0xc) |
2018                                              S2_X(0x1) | S2_Y(0x6) |
2019                                              S3_X(0xa) | S3_Y(0xe)));
2020         WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD1, (S4_X(0x6) | S4_Y(0x1) |
2021                                              S5_X(0x0) | S5_Y(0x0) |
2022                                              S6_X(0xb) | S6_Y(0x4) |
2023                                              S7_X(0x7) | S7_Y(0x8)));
2024
2025         WREG32(VGT_STRMOUT_EN, 0);
2026         tmp = rdev->config.r600.max_pipes * 16;
2027         switch (rdev->family) {
2028         case CHIP_RV610:
2029         case CHIP_RV620:
2030         case CHIP_RS780:
2031         case CHIP_RS880:
2032                 tmp += 32;
2033                 break;
2034         case CHIP_RV670:
2035                 tmp += 128;
2036                 break;
2037         default:
2038                 break;
2039         }
2040         if (tmp > 256) {
2041                 tmp = 256;
2042         }
2043         WREG32(VGT_ES_PER_GS, 128);
2044         WREG32(VGT_GS_PER_ES, tmp);
2045         WREG32(VGT_GS_PER_VS, 2);
2046         WREG32(VGT_GS_VERTEX_REUSE, 16);
2047
2048         /* more default values. 2D/3D driver should adjust as needed */
2049         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
2050         WREG32(VGT_STRMOUT_EN, 0);
2051         WREG32(SX_MISC, 0);
2052         WREG32(PA_SC_MODE_CNTL, 0);
2053         WREG32(PA_SC_AA_CONFIG, 0);
2054         WREG32(PA_SC_LINE_STIPPLE, 0);
2055         WREG32(SPI_INPUT_Z, 0);
2056         WREG32(SPI_PS_IN_CONTROL_0, NUM_INTERP(2));
2057         WREG32(CB_COLOR7_FRAG, 0);
2058
2059         /* Clear render buffer base addresses */
2060         WREG32(CB_COLOR0_BASE, 0);
2061         WREG32(CB_COLOR1_BASE, 0);
2062         WREG32(CB_COLOR2_BASE, 0);
2063         WREG32(CB_COLOR3_BASE, 0);
2064         WREG32(CB_COLOR4_BASE, 0);
2065         WREG32(CB_COLOR5_BASE, 0);
2066         WREG32(CB_COLOR6_BASE, 0);
2067         WREG32(CB_COLOR7_BASE, 0);
2068         WREG32(CB_COLOR7_FRAG, 0);
2069
2070         switch (rdev->family) {
2071         case CHIP_RV610:
2072         case CHIP_RV620:
2073         case CHIP_RS780:
2074         case CHIP_RS880:
2075                 tmp = TC_L2_SIZE(8);
2076                 break;
2077         case CHIP_RV630:
2078         case CHIP_RV635:
2079                 tmp = TC_L2_SIZE(4);
2080                 break;
2081         case CHIP_R600:
2082                 tmp = TC_L2_SIZE(0) | L2_DISABLE_LATE_HIT;
2083                 break;
2084         default:
2085                 tmp = TC_L2_SIZE(0);
2086                 break;
2087         }
2088         WREG32(TC_CNTL, tmp);
2089
2090         tmp = RREG32(HDP_HOST_PATH_CNTL);
2091         WREG32(HDP_HOST_PATH_CNTL, tmp);
2092
2093         tmp = RREG32(ARB_POP);
2094         tmp |= ENABLE_TC128;
2095         WREG32(ARB_POP, tmp);
2096
2097         WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
2098         WREG32(PA_CL_ENHANCE, (CLIP_VTX_REORDER_ENA |
2099                                NUM_CLIP_SEQ(3)));
2100         WREG32(PA_SC_ENHANCE, FORCE_EOV_MAX_CLK_CNT(4095));
2101         WREG32(VC_ENHANCE, 0);
2102 }
2103
2104
2105 /*
2106  * Indirect registers accessor
2107  */
2108 u32 r600_pciep_rreg(struct radeon_device *rdev, u32 reg)
2109 {
2110         u32 r;
2111
2112         WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
2113         (void)RREG32(PCIE_PORT_INDEX);
2114         r = RREG32(PCIE_PORT_DATA);
2115         return r;
2116 }
2117
2118 void r600_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
2119 {
2120         WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
2121         (void)RREG32(PCIE_PORT_INDEX);
2122         WREG32(PCIE_PORT_DATA, (v));
2123         (void)RREG32(PCIE_PORT_DATA);
2124 }
2125
2126 /*
2127  * CP & Ring
2128  */
2129 void r600_cp_stop(struct radeon_device *rdev)
2130 {
2131         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
2132         WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
2133         WREG32(SCRATCH_UMSK, 0);
2134         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
2135 }
2136
2137 int r600_init_microcode(struct radeon_device *rdev)
2138 {
2139         struct platform_device *pdev;
2140         const char *chip_name;
2141         const char *rlc_chip_name;
2142         size_t pfp_req_size, me_req_size, rlc_req_size;
2143         char fw_name[30];
2144         int err;
2145
2146         DRM_DEBUG("\n");
2147
2148         pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
2149         err = IS_ERR(pdev);
2150         if (err) {
2151                 printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
2152                 return -EINVAL;
2153         }
2154
2155         switch (rdev->family) {
2156         case CHIP_R600:
2157                 chip_name = "R600";
2158                 rlc_chip_name = "R600";
2159                 break;
2160         case CHIP_RV610:
2161                 chip_name = "RV610";
2162                 rlc_chip_name = "R600";
2163                 break;
2164         case CHIP_RV630:
2165                 chip_name = "RV630";
2166                 rlc_chip_name = "R600";
2167                 break;
2168         case CHIP_RV620:
2169                 chip_name = "RV620";
2170                 rlc_chip_name = "R600";
2171                 break;
2172         case CHIP_RV635:
2173                 chip_name = "RV635";
2174                 rlc_chip_name = "R600";
2175                 break;
2176         case CHIP_RV670:
2177                 chip_name = "RV670";
2178                 rlc_chip_name = "R600";
2179                 break;
2180         case CHIP_RS780:
2181         case CHIP_RS880:
2182                 chip_name = "RS780";
2183                 rlc_chip_name = "R600";
2184                 break;
2185         case CHIP_RV770:
2186                 chip_name = "RV770";
2187                 rlc_chip_name = "R700";
2188                 break;
2189         case CHIP_RV730:
2190         case CHIP_RV740:
2191                 chip_name = "RV730";
2192                 rlc_chip_name = "R700";
2193                 break;
2194         case CHIP_RV710:
2195                 chip_name = "RV710";
2196                 rlc_chip_name = "R700";
2197                 break;
2198         case CHIP_CEDAR:
2199                 chip_name = "CEDAR";
2200                 rlc_chip_name = "CEDAR";
2201                 break;
2202         case CHIP_REDWOOD:
2203                 chip_name = "REDWOOD";
2204                 rlc_chip_name = "REDWOOD";
2205                 break;
2206         case CHIP_JUNIPER:
2207                 chip_name = "JUNIPER";
2208                 rlc_chip_name = "JUNIPER";
2209                 break;
2210         case CHIP_CYPRESS:
2211         case CHIP_HEMLOCK:
2212                 chip_name = "CYPRESS";
2213                 rlc_chip_name = "CYPRESS";
2214                 break;
2215         case CHIP_PALM:
2216                 chip_name = "PALM";
2217                 rlc_chip_name = "SUMO";
2218                 break;
2219         case CHIP_SUMO:
2220                 chip_name = "SUMO";
2221                 rlc_chip_name = "SUMO";
2222                 break;
2223         case CHIP_SUMO2:
2224                 chip_name = "SUMO2";
2225                 rlc_chip_name = "SUMO";
2226                 break;
2227         default: BUG();
2228         }
2229
2230         if (rdev->family >= CHIP_CEDAR) {
2231                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
2232                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
2233                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
2234         } else if (rdev->family >= CHIP_RV770) {
2235                 pfp_req_size = R700_PFP_UCODE_SIZE * 4;
2236                 me_req_size = R700_PM4_UCODE_SIZE * 4;
2237                 rlc_req_size = R700_RLC_UCODE_SIZE * 4;
2238         } else {
2239                 pfp_req_size = R600_PFP_UCODE_SIZE * 4;
2240                 me_req_size = R600_PM4_UCODE_SIZE * 12;
2241                 rlc_req_size = R600_RLC_UCODE_SIZE * 4;
2242         }
2243
2244         DRM_INFO("Loading %s Microcode\n", chip_name);
2245
2246         snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
2247         err = request_firmware(&rdev->pfp_fw, fw_name, &pdev->dev);
2248         if (err)
2249                 goto out;
2250         if (rdev->pfp_fw->size != pfp_req_size) {
2251                 printk(KERN_ERR
2252                        "r600_cp: Bogus length %zu in firmware \"%s\"\n",
2253                        rdev->pfp_fw->size, fw_name);
2254                 err = -EINVAL;
2255                 goto out;
2256         }
2257
2258         snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
2259         err = request_firmware(&rdev->me_fw, fw_name, &pdev->dev);
2260         if (err)
2261                 goto out;
2262         if (rdev->me_fw->size != me_req_size) {
2263                 printk(KERN_ERR
2264                        "r600_cp: Bogus length %zu in firmware \"%s\"\n",
2265                        rdev->me_fw->size, fw_name);
2266                 err = -EINVAL;
2267         }
2268
2269         snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
2270         err = request_firmware(&rdev->rlc_fw, fw_name, &pdev->dev);
2271         if (err)
2272                 goto out;
2273         if (rdev->rlc_fw->size != rlc_req_size) {
2274                 printk(KERN_ERR
2275                        "r600_rlc: Bogus length %zu in firmware \"%s\"\n",
2276                        rdev->rlc_fw->size, fw_name);
2277                 err = -EINVAL;
2278         }
2279
2280 out:
2281         platform_device_unregister(pdev);
2282
2283         if (err) {
2284                 if (err != -EINVAL)
2285                         printk(KERN_ERR
2286                                "r600_cp: Failed to load firmware \"%s\"\n",
2287                                fw_name);
2288                 release_firmware(rdev->pfp_fw);
2289                 rdev->pfp_fw = NULL;
2290                 release_firmware(rdev->me_fw);
2291                 rdev->me_fw = NULL;
2292                 release_firmware(rdev->rlc_fw);
2293                 rdev->rlc_fw = NULL;
2294         }
2295         return err;
2296 }
2297
2298 static int r600_cp_load_microcode(struct radeon_device *rdev)
2299 {
2300         const __be32 *fw_data;
2301         int i;
2302
2303         if (!rdev->me_fw || !rdev->pfp_fw)
2304                 return -EINVAL;
2305
2306         r600_cp_stop(rdev);
2307
2308         WREG32(CP_RB_CNTL,
2309 #ifdef __BIG_ENDIAN
2310                BUF_SWAP_32BIT |
2311 #endif
2312                RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2313
2314         /* Reset cp */
2315         WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
2316         RREG32(GRBM_SOFT_RESET);
2317         mdelay(15);
2318         WREG32(GRBM_SOFT_RESET, 0);
2319
2320         WREG32(CP_ME_RAM_WADDR, 0);
2321
2322         fw_data = (const __be32 *)rdev->me_fw->data;
2323         WREG32(CP_ME_RAM_WADDR, 0);
2324         for (i = 0; i < R600_PM4_UCODE_SIZE * 3; i++)
2325                 WREG32(CP_ME_RAM_DATA,
2326                        be32_to_cpup(fw_data++));
2327
2328         fw_data = (const __be32 *)rdev->pfp_fw->data;
2329         WREG32(CP_PFP_UCODE_ADDR, 0);
2330         for (i = 0; i < R600_PFP_UCODE_SIZE; i++)
2331                 WREG32(CP_PFP_UCODE_DATA,
2332                        be32_to_cpup(fw_data++));
2333
2334         WREG32(CP_PFP_UCODE_ADDR, 0);
2335         WREG32(CP_ME_RAM_WADDR, 0);
2336         WREG32(CP_ME_RAM_RADDR, 0);
2337         return 0;
2338 }
2339
2340 int r600_cp_start(struct radeon_device *rdev)
2341 {
2342         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2343         int r;
2344         uint32_t cp_me;
2345
2346         r = radeon_ring_lock(rdev, ring, 7);
2347         if (r) {
2348                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2349                 return r;
2350         }
2351         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2352         radeon_ring_write(ring, 0x1);
2353         if (rdev->family >= CHIP_RV770) {
2354                 radeon_ring_write(ring, 0x0);
2355                 radeon_ring_write(ring, rdev->config.rv770.max_hw_contexts - 1);
2356         } else {
2357                 radeon_ring_write(ring, 0x3);
2358                 radeon_ring_write(ring, rdev->config.r600.max_hw_contexts - 1);
2359         }
2360         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2361         radeon_ring_write(ring, 0);
2362         radeon_ring_write(ring, 0);
2363         radeon_ring_unlock_commit(rdev, ring);
2364
2365         cp_me = 0xff;
2366         WREG32(R_0086D8_CP_ME_CNTL, cp_me);
2367         return 0;
2368 }
2369
2370 int r600_cp_resume(struct radeon_device *rdev)
2371 {
2372         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2373         u32 tmp;
2374         u32 rb_bufsz;
2375         int r;
2376
2377         /* Reset cp */
2378         WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
2379         RREG32(GRBM_SOFT_RESET);
2380         mdelay(15);
2381         WREG32(GRBM_SOFT_RESET, 0);
2382
2383         /* Set ring buffer size */
2384         rb_bufsz = drm_order(ring->ring_size / 8);
2385         tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2386 #ifdef __BIG_ENDIAN
2387         tmp |= BUF_SWAP_32BIT;
2388 #endif
2389         WREG32(CP_RB_CNTL, tmp);
2390         WREG32(CP_SEM_WAIT_TIMER, 0x0);
2391
2392         /* Set the write pointer delay */
2393         WREG32(CP_RB_WPTR_DELAY, 0);
2394
2395         /* Initialize the ring buffer's read and write pointers */
2396         WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2397         WREG32(CP_RB_RPTR_WR, 0);
2398         ring->wptr = 0;
2399         WREG32(CP_RB_WPTR, ring->wptr);
2400
2401         /* set the wb address whether it's enabled or not */
2402         WREG32(CP_RB_RPTR_ADDR,
2403                ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
2404         WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2405         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2406
2407         if (rdev->wb.enabled)
2408                 WREG32(SCRATCH_UMSK, 0xff);
2409         else {
2410                 tmp |= RB_NO_UPDATE;
2411                 WREG32(SCRATCH_UMSK, 0);
2412         }
2413
2414         mdelay(1);
2415         WREG32(CP_RB_CNTL, tmp);
2416
2417         WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
2418         WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
2419
2420         ring->rptr = RREG32(CP_RB_RPTR);
2421
2422         r600_cp_start(rdev);
2423         ring->ready = true;
2424         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
2425         if (r) {
2426                 ring->ready = false;
2427                 return r;
2428         }
2429         return 0;
2430 }
2431
2432 void r600_ring_init(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ring_size)
2433 {
2434         u32 rb_bufsz;
2435         int r;
2436
2437         /* Align ring size */
2438         rb_bufsz = drm_order(ring_size / 8);
2439         ring_size = (1 << (rb_bufsz + 1)) * 4;
2440         ring->ring_size = ring_size;
2441         ring->align_mask = 16 - 1;
2442
2443         if (radeon_ring_supports_scratch_reg(rdev, ring)) {
2444                 r = radeon_scratch_get(rdev, &ring->rptr_save_reg);
2445                 if (r) {
2446                         DRM_ERROR("failed to get scratch reg for rptr save (%d).\n", r);
2447                         ring->rptr_save_reg = 0;
2448                 }
2449         }
2450 }
2451
2452 void r600_cp_fini(struct radeon_device *rdev)
2453 {
2454         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2455         r600_cp_stop(rdev);
2456         radeon_ring_fini(rdev, ring);
2457         radeon_scratch_free(rdev, ring->rptr_save_reg);
2458 }
2459
2460 /*
2461  * DMA
2462  * Starting with R600, the GPU has an asynchronous
2463  * DMA engine.  The programming model is very similar
2464  * to the 3D engine (ring buffer, IBs, etc.), but the
2465  * DMA controller has it's own packet format that is
2466  * different form the PM4 format used by the 3D engine.
2467  * It supports copying data, writing embedded data,
2468  * solid fills, and a number of other things.  It also
2469  * has support for tiling/detiling of buffers.
2470  */
2471 /**
2472  * r600_dma_stop - stop the async dma engine
2473  *
2474  * @rdev: radeon_device pointer
2475  *
2476  * Stop the async dma engine (r6xx-evergreen).
2477  */
2478 void r600_dma_stop(struct radeon_device *rdev)
2479 {
2480         u32 rb_cntl = RREG32(DMA_RB_CNTL);
2481
2482         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
2483
2484         rb_cntl &= ~DMA_RB_ENABLE;
2485         WREG32(DMA_RB_CNTL, rb_cntl);
2486
2487         rdev->ring[R600_RING_TYPE_DMA_INDEX].ready = false;
2488 }
2489
2490 /**
2491  * r600_dma_resume - setup and start the async dma engine
2492  *
2493  * @rdev: radeon_device pointer
2494  *
2495  * Set up the DMA ring buffer and enable it. (r6xx-evergreen).
2496  * Returns 0 for success, error for failure.
2497  */
2498 int r600_dma_resume(struct radeon_device *rdev)
2499 {
2500         struct radeon_ring *ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2501         u32 rb_cntl, dma_cntl, ib_cntl;
2502         u32 rb_bufsz;
2503         int r;
2504
2505         /* Reset dma */
2506         if (rdev->family >= CHIP_RV770)
2507                 WREG32(SRBM_SOFT_RESET, RV770_SOFT_RESET_DMA);
2508         else
2509                 WREG32(SRBM_SOFT_RESET, SOFT_RESET_DMA);
2510         RREG32(SRBM_SOFT_RESET);
2511         udelay(50);
2512         WREG32(SRBM_SOFT_RESET, 0);
2513
2514         WREG32(DMA_SEM_INCOMPLETE_TIMER_CNTL, 0);
2515         WREG32(DMA_SEM_WAIT_FAIL_TIMER_CNTL, 0);
2516
2517         /* Set ring buffer size in dwords */
2518         rb_bufsz = drm_order(ring->ring_size / 4);
2519         rb_cntl = rb_bufsz << 1;
2520 #ifdef __BIG_ENDIAN
2521         rb_cntl |= DMA_RB_SWAP_ENABLE | DMA_RPTR_WRITEBACK_SWAP_ENABLE;
2522 #endif
2523         WREG32(DMA_RB_CNTL, rb_cntl);
2524
2525         /* Initialize the ring buffer's read and write pointers */
2526         WREG32(DMA_RB_RPTR, 0);
2527         WREG32(DMA_RB_WPTR, 0);
2528
2529         /* set the wb address whether it's enabled or not */
2530         WREG32(DMA_RB_RPTR_ADDR_HI,
2531                upper_32_bits(rdev->wb.gpu_addr + R600_WB_DMA_RPTR_OFFSET) & 0xFF);
2532         WREG32(DMA_RB_RPTR_ADDR_LO,
2533                ((rdev->wb.gpu_addr + R600_WB_DMA_RPTR_OFFSET) & 0xFFFFFFFC));
2534
2535         if (rdev->wb.enabled)
2536                 rb_cntl |= DMA_RPTR_WRITEBACK_ENABLE;
2537
2538         WREG32(DMA_RB_BASE, ring->gpu_addr >> 8);
2539
2540         /* enable DMA IBs */
2541         ib_cntl = DMA_IB_ENABLE;
2542 #ifdef __BIG_ENDIAN
2543         ib_cntl |= DMA_IB_SWAP_ENABLE;
2544 #endif
2545         WREG32(DMA_IB_CNTL, ib_cntl);
2546
2547         dma_cntl = RREG32(DMA_CNTL);
2548         dma_cntl &= ~CTXEMPTY_INT_ENABLE;
2549         WREG32(DMA_CNTL, dma_cntl);
2550
2551         if (rdev->family >= CHIP_RV770)
2552                 WREG32(DMA_MODE, 1);
2553
2554         ring->wptr = 0;
2555         WREG32(DMA_RB_WPTR, ring->wptr << 2);
2556
2557         ring->rptr = RREG32(DMA_RB_RPTR) >> 2;
2558
2559         WREG32(DMA_RB_CNTL, rb_cntl | DMA_RB_ENABLE);
2560
2561         ring->ready = true;
2562
2563         r = radeon_ring_test(rdev, R600_RING_TYPE_DMA_INDEX, ring);
2564         if (r) {
2565                 ring->ready = false;
2566                 return r;
2567         }
2568
2569         radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
2570
2571         return 0;
2572 }
2573
2574 /**
2575  * r600_dma_fini - tear down the async dma engine
2576  *
2577  * @rdev: radeon_device pointer
2578  *
2579  * Stop the async dma engine and free the ring (r6xx-evergreen).
2580  */
2581 void r600_dma_fini(struct radeon_device *rdev)
2582 {
2583         r600_dma_stop(rdev);
2584         radeon_ring_fini(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX]);
2585 }
2586
2587 /*
2588  * UVD
2589  */
2590 int r600_uvd_rbc_start(struct radeon_device *rdev)
2591 {
2592         struct radeon_ring *ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2593         uint64_t rptr_addr;
2594         uint32_t rb_bufsz, tmp;
2595         int r;
2596
2597         rptr_addr = rdev->wb.gpu_addr + R600_WB_UVD_RPTR_OFFSET;
2598
2599         if (upper_32_bits(rptr_addr) != upper_32_bits(ring->gpu_addr)) {
2600                 DRM_ERROR("UVD ring and rptr not in the same 4GB segment!\n");
2601                 return -EINVAL;
2602         }
2603
2604         /* force RBC into idle state */
2605         WREG32(UVD_RBC_RB_CNTL, 0x11010101);
2606
2607         /* Set the write pointer delay */
2608         WREG32(UVD_RBC_RB_WPTR_CNTL, 0);
2609
2610         /* set the wb address */
2611         WREG32(UVD_RBC_RB_RPTR_ADDR, rptr_addr >> 2);
2612
2613         /* programm the 4GB memory segment for rptr and ring buffer */
2614         WREG32(UVD_LMI_EXT40_ADDR, upper_32_bits(rptr_addr) |
2615                                    (0x7 << 16) | (0x1 << 31));
2616
2617         /* Initialize the ring buffer's read and write pointers */
2618         WREG32(UVD_RBC_RB_RPTR, 0x0);
2619
2620         ring->wptr = ring->rptr = RREG32(UVD_RBC_RB_RPTR);
2621         WREG32(UVD_RBC_RB_WPTR, ring->wptr);
2622
2623         /* set the ring address */
2624         WREG32(UVD_RBC_RB_BASE, ring->gpu_addr);
2625
2626         /* Set ring buffer size */
2627         rb_bufsz = drm_order(ring->ring_size);
2628         rb_bufsz = (0x1 << 8) | rb_bufsz;
2629         WREG32(UVD_RBC_RB_CNTL, rb_bufsz);
2630
2631         ring->ready = true;
2632         r = radeon_ring_test(rdev, R600_RING_TYPE_UVD_INDEX, ring);
2633         if (r) {
2634                 ring->ready = false;
2635                 return r;
2636         }
2637
2638         r = radeon_ring_lock(rdev, ring, 10);
2639         if (r) {
2640                 DRM_ERROR("radeon: ring failed to lock UVD ring (%d).\n", r);
2641                 return r;
2642         }
2643
2644         tmp = PACKET0(UVD_SEMA_WAIT_FAULT_TIMEOUT_CNTL, 0);
2645         radeon_ring_write(ring, tmp);
2646         radeon_ring_write(ring, 0xFFFFF);
2647
2648         tmp = PACKET0(UVD_SEMA_WAIT_INCOMPLETE_TIMEOUT_CNTL, 0);
2649         radeon_ring_write(ring, tmp);
2650         radeon_ring_write(ring, 0xFFFFF);
2651
2652         tmp = PACKET0(UVD_SEMA_SIGNAL_INCOMPLETE_TIMEOUT_CNTL, 0);
2653         radeon_ring_write(ring, tmp);
2654         radeon_ring_write(ring, 0xFFFFF);
2655
2656         /* Clear timeout status bits */
2657         radeon_ring_write(ring, PACKET0(UVD_SEMA_TIMEOUT_STATUS, 0));
2658         radeon_ring_write(ring, 0x8);
2659
2660         radeon_ring_write(ring, PACKET0(UVD_SEMA_CNTL, 0));
2661         radeon_ring_write(ring, 3);
2662
2663         radeon_ring_unlock_commit(rdev, ring);
2664
2665         return 0;
2666 }
2667
2668 void r600_uvd_rbc_stop(struct radeon_device *rdev)
2669 {
2670         struct radeon_ring *ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2671
2672         /* force RBC into idle state */
2673         WREG32(UVD_RBC_RB_CNTL, 0x11010101);
2674         ring->ready = false;
2675 }
2676
2677 int r600_uvd_init(struct radeon_device *rdev)
2678 {
2679         int i, j, r;
2680         /* disable byte swapping */
2681         u32 lmi_swap_cntl = 0;
2682         u32 mp_swap_cntl = 0;
2683
2684         /* raise clocks while booting up the VCPU */
2685         radeon_set_uvd_clocks(rdev, 53300, 40000);
2686
2687         /* disable clock gating */
2688         WREG32(UVD_CGC_GATE, 0);
2689
2690         /* disable interupt */
2691         WREG32_P(UVD_MASTINT_EN, 0, ~(1 << 1));
2692
2693         /* put LMI, VCPU, RBC etc... into reset */
2694         WREG32(UVD_SOFT_RESET, LMI_SOFT_RESET | VCPU_SOFT_RESET |
2695                LBSI_SOFT_RESET | RBC_SOFT_RESET | CSM_SOFT_RESET |
2696                CXW_SOFT_RESET | TAP_SOFT_RESET | LMI_UMC_SOFT_RESET);
2697         mdelay(5);
2698
2699         /* take UVD block out of reset */
2700         WREG32_P(SRBM_SOFT_RESET, 0, ~SOFT_RESET_UVD);
2701         mdelay(5);
2702
2703         /* initialize UVD memory controller */
2704         WREG32(UVD_LMI_CTRL, 0x40 | (1 << 8) | (1 << 13) |
2705                              (1 << 21) | (1 << 9) | (1 << 20));
2706
2707 #ifdef __BIG_ENDIAN
2708         /* swap (8 in 32) RB and IB */
2709         lmi_swap_cntl = 0xa;
2710         mp_swap_cntl = 0;
2711 #endif
2712         WREG32(UVD_LMI_SWAP_CNTL, lmi_swap_cntl);
2713         WREG32(UVD_MP_SWAP_CNTL, mp_swap_cntl);
2714
2715         WREG32(UVD_MPC_SET_MUXA0, 0x40c2040);
2716         WREG32(UVD_MPC_SET_MUXA1, 0x0);
2717         WREG32(UVD_MPC_SET_MUXB0, 0x40c2040);
2718         WREG32(UVD_MPC_SET_MUXB1, 0x0);
2719         WREG32(UVD_MPC_SET_ALU, 0);
2720         WREG32(UVD_MPC_SET_MUX, 0x88);
2721
2722         /* Stall UMC */
2723         WREG32_P(UVD_LMI_CTRL2, 1 << 8, ~(1 << 8));
2724         WREG32_P(UVD_RB_ARB_CTRL, 1 << 3, ~(1 << 3));
2725
2726         /* take all subblocks out of reset, except VCPU */
2727         WREG32(UVD_SOFT_RESET, VCPU_SOFT_RESET);
2728         mdelay(5);
2729
2730         /* enable VCPU clock */
2731         WREG32(UVD_VCPU_CNTL,  1 << 9);
2732
2733         /* enable UMC */
2734         WREG32_P(UVD_LMI_CTRL2, 0, ~(1 << 8));
2735
2736         /* boot up the VCPU */
2737         WREG32(UVD_SOFT_RESET, 0);
2738         mdelay(10);
2739
2740         WREG32_P(UVD_RB_ARB_CTRL, 0, ~(1 << 3));
2741
2742         for (i = 0; i < 10; ++i) {
2743                 uint32_t status;
2744                 for (j = 0; j < 100; ++j) {
2745                         status = RREG32(UVD_STATUS);
2746                         if (status & 2)
2747                                 break;
2748                         mdelay(10);
2749                 }
2750                 r = 0;
2751                 if (status & 2)
2752                         break;
2753
2754                 DRM_ERROR("UVD not responding, trying to reset the VCPU!!!\n");
2755                 WREG32_P(UVD_SOFT_RESET, VCPU_SOFT_RESET, ~VCPU_SOFT_RESET);
2756                 mdelay(10);
2757                 WREG32_P(UVD_SOFT_RESET, 0, ~VCPU_SOFT_RESET);
2758                 mdelay(10);
2759                 r = -1;
2760         }
2761
2762         if (r) {
2763                 DRM_ERROR("UVD not responding, giving up!!!\n");
2764                 radeon_set_uvd_clocks(rdev, 0, 0);
2765                 return r;
2766         }
2767
2768         /* enable interupt */
2769         WREG32_P(UVD_MASTINT_EN, 3<<1, ~(3 << 1));
2770
2771         r = r600_uvd_rbc_start(rdev);
2772         if (!r)
2773                 DRM_INFO("UVD initialized successfully.\n");
2774
2775         /* lower clocks again */
2776         radeon_set_uvd_clocks(rdev, 0, 0);
2777
2778         return r;
2779 }
2780
2781 /*
2782  * GPU scratch registers helpers function.
2783  */
2784 void r600_scratch_init(struct radeon_device *rdev)
2785 {
2786         int i;
2787
2788         rdev->scratch.num_reg = 7;
2789         rdev->scratch.reg_base = SCRATCH_REG0;
2790         for (i = 0; i < rdev->scratch.num_reg; i++) {
2791                 rdev->scratch.free[i] = true;
2792                 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
2793         }
2794 }
2795
2796 int r600_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
2797 {
2798         uint32_t scratch;
2799         uint32_t tmp = 0;
2800         unsigned i;
2801         int r;
2802
2803         r = radeon_scratch_get(rdev, &scratch);
2804         if (r) {
2805                 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
2806                 return r;
2807         }
2808         WREG32(scratch, 0xCAFEDEAD);
2809         r = radeon_ring_lock(rdev, ring, 3);
2810         if (r) {
2811                 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
2812                 radeon_scratch_free(rdev, scratch);
2813                 return r;
2814         }
2815         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2816         radeon_ring_write(ring, ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
2817         radeon_ring_write(ring, 0xDEADBEEF);
2818         radeon_ring_unlock_commit(rdev, ring);
2819         for (i = 0; i < rdev->usec_timeout; i++) {
2820                 tmp = RREG32(scratch);
2821                 if (tmp == 0xDEADBEEF)
2822                         break;
2823                 DRM_UDELAY(1);
2824         }
2825         if (i < rdev->usec_timeout) {
2826                 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
2827         } else {
2828                 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
2829                           ring->idx, scratch, tmp);
2830                 r = -EINVAL;
2831         }
2832         radeon_scratch_free(rdev, scratch);
2833         return r;
2834 }
2835
2836 /**
2837  * r600_dma_ring_test - simple async dma engine test
2838  *
2839  * @rdev: radeon_device pointer
2840  * @ring: radeon_ring structure holding ring information
2841  *
2842  * Test the DMA engine by writing using it to write an
2843  * value to memory. (r6xx-SI).
2844  * Returns 0 for success, error for failure.
2845  */
2846 int r600_dma_ring_test(struct radeon_device *rdev,
2847                        struct radeon_ring *ring)
2848 {
2849         unsigned i;
2850         int r;
2851         void __iomem *ptr = (void *)rdev->vram_scratch.ptr;
2852         u32 tmp;
2853
2854         if (!ptr) {
2855                 DRM_ERROR("invalid vram scratch pointer\n");
2856                 return -EINVAL;
2857         }
2858
2859         tmp = 0xCAFEDEAD;
2860         writel(tmp, ptr);
2861
2862         r = radeon_ring_lock(rdev, ring, 4);
2863         if (r) {
2864                 DRM_ERROR("radeon: dma failed to lock ring %d (%d).\n", ring->idx, r);
2865                 return r;
2866         }
2867         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1));
2868         radeon_ring_write(ring, rdev->vram_scratch.gpu_addr & 0xfffffffc);
2869         radeon_ring_write(ring, upper_32_bits(rdev->vram_scratch.gpu_addr) & 0xff);
2870         radeon_ring_write(ring, 0xDEADBEEF);
2871         radeon_ring_unlock_commit(rdev, ring);
2872
2873         for (i = 0; i < rdev->usec_timeout; i++) {
2874                 tmp = readl(ptr);
2875                 if (tmp == 0xDEADBEEF)
2876                         break;
2877                 DRM_UDELAY(1);
2878         }
2879
2880         if (i < rdev->usec_timeout) {
2881                 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
2882         } else {
2883                 DRM_ERROR("radeon: ring %d test failed (0x%08X)\n",
2884                           ring->idx, tmp);
2885                 r = -EINVAL;
2886         }
2887         return r;
2888 }
2889
2890 int r600_uvd_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
2891 {
2892         uint32_t tmp = 0;
2893         unsigned i;
2894         int r;
2895
2896         WREG32(UVD_CONTEXT_ID, 0xCAFEDEAD);
2897         r = radeon_ring_lock(rdev, ring, 3);
2898         if (r) {
2899                 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n",
2900                           ring->idx, r);
2901                 return r;
2902         }
2903         radeon_ring_write(ring, PACKET0(UVD_CONTEXT_ID, 0));
2904         radeon_ring_write(ring, 0xDEADBEEF);
2905         radeon_ring_unlock_commit(rdev, ring);
2906         for (i = 0; i < rdev->usec_timeout; i++) {
2907                 tmp = RREG32(UVD_CONTEXT_ID);
2908                 if (tmp == 0xDEADBEEF)
2909                         break;
2910                 DRM_UDELAY(1);
2911         }
2912
2913         if (i < rdev->usec_timeout) {
2914                 DRM_INFO("ring test on %d succeeded in %d usecs\n",
2915                          ring->idx, i);
2916         } else {
2917                 DRM_ERROR("radeon: ring %d test failed (0x%08X)\n",
2918                           ring->idx, tmp);
2919                 r = -EINVAL;
2920         }
2921         return r;
2922 }
2923
2924 /*
2925  * CP fences/semaphores
2926  */
2927
2928 void r600_fence_ring_emit(struct radeon_device *rdev,
2929                           struct radeon_fence *fence)
2930 {
2931         struct radeon_ring *ring = &rdev->ring[fence->ring];
2932
2933         if (rdev->wb.use_event) {
2934                 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
2935                 /* flush read cache over gart */
2936                 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
2937                 radeon_ring_write(ring, PACKET3_TC_ACTION_ENA |
2938                                         PACKET3_VC_ACTION_ENA |
2939                                         PACKET3_SH_ACTION_ENA);
2940                 radeon_ring_write(ring, 0xFFFFFFFF);
2941                 radeon_ring_write(ring, 0);
2942                 radeon_ring_write(ring, 10); /* poll interval */
2943                 /* EVENT_WRITE_EOP - flush caches, send int */
2944                 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
2945                 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
2946                 radeon_ring_write(ring, addr & 0xffffffff);
2947                 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
2948                 radeon_ring_write(ring, fence->seq);
2949                 radeon_ring_write(ring, 0);
2950         } else {
2951                 /* flush read cache over gart */
2952                 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
2953                 radeon_ring_write(ring, PACKET3_TC_ACTION_ENA |
2954                                         PACKET3_VC_ACTION_ENA |
2955                                         PACKET3_SH_ACTION_ENA);
2956                 radeon_ring_write(ring, 0xFFFFFFFF);
2957                 radeon_ring_write(ring, 0);
2958                 radeon_ring_write(ring, 10); /* poll interval */
2959                 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
2960                 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT) | EVENT_INDEX(0));
2961                 /* wait for 3D idle clean */
2962                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2963                 radeon_ring_write(ring, (WAIT_UNTIL - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
2964                 radeon_ring_write(ring, WAIT_3D_IDLE_bit | WAIT_3D_IDLECLEAN_bit);
2965                 /* Emit fence sequence & fire IRQ */
2966                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2967                 radeon_ring_write(ring, ((rdev->fence_drv[fence->ring].scratch_reg - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
2968                 radeon_ring_write(ring, fence->seq);
2969                 /* CP_INTERRUPT packet 3 no longer exists, use packet 0 */
2970                 radeon_ring_write(ring, PACKET0(CP_INT_STATUS, 0));
2971                 radeon_ring_write(ring, RB_INT_STAT);
2972         }
2973 }
2974
2975 void r600_uvd_fence_emit(struct radeon_device *rdev,
2976                          struct radeon_fence *fence)
2977 {
2978         struct radeon_ring *ring = &rdev->ring[fence->ring];
2979         uint32_t addr = rdev->fence_drv[fence->ring].gpu_addr;
2980
2981         radeon_ring_write(ring, PACKET0(UVD_CONTEXT_ID, 0));
2982         radeon_ring_write(ring, fence->seq);
2983         radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_DATA0, 0));
2984         radeon_ring_write(ring, addr & 0xffffffff);
2985         radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_DATA1, 0));
2986         radeon_ring_write(ring, upper_32_bits(addr) & 0xff);
2987         radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_CMD, 0));
2988         radeon_ring_write(ring, 0);
2989
2990         radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_DATA0, 0));
2991         radeon_ring_write(ring, 0);
2992         radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_DATA1, 0));
2993         radeon_ring_write(ring, 0);
2994         radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_CMD, 0));
2995         radeon_ring_write(ring, 2);
2996         return;
2997 }
2998
2999 void r600_semaphore_ring_emit(struct radeon_device *rdev,
3000                               struct radeon_ring *ring,
3001                               struct radeon_semaphore *semaphore,
3002                               bool emit_wait)
3003 {
3004         uint64_t addr = semaphore->gpu_addr;
3005         unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3006
3007         if (rdev->family < CHIP_CAYMAN)
3008                 sel |= PACKET3_SEM_WAIT_ON_SIGNAL;
3009
3010         radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3011         radeon_ring_write(ring, addr & 0xffffffff);
3012         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | sel);
3013 }
3014
3015 /*
3016  * DMA fences/semaphores
3017  */
3018
3019 /**
3020  * r600_dma_fence_ring_emit - emit a fence on the DMA ring
3021  *
3022  * @rdev: radeon_device pointer
3023  * @fence: radeon fence object
3024  *
3025  * Add a DMA fence packet to the ring to write
3026  * the fence seq number and DMA trap packet to generate
3027  * an interrupt if needed (r6xx-r7xx).
3028  */
3029 void r600_dma_fence_ring_emit(struct radeon_device *rdev,
3030                               struct radeon_fence *fence)
3031 {
3032         struct radeon_ring *ring = &rdev->ring[fence->ring];
3033         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3034
3035         /* write the fence */
3036         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0, 0));
3037         radeon_ring_write(ring, addr & 0xfffffffc);
3038         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff));
3039         radeon_ring_write(ring, lower_32_bits(fence->seq));
3040         /* generate an interrupt */
3041         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_TRAP, 0, 0, 0));
3042 }
3043
3044 /**
3045  * r600_dma_semaphore_ring_emit - emit a semaphore on the dma ring
3046  *
3047  * @rdev: radeon_device pointer
3048  * @ring: radeon_ring structure holding ring information
3049  * @semaphore: radeon semaphore object
3050  * @emit_wait: wait or signal semaphore
3051  *
3052  * Add a DMA semaphore packet to the ring wait on or signal
3053  * other rings (r6xx-SI).
3054  */
3055 void r600_dma_semaphore_ring_emit(struct radeon_device *rdev,
3056                                   struct radeon_ring *ring,
3057                                   struct radeon_semaphore *semaphore,
3058                                   bool emit_wait)
3059 {
3060         u64 addr = semaphore->gpu_addr;
3061         u32 s = emit_wait ? 0 : 1;
3062
3063         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SEMAPHORE, 0, s, 0));
3064         radeon_ring_write(ring, addr & 0xfffffffc);
3065         radeon_ring_write(ring, upper_32_bits(addr) & 0xff);
3066 }
3067
3068 void r600_uvd_semaphore_emit(struct radeon_device *rdev,
3069                              struct radeon_ring *ring,
3070                              struct radeon_semaphore *semaphore,
3071                              bool emit_wait)
3072 {
3073         uint64_t addr = semaphore->gpu_addr;
3074
3075         radeon_ring_write(ring, PACKET0(UVD_SEMA_ADDR_LOW, 0));
3076         radeon_ring_write(ring, (addr >> 3) & 0x000FFFFF);
3077
3078         radeon_ring_write(ring, PACKET0(UVD_SEMA_ADDR_HIGH, 0));
3079         radeon_ring_write(ring, (addr >> 23) & 0x000FFFFF);
3080
3081         radeon_ring_write(ring, PACKET0(UVD_SEMA_CMD, 0));
3082         radeon_ring_write(ring, emit_wait ? 1 : 0);
3083 }
3084
3085 int r600_copy_blit(struct radeon_device *rdev,
3086                    uint64_t src_offset,
3087                    uint64_t dst_offset,
3088                    unsigned num_gpu_pages,
3089                    struct radeon_fence **fence)
3090 {
3091         struct radeon_semaphore *sem = NULL;
3092         struct radeon_sa_bo *vb = NULL;
3093         int r;
3094
3095         r = r600_blit_prepare_copy(rdev, num_gpu_pages, fence, &vb, &sem);
3096         if (r) {
3097                 return r;
3098         }
3099         r600_kms_blit_copy(rdev, src_offset, dst_offset, num_gpu_pages, vb);
3100         r600_blit_done_copy(rdev, fence, vb, sem);
3101         return 0;
3102 }
3103
3104 /**
3105  * r600_copy_dma - copy pages using the DMA engine
3106  *
3107  * @rdev: radeon_device pointer
3108  * @src_offset: src GPU address
3109  * @dst_offset: dst GPU address
3110  * @num_gpu_pages: number of GPU pages to xfer
3111  * @fence: radeon fence object
3112  *
3113  * Copy GPU paging using the DMA engine (r6xx).
3114  * Used by the radeon ttm implementation to move pages if
3115  * registered as the asic copy callback.
3116  */
3117 int r600_copy_dma(struct radeon_device *rdev,
3118                   uint64_t src_offset, uint64_t dst_offset,
3119                   unsigned num_gpu_pages,
3120                   struct radeon_fence **fence)
3121 {
3122         struct radeon_semaphore *sem = NULL;
3123         int ring_index = rdev->asic->copy.dma_ring_index;
3124         struct radeon_ring *ring = &rdev->ring[ring_index];
3125         u32 size_in_dw, cur_size_in_dw;
3126         int i, num_loops;
3127         int r = 0;
3128
3129         r = radeon_semaphore_create(rdev, &sem);
3130         if (r) {
3131                 DRM_ERROR("radeon: moving bo (%d).\n", r);
3132                 return r;
3133         }
3134
3135         size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4;
3136         num_loops = DIV_ROUND_UP(size_in_dw, 0xFFFE);
3137         r = radeon_ring_lock(rdev, ring, num_loops * 4 + 8);
3138         if (r) {
3139                 DRM_ERROR("radeon: moving bo (%d).\n", r);
3140                 radeon_semaphore_free(rdev, &sem, NULL);
3141                 return r;
3142         }
3143
3144         if (radeon_fence_need_sync(*fence, ring->idx)) {
3145                 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
3146                                             ring->idx);
3147                 radeon_fence_note_sync(*fence, ring->idx);
3148         } else {
3149                 radeon_semaphore_free(rdev, &sem, NULL);
3150         }
3151
3152         for (i = 0; i < num_loops; i++) {
3153                 cur_size_in_dw = size_in_dw;
3154                 if (cur_size_in_dw > 0xFFFE)
3155                         cur_size_in_dw = 0xFFFE;
3156                 size_in_dw -= cur_size_in_dw;
3157                 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 0, 0, cur_size_in_dw));
3158                 radeon_ring_write(ring, dst_offset & 0xfffffffc);
3159                 radeon_ring_write(ring, src_offset & 0xfffffffc);
3160                 radeon_ring_write(ring, (((upper_32_bits(dst_offset) & 0xff) << 16) |
3161                                          (upper_32_bits(src_offset) & 0xff)));
3162                 src_offset += cur_size_in_dw * 4;
3163                 dst_offset += cur_size_in_dw * 4;
3164         }
3165
3166         r = radeon_fence_emit(rdev, fence, ring->idx);
3167         if (r) {
3168                 radeon_ring_unlock_undo(rdev, ring);
3169                 return r;
3170         }
3171
3172         radeon_ring_unlock_commit(rdev, ring);
3173         radeon_semaphore_free(rdev, &sem, *fence);
3174
3175         return r;
3176 }
3177
3178 int r600_set_surface_reg(struct radeon_device *rdev, int reg,
3179                          uint32_t tiling_flags, uint32_t pitch,
3180                          uint32_t offset, uint32_t obj_size)
3181 {
3182         /* FIXME: implement */
3183         return 0;
3184 }
3185
3186 void r600_clear_surface_reg(struct radeon_device *rdev, int reg)
3187 {
3188         /* FIXME: implement */
3189 }
3190
3191 static int r600_startup(struct radeon_device *rdev)
3192 {
3193         struct radeon_ring *ring;
3194         int r;
3195
3196         /* enable pcie gen2 link */
3197         r600_pcie_gen2_enable(rdev);
3198
3199         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
3200                 r = r600_init_microcode(rdev);
3201                 if (r) {
3202                         DRM_ERROR("Failed to load firmware!\n");
3203                         return r;
3204                 }
3205         }
3206
3207         r = r600_vram_scratch_init(rdev);
3208         if (r)
3209                 return r;
3210
3211         r600_mc_program(rdev);
3212         if (rdev->flags & RADEON_IS_AGP) {
3213                 r600_agp_enable(rdev);
3214         } else {
3215                 r = r600_pcie_gart_enable(rdev);
3216                 if (r)
3217                         return r;
3218         }
3219         r600_gpu_init(rdev);
3220         r = r600_blit_init(rdev);
3221         if (r) {
3222                 r600_blit_fini(rdev);
3223                 rdev->asic->copy.copy = NULL;
3224                 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
3225         }
3226
3227         /* allocate wb buffer */
3228         r = radeon_wb_init(rdev);
3229         if (r)
3230                 return r;
3231
3232         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
3233         if (r) {
3234                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
3235                 return r;
3236         }
3237
3238         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
3239         if (r) {
3240                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
3241                 return r;
3242         }
3243
3244         /* Enable IRQ */
3245         if (!rdev->irq.installed) {
3246                 r = radeon_irq_kms_init(rdev);
3247                 if (r)
3248                         return r;
3249         }
3250
3251         r = r600_irq_init(rdev);
3252         if (r) {
3253                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
3254                 radeon_irq_kms_fini(rdev);
3255                 return r;
3256         }
3257         r600_irq_set(rdev);
3258
3259         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3260         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
3261                              R600_CP_RB_RPTR, R600_CP_RB_WPTR,
3262                              0, 0xfffff, RADEON_CP_PACKET2);
3263         if (r)
3264                 return r;
3265
3266         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
3267         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
3268                              DMA_RB_RPTR, DMA_RB_WPTR,
3269                              2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
3270         if (r)
3271                 return r;
3272
3273         r = r600_cp_load_microcode(rdev);
3274         if (r)
3275                 return r;
3276         r = r600_cp_resume(rdev);
3277         if (r)
3278                 return r;
3279
3280         r = r600_dma_resume(rdev);
3281         if (r)
3282                 return r;
3283
3284         r = radeon_ib_pool_init(rdev);
3285         if (r) {
3286                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
3287                 return r;
3288         }
3289
3290         r = r600_audio_init(rdev);
3291         if (r) {
3292                 DRM_ERROR("radeon: audio init failed\n");
3293                 return r;
3294         }
3295
3296         return 0;
3297 }
3298
3299 void r600_vga_set_state(struct radeon_device *rdev, bool state)
3300 {
3301         uint32_t temp;
3302
3303         temp = RREG32(CONFIG_CNTL);
3304         if (state == false) {
3305                 temp &= ~(1<<0);
3306                 temp |= (1<<1);
3307         } else {
3308                 temp &= ~(1<<1);
3309         }
3310         WREG32(CONFIG_CNTL, temp);
3311 }
3312
3313 int r600_resume(struct radeon_device *rdev)
3314 {
3315         int r;
3316
3317         /* Do not reset GPU before posting, on r600 hw unlike on r500 hw,
3318          * posting will perform necessary task to bring back GPU into good
3319          * shape.
3320          */
3321         /* post card */
3322         atom_asic_init(rdev->mode_info.atom_context);
3323
3324         rdev->accel_working = true;
3325         r = r600_startup(rdev);
3326         if (r) {
3327                 DRM_ERROR("r600 startup failed on resume\n");
3328                 rdev->accel_working = false;
3329                 return r;
3330         }
3331
3332         return r;
3333 }
3334
3335 int r600_suspend(struct radeon_device *rdev)
3336 {
3337         r600_audio_fini(rdev);
3338         r600_cp_stop(rdev);
3339         r600_dma_stop(rdev);
3340         r600_irq_suspend(rdev);
3341         radeon_wb_disable(rdev);
3342         r600_pcie_gart_disable(rdev);
3343
3344         return 0;
3345 }
3346
3347 /* Plan is to move initialization in that function and use
3348  * helper function so that radeon_device_init pretty much
3349  * do nothing more than calling asic specific function. This
3350  * should also allow to remove a bunch of callback function
3351  * like vram_info.
3352  */
3353 int r600_init(struct radeon_device *rdev)
3354 {
3355         int r;
3356
3357         if (r600_debugfs_mc_info_init(rdev)) {
3358                 DRM_ERROR("Failed to register debugfs file for mc !\n");
3359         }
3360         /* Read BIOS */
3361         if (!radeon_get_bios(rdev)) {
3362                 if (ASIC_IS_AVIVO(rdev))
3363                         return -EINVAL;
3364         }
3365         /* Must be an ATOMBIOS */
3366         if (!rdev->is_atom_bios) {
3367                 dev_err(rdev->dev, "Expecting atombios for R600 GPU\n");
3368                 return -EINVAL;
3369         }
3370         r = radeon_atombios_init(rdev);
3371         if (r)
3372                 return r;
3373         /* Post card if necessary */
3374         if (!radeon_card_posted(rdev)) {
3375                 if (!rdev->bios) {
3376                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
3377                         return -EINVAL;
3378                 }
3379                 DRM_INFO("GPU not posted. posting now...\n");
3380                 atom_asic_init(rdev->mode_info.atom_context);
3381         }
3382         /* Initialize scratch registers */
3383         r600_scratch_init(rdev);
3384         /* Initialize surface registers */
3385         radeon_surface_init(rdev);
3386         /* Initialize clocks */
3387         radeon_get_clock_info(rdev->ddev);
3388         /* Fence driver */
3389         r = radeon_fence_driver_init(rdev);
3390         if (r)
3391                 return r;
3392         if (rdev->flags & RADEON_IS_AGP) {
3393                 r = radeon_agp_init(rdev);
3394                 if (r)
3395                         radeon_agp_disable(rdev);
3396         }
3397         r = r600_mc_init(rdev);
3398         if (r)
3399                 return r;
3400         /* Memory manager */
3401         r = radeon_bo_init(rdev);
3402         if (r)
3403                 return r;
3404
3405         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
3406         r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
3407
3408         rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
3409         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
3410
3411         rdev->ih.ring_obj = NULL;
3412         r600_ih_ring_init(rdev, 64 * 1024);
3413
3414         r = r600_pcie_gart_init(rdev);
3415         if (r)
3416                 return r;
3417
3418         rdev->accel_working = true;
3419         r = r600_startup(rdev);
3420         if (r) {
3421                 dev_err(rdev->dev, "disabling GPU acceleration\n");
3422                 r600_cp_fini(rdev);
3423                 r600_dma_fini(rdev);
3424                 r600_irq_fini(rdev);
3425                 radeon_wb_fini(rdev);
3426                 radeon_ib_pool_fini(rdev);
3427                 radeon_irq_kms_fini(rdev);
3428                 r600_pcie_gart_fini(rdev);
3429                 rdev->accel_working = false;
3430         }
3431
3432         return 0;
3433 }
3434
3435 void r600_fini(struct radeon_device *rdev)
3436 {
3437         r600_audio_fini(rdev);
3438         r600_blit_fini(rdev);
3439         r600_cp_fini(rdev);
3440         r600_dma_fini(rdev);
3441         r600_irq_fini(rdev);
3442         radeon_wb_fini(rdev);
3443         radeon_ib_pool_fini(rdev);
3444         radeon_irq_kms_fini(rdev);
3445         r600_pcie_gart_fini(rdev);
3446         r600_vram_scratch_fini(rdev);
3447         radeon_agp_fini(rdev);
3448         radeon_gem_fini(rdev);
3449         radeon_fence_driver_fini(rdev);
3450         radeon_bo_fini(rdev);
3451         radeon_atombios_fini(rdev);
3452         kfree(rdev->bios);
3453         rdev->bios = NULL;
3454 }
3455
3456
3457 /*
3458  * CS stuff
3459  */
3460 void r600_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3461 {
3462         struct radeon_ring *ring = &rdev->ring[ib->ring];
3463         u32 next_rptr;
3464
3465         if (ring->rptr_save_reg) {
3466                 next_rptr = ring->wptr + 3 + 4;
3467                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3468                 radeon_ring_write(ring, ((ring->rptr_save_reg -
3469                                          PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
3470                 radeon_ring_write(ring, next_rptr);
3471         } else if (rdev->wb.enabled) {
3472                 next_rptr = ring->wptr + 5 + 4;
3473                 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
3474                 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3475                 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
3476                 radeon_ring_write(ring, next_rptr);
3477                 radeon_ring_write(ring, 0);
3478         }
3479
3480         radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
3481         radeon_ring_write(ring,
3482 #ifdef __BIG_ENDIAN
3483                           (2 << 0) |
3484 #endif
3485                           (ib->gpu_addr & 0xFFFFFFFC));
3486         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
3487         radeon_ring_write(ring, ib->length_dw);
3488 }
3489
3490 void r600_uvd_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3491 {
3492         struct radeon_ring *ring = &rdev->ring[ib->ring];
3493
3494         radeon_ring_write(ring, PACKET0(UVD_RBC_IB_BASE, 0));
3495         radeon_ring_write(ring, ib->gpu_addr);
3496         radeon_ring_write(ring, PACKET0(UVD_RBC_IB_SIZE, 0));
3497         radeon_ring_write(ring, ib->length_dw);
3498 }
3499
3500 int r600_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3501 {
3502         struct radeon_ib ib;
3503         uint32_t scratch;
3504         uint32_t tmp = 0;
3505         unsigned i;
3506         int r;
3507
3508         r = radeon_scratch_get(rdev, &scratch);
3509         if (r) {
3510                 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3511                 return r;
3512         }
3513         WREG32(scratch, 0xCAFEDEAD);
3514         r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3515         if (r) {
3516                 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3517                 goto free_scratch;
3518         }
3519         ib.ptr[0] = PACKET3(PACKET3_SET_CONFIG_REG, 1);
3520         ib.ptr[1] = ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
3521         ib.ptr[2] = 0xDEADBEEF;
3522         ib.length_dw = 3;
3523         r = radeon_ib_schedule(rdev, &ib, NULL);
3524         if (r) {
3525                 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3526                 goto free_ib;
3527         }
3528         r = radeon_fence_wait(ib.fence, false);
3529         if (r) {
3530                 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3531                 goto free_ib;
3532         }
3533         for (i = 0; i < rdev->usec_timeout; i++) {
3534                 tmp = RREG32(scratch);
3535                 if (tmp == 0xDEADBEEF)
3536                         break;
3537                 DRM_UDELAY(1);
3538         }
3539         if (i < rdev->usec_timeout) {
3540                 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3541         } else {
3542                 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3543                           scratch, tmp);
3544                 r = -EINVAL;
3545         }
3546 free_ib:
3547         radeon_ib_free(rdev, &ib);
3548 free_scratch:
3549         radeon_scratch_free(rdev, scratch);
3550         return r;
3551 }
3552
3553 /**
3554  * r600_dma_ib_test - test an IB on the DMA engine
3555  *
3556  * @rdev: radeon_device pointer
3557  * @ring: radeon_ring structure holding ring information
3558  *
3559  * Test a simple IB in the DMA ring (r6xx-SI).
3560  * Returns 0 on success, error on failure.
3561  */
3562 int r600_dma_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3563 {
3564         struct radeon_ib ib;
3565         unsigned i;
3566         int r;
3567         void __iomem *ptr = (void *)rdev->vram_scratch.ptr;
3568         u32 tmp = 0;
3569
3570         if (!ptr) {
3571                 DRM_ERROR("invalid vram scratch pointer\n");
3572                 return -EINVAL;
3573         }
3574
3575         tmp = 0xCAFEDEAD;
3576         writel(tmp, ptr);
3577
3578         r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3579         if (r) {
3580                 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3581                 return r;
3582         }
3583
3584         ib.ptr[0] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1);
3585         ib.ptr[1] = rdev->vram_scratch.gpu_addr & 0xfffffffc;
3586         ib.ptr[2] = upper_32_bits(rdev->vram_scratch.gpu_addr) & 0xff;
3587         ib.ptr[3] = 0xDEADBEEF;
3588         ib.length_dw = 4;
3589
3590         r = radeon_ib_schedule(rdev, &ib, NULL);
3591         if (r) {
3592                 radeon_ib_free(rdev, &ib);
3593                 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3594                 return r;
3595         }
3596         r = radeon_fence_wait(ib.fence, false);
3597         if (r) {
3598                 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3599                 return r;
3600         }
3601         for (i = 0; i < rdev->usec_timeout; i++) {
3602                 tmp = readl(ptr);
3603                 if (tmp == 0xDEADBEEF)
3604                         break;
3605                 DRM_UDELAY(1);
3606         }
3607         if (i < rdev->usec_timeout) {
3608                 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3609         } else {
3610                 DRM_ERROR("radeon: ib test failed (0x%08X)\n", tmp);
3611                 r = -EINVAL;
3612         }
3613         radeon_ib_free(rdev, &ib);
3614         return r;
3615 }
3616
3617 int r600_uvd_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3618 {
3619         struct radeon_fence *fence = NULL;
3620         int r;
3621
3622         r = radeon_set_uvd_clocks(rdev, 53300, 40000);
3623         if (r) {
3624                 DRM_ERROR("radeon: failed to raise UVD clocks (%d).\n", r);
3625                 return r;
3626         }
3627
3628         r = radeon_uvd_get_create_msg(rdev, ring->idx, 1, NULL);
3629         if (r) {
3630                 DRM_ERROR("radeon: failed to get create msg (%d).\n", r);
3631                 goto error;
3632         }
3633
3634         r = radeon_uvd_get_destroy_msg(rdev, ring->idx, 1, &fence);
3635         if (r) {
3636                 DRM_ERROR("radeon: failed to get destroy ib (%d).\n", r);
3637                 goto error;
3638         }
3639
3640         r = radeon_fence_wait(fence, false);
3641         if (r) {
3642                 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3643                 goto error;
3644         }
3645         DRM_INFO("ib test on ring %d succeeded\n",  ring->idx);
3646 error:
3647         radeon_fence_unref(&fence);
3648         radeon_set_uvd_clocks(rdev, 0, 0);
3649         return r;
3650 }
3651
3652 /**
3653  * r600_dma_ring_ib_execute - Schedule an IB on the DMA engine
3654  *
3655  * @rdev: radeon_device pointer
3656  * @ib: IB object to schedule
3657  *
3658  * Schedule an IB in the DMA ring (r6xx-r7xx).
3659  */
3660 void r600_dma_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3661 {
3662         struct radeon_ring *ring = &rdev->ring[ib->ring];
3663
3664         if (rdev->wb.enabled) {
3665                 u32 next_rptr = ring->wptr + 4;
3666                 while ((next_rptr & 7) != 5)
3667                         next_rptr++;
3668                 next_rptr += 3;
3669                 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1));
3670                 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3671                 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff);
3672                 radeon_ring_write(ring, next_rptr);
3673         }
3674
3675         /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring.
3676          * Pad as necessary with NOPs.
3677          */
3678         while ((ring->wptr & 7) != 5)
3679                 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
3680         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_INDIRECT_BUFFER, 0, 0, 0));
3681         radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0));
3682         radeon_ring_write(ring, (ib->length_dw << 16) | (upper_32_bits(ib->gpu_addr) & 0xFF));
3683
3684 }
3685
3686 /*
3687  * Interrupts
3688  *
3689  * Interrupts use a ring buffer on r6xx/r7xx hardware.  It works pretty
3690  * the same as the CP ring buffer, but in reverse.  Rather than the CPU
3691  * writing to the ring and the GPU consuming, the GPU writes to the ring
3692  * and host consumes.  As the host irq handler processes interrupts, it
3693  * increments the rptr.  When the rptr catches up with the wptr, all the
3694  * current interrupts have been processed.
3695  */
3696
3697 void r600_ih_ring_init(struct radeon_device *rdev, unsigned ring_size)
3698 {
3699         u32 rb_bufsz;
3700
3701         /* Align ring size */
3702         rb_bufsz = drm_order(ring_size / 4);
3703         ring_size = (1 << rb_bufsz) * 4;
3704         rdev->ih.ring_size = ring_size;
3705         rdev->ih.ptr_mask = rdev->ih.ring_size - 1;
3706         rdev->ih.rptr = 0;
3707 }
3708
3709 int r600_ih_ring_alloc(struct radeon_device *rdev)
3710 {
3711         int r;
3712
3713         /* Allocate ring buffer */
3714         if (rdev->ih.ring_obj == NULL) {
3715                 r = radeon_bo_create(rdev, rdev->ih.ring_size,
3716                                      PAGE_SIZE, true,
3717                                      RADEON_GEM_DOMAIN_GTT,
3718                                      NULL, &rdev->ih.ring_obj);
3719                 if (r) {
3720                         DRM_ERROR("radeon: failed to create ih ring buffer (%d).\n", r);
3721                         return r;
3722                 }
3723                 r = radeon_bo_reserve(rdev->ih.ring_obj, false);
3724                 if (unlikely(r != 0))
3725                         return r;
3726                 r = radeon_bo_pin(rdev->ih.ring_obj,
3727                                   RADEON_GEM_DOMAIN_GTT,
3728                                   &rdev->ih.gpu_addr);
3729                 if (r) {
3730                         radeon_bo_unreserve(rdev->ih.ring_obj);
3731                         DRM_ERROR("radeon: failed to pin ih ring buffer (%d).\n", r);
3732                         return r;
3733                 }
3734                 r = radeon_bo_kmap(rdev->ih.ring_obj,
3735                                    (void **)&rdev->ih.ring);
3736                 radeon_bo_unreserve(rdev->ih.ring_obj);
3737                 if (r) {
3738                         DRM_ERROR("radeon: failed to map ih ring buffer (%d).\n", r);
3739                         return r;
3740                 }
3741         }
3742         return 0;
3743 }
3744
3745 void r600_ih_ring_fini(struct radeon_device *rdev)
3746 {
3747         int r;
3748         if (rdev->ih.ring_obj) {
3749                 r = radeon_bo_reserve(rdev->ih.ring_obj, false);
3750                 if (likely(r == 0)) {
3751                         radeon_bo_kunmap(rdev->ih.ring_obj);
3752                         radeon_bo_unpin(rdev->ih.ring_obj);
3753                         radeon_bo_unreserve(rdev->ih.ring_obj);
3754                 }
3755                 radeon_bo_unref(&rdev->ih.ring_obj);
3756                 rdev->ih.ring = NULL;
3757                 rdev->ih.ring_obj = NULL;
3758         }
3759 }
3760
3761 void r600_rlc_stop(struct radeon_device *rdev)
3762 {
3763
3764         if ((rdev->family >= CHIP_RV770) &&
3765             (rdev->family <= CHIP_RV740)) {
3766                 /* r7xx asics need to soft reset RLC before halting */
3767                 WREG32(SRBM_SOFT_RESET, SOFT_RESET_RLC);
3768                 RREG32(SRBM_SOFT_RESET);
3769                 mdelay(15);
3770                 WREG32(SRBM_SOFT_RESET, 0);
3771                 RREG32(SRBM_SOFT_RESET);
3772         }
3773
3774         WREG32(RLC_CNTL, 0);
3775 }
3776
3777 static void r600_rlc_start(struct radeon_device *rdev)
3778 {
3779         WREG32(RLC_CNTL, RLC_ENABLE);
3780 }
3781
3782 static int r600_rlc_resume(struct radeon_device *rdev)
3783 {
3784         u32 i;
3785         const __be32 *fw_data;
3786
3787         if (!rdev->rlc_fw)
3788                 return -EINVAL;
3789
3790         r600_rlc_stop(rdev);
3791
3792         WREG32(RLC_HB_CNTL, 0);
3793
3794         WREG32(RLC_HB_BASE, 0);
3795         WREG32(RLC_HB_RPTR, 0);
3796         WREG32(RLC_HB_WPTR, 0);
3797         WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
3798         WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
3799         WREG32(RLC_MC_CNTL, 0);
3800         WREG32(RLC_UCODE_CNTL, 0);
3801
3802         fw_data = (const __be32 *)rdev->rlc_fw->data;
3803         if (rdev->family >= CHIP_RV770) {
3804                 for (i = 0; i < R700_RLC_UCODE_SIZE; i++) {
3805                         WREG32(RLC_UCODE_ADDR, i);
3806                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
3807                 }
3808         } else {
3809                 for (i = 0; i < R600_RLC_UCODE_SIZE; i++) {
3810                         WREG32(RLC_UCODE_ADDR, i);
3811                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
3812                 }
3813         }
3814         WREG32(RLC_UCODE_ADDR, 0);
3815
3816         r600_rlc_start(rdev);
3817
3818         return 0;
3819 }
3820
3821 static void r600_enable_interrupts(struct radeon_device *rdev)
3822 {
3823         u32 ih_cntl = RREG32(IH_CNTL);
3824         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
3825
3826         ih_cntl |= ENABLE_INTR;
3827         ih_rb_cntl |= IH_RB_ENABLE;
3828         WREG32(IH_CNTL, ih_cntl);
3829         WREG32(IH_RB_CNTL, ih_rb_cntl);
3830         rdev->ih.enabled = true;
3831 }
3832
3833 void r600_disable_interrupts(struct radeon_device *rdev)
3834 {
3835         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
3836         u32 ih_cntl = RREG32(IH_CNTL);
3837
3838         ih_rb_cntl &= ~IH_RB_ENABLE;
3839         ih_cntl &= ~ENABLE_INTR;
3840         WREG32(IH_RB_CNTL, ih_rb_cntl);
3841         WREG32(IH_CNTL, ih_cntl);
3842         /* set rptr, wptr to 0 */
3843         WREG32(IH_RB_RPTR, 0);
3844         WREG32(IH_RB_WPTR, 0);
3845         rdev->ih.enabled = false;
3846         rdev->ih.rptr = 0;
3847 }
3848
3849 static void r600_disable_interrupt_state(struct radeon_device *rdev)
3850 {
3851         u32 tmp;
3852
3853         WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
3854         tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
3855         WREG32(DMA_CNTL, tmp);
3856         WREG32(GRBM_INT_CNTL, 0);
3857         WREG32(DxMODE_INT_MASK, 0);
3858         WREG32(D1GRPH_INTERRUPT_CONTROL, 0);
3859         WREG32(D2GRPH_INTERRUPT_CONTROL, 0);
3860         if (ASIC_IS_DCE3(rdev)) {
3861                 WREG32(DCE3_DACA_AUTODETECT_INT_CONTROL, 0);
3862                 WREG32(DCE3_DACB_AUTODETECT_INT_CONTROL, 0);
3863                 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3864                 WREG32(DC_HPD1_INT_CONTROL, tmp);
3865                 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3866                 WREG32(DC_HPD2_INT_CONTROL, tmp);
3867                 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3868                 WREG32(DC_HPD3_INT_CONTROL, tmp);
3869                 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3870                 WREG32(DC_HPD4_INT_CONTROL, tmp);
3871                 if (ASIC_IS_DCE32(rdev)) {
3872                         tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3873                         WREG32(DC_HPD5_INT_CONTROL, tmp);
3874                         tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3875                         WREG32(DC_HPD6_INT_CONTROL, tmp);
3876                         tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3877                         WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, tmp);
3878                         tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3879                         WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, tmp);
3880                 } else {
3881                         tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3882                         WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
3883                         tmp = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3884                         WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, tmp);
3885                 }
3886         } else {
3887                 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
3888                 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
3889                 tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
3890                 WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
3891                 tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
3892                 WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
3893                 tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
3894                 WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
3895                 tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3896                 WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
3897                 tmp = RREG32(HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3898                 WREG32(HDMI1_AUDIO_PACKET_CONTROL, tmp);
3899         }
3900 }
3901
3902 int r600_irq_init(struct radeon_device *rdev)
3903 {
3904         int ret = 0;
3905         int rb_bufsz;
3906         u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
3907
3908         /* allocate ring */
3909         ret = r600_ih_ring_alloc(rdev);
3910         if (ret)
3911                 return ret;
3912
3913         /* disable irqs */
3914         r600_disable_interrupts(rdev);
3915
3916         /* init rlc */
3917         if (rdev->family >= CHIP_CEDAR)
3918                 ret = evergreen_rlc_resume(rdev);
3919         else
3920                 ret = r600_rlc_resume(rdev);
3921         if (ret) {
3922                 r600_ih_ring_fini(rdev);
3923                 return ret;
3924         }
3925
3926         /* setup interrupt control */
3927         /* set dummy read address to ring address */
3928         WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
3929         interrupt_cntl = RREG32(INTERRUPT_CNTL);
3930         /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
3931          * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
3932          */
3933         interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
3934         /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
3935         interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
3936         WREG32(INTERRUPT_CNTL, interrupt_cntl);
3937
3938         WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
3939         rb_bufsz = drm_order(rdev->ih.ring_size / 4);
3940
3941         ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
3942                       IH_WPTR_OVERFLOW_CLEAR |
3943                       (rb_bufsz << 1));
3944
3945         if (rdev->wb.enabled)
3946                 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
3947
3948         /* set the writeback address whether it's enabled or not */
3949         WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
3950         WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
3951
3952         WREG32(IH_RB_CNTL, ih_rb_cntl);
3953
3954         /* set rptr, wptr to 0 */
3955         WREG32(IH_RB_RPTR, 0);
3956         WREG32(IH_RB_WPTR, 0);
3957
3958         /* Default settings for IH_CNTL (disabled at first) */
3959         ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10);
3960         /* RPTR_REARM only works if msi's are enabled */
3961         if (rdev->msi_enabled)
3962                 ih_cntl |= RPTR_REARM;
3963         WREG32(IH_CNTL, ih_cntl);
3964
3965         /* force the active interrupt state to all disabled */
3966         if (rdev->family >= CHIP_CEDAR)
3967                 evergreen_disable_interrupt_state(rdev);
3968         else
3969                 r600_disable_interrupt_state(rdev);
3970
3971         /* at this point everything should be setup correctly to enable master */
3972         pci_set_master(rdev->pdev);
3973
3974         /* enable irqs */
3975         r600_enable_interrupts(rdev);
3976
3977         return ret;
3978 }
3979
3980 void r600_irq_suspend(struct radeon_device *rdev)
3981 {
3982         r600_irq_disable(rdev);
3983         r600_rlc_stop(rdev);
3984 }
3985
3986 void r600_irq_fini(struct radeon_device *rdev)
3987 {
3988         r600_irq_suspend(rdev);
3989         r600_ih_ring_fini(rdev);
3990 }
3991
3992 int r600_irq_set(struct radeon_device *rdev)
3993 {
3994         u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
3995         u32 mode_int = 0;
3996         u32 hpd1, hpd2, hpd3, hpd4 = 0, hpd5 = 0, hpd6 = 0;
3997         u32 grbm_int_cntl = 0;
3998         u32 hdmi0, hdmi1;
3999         u32 d1grph = 0, d2grph = 0;
4000         u32 dma_cntl;
4001         u32 thermal_int = 0;
4002
4003         if (!rdev->irq.installed) {
4004                 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4005                 return -EINVAL;
4006         }
4007         /* don't enable anything if the ih is disabled */
4008         if (!rdev->ih.enabled) {
4009                 r600_disable_interrupts(rdev);
4010                 /* force the active interrupt state to all disabled */
4011                 r600_disable_interrupt_state(rdev);
4012                 return 0;
4013         }
4014
4015         if (ASIC_IS_DCE3(rdev)) {
4016                 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4017                 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4018                 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
4019                 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
4020                 if (ASIC_IS_DCE32(rdev)) {
4021                         hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
4022                         hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
4023                         hdmi0 = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4024                         hdmi1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4025                 } else {
4026                         hdmi0 = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
4027                         hdmi1 = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
4028                 }
4029         } else {
4030                 hpd1 = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4031                 hpd2 = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4032                 hpd3 = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & ~DC_HPDx_INT_EN;
4033                 hdmi0 = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
4034                 hdmi1 = RREG32(HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
4035         }
4036
4037         dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4038
4039         if ((rdev->family > CHIP_R600) && (rdev->family < CHIP_RV770)) {
4040                 thermal_int = RREG32(CG_THERMAL_INT) &
4041                         ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4042                 if (rdev->irq.dpm_thermal) {
4043                         DRM_DEBUG("dpm thermal\n");
4044                         thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4045                 }
4046         }
4047
4048         if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4049                 DRM_DEBUG("r600_irq_set: sw int\n");
4050                 cp_int_cntl |= RB_INT_ENABLE;
4051                 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4052         }
4053
4054         if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4055                 DRM_DEBUG("r600_irq_set: sw int dma\n");
4056                 dma_cntl |= TRAP_ENABLE;
4057         }
4058
4059         if (rdev->irq.crtc_vblank_int[0] ||
4060             atomic_read(&rdev->irq.pflip[0])) {
4061                 DRM_DEBUG("r600_irq_set: vblank 0\n");
4062                 mode_int |= D1MODE_VBLANK_INT_MASK;
4063         }
4064         if (rdev->irq.crtc_vblank_int[1] ||
4065             atomic_read(&rdev->irq.pflip[1])) {
4066                 DRM_DEBUG("r600_irq_set: vblank 1\n");
4067                 mode_int |= D2MODE_VBLANK_INT_MASK;
4068         }
4069         if (rdev->irq.hpd[0]) {
4070                 DRM_DEBUG("r600_irq_set: hpd 1\n");
4071                 hpd1 |= DC_HPDx_INT_EN;
4072         }
4073         if (rdev->irq.hpd[1]) {
4074                 DRM_DEBUG("r600_irq_set: hpd 2\n");
4075                 hpd2 |= DC_HPDx_INT_EN;
4076         }
4077         if (rdev->irq.hpd[2]) {
4078                 DRM_DEBUG("r600_irq_set: hpd 3\n");
4079                 hpd3 |= DC_HPDx_INT_EN;
4080         }
4081         if (rdev->irq.hpd[3]) {
4082                 DRM_DEBUG("r600_irq_set: hpd 4\n");
4083                 hpd4 |= DC_HPDx_INT_EN;
4084         }
4085         if (rdev->irq.hpd[4]) {
4086                 DRM_DEBUG("r600_irq_set: hpd 5\n");
4087                 hpd5 |= DC_HPDx_INT_EN;
4088         }
4089         if (rdev->irq.hpd[5]) {
4090                 DRM_DEBUG("r600_irq_set: hpd 6\n");
4091                 hpd6 |= DC_HPDx_INT_EN;
4092         }
4093         if (rdev->irq.afmt[0]) {
4094                 DRM_DEBUG("r600_irq_set: hdmi 0\n");
4095                 hdmi0 |= HDMI0_AZ_FORMAT_WTRIG_MASK;
4096         }
4097         if (rdev->irq.afmt[1]) {
4098                 DRM_DEBUG("r600_irq_set: hdmi 0\n");
4099                 hdmi1 |= HDMI0_AZ_FORMAT_WTRIG_MASK;
4100         }
4101
4102         WREG32(CP_INT_CNTL, cp_int_cntl);
4103         WREG32(DMA_CNTL, dma_cntl);
4104         WREG32(DxMODE_INT_MASK, mode_int);
4105         WREG32(D1GRPH_INTERRUPT_CONTROL, d1grph);
4106         WREG32(D2GRPH_INTERRUPT_CONTROL, d2grph);
4107         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4108         if (ASIC_IS_DCE3(rdev)) {
4109                 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4110                 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4111                 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4112                 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4113                 if (ASIC_IS_DCE32(rdev)) {
4114                         WREG32(DC_HPD5_INT_CONTROL, hpd5);
4115                         WREG32(DC_HPD6_INT_CONTROL, hpd6);
4116                         WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, hdmi0);
4117                         WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, hdmi1);
4118                 } else {
4119                         WREG32(HDMI0_AUDIO_PACKET_CONTROL, hdmi0);
4120                         WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, hdmi1);
4121                 }
4122         } else {
4123                 WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, hpd1);
4124                 WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, hpd2);
4125                 WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, hpd3);
4126                 WREG32(HDMI0_AUDIO_PACKET_CONTROL, hdmi0);
4127                 WREG32(HDMI1_AUDIO_PACKET_CONTROL, hdmi1);
4128         }
4129         if ((rdev->family > CHIP_R600) && (rdev->family < CHIP_RV770)) {
4130                 WREG32(CG_THERMAL_INT, thermal_int);
4131         }
4132
4133         return 0;
4134 }
4135
4136 static void r600_irq_ack(struct radeon_device *rdev)
4137 {
4138         u32 tmp;
4139
4140         if (ASIC_IS_DCE3(rdev)) {
4141                 rdev->irq.stat_regs.r600.disp_int = RREG32(DCE3_DISP_INTERRUPT_STATUS);
4142                 rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE);
4143                 rdev->irq.stat_regs.r600.disp_int_cont2 = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE2);
4144                 if (ASIC_IS_DCE32(rdev)) {
4145                         rdev->irq.stat_regs.r600.hdmi0_status = RREG32(AFMT_STATUS + DCE3_HDMI_OFFSET0);
4146                         rdev->irq.stat_regs.r600.hdmi1_status = RREG32(AFMT_STATUS + DCE3_HDMI_OFFSET1);
4147                 } else {
4148                         rdev->irq.stat_regs.r600.hdmi0_status = RREG32(HDMI0_STATUS);
4149                         rdev->irq.stat_regs.r600.hdmi1_status = RREG32(DCE3_HDMI1_STATUS);
4150                 }
4151         } else {
4152                 rdev->irq.stat_regs.r600.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4153                 rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4154                 rdev->irq.stat_regs.r600.disp_int_cont2 = 0;
4155                 rdev->irq.stat_regs.r600.hdmi0_status = RREG32(HDMI0_STATUS);
4156                 rdev->irq.stat_regs.r600.hdmi1_status = RREG32(HDMI1_STATUS);
4157         }
4158         rdev->irq.stat_regs.r600.d1grph_int = RREG32(D1GRPH_INTERRUPT_STATUS);
4159         rdev->irq.stat_regs.r600.d2grph_int = RREG32(D2GRPH_INTERRUPT_STATUS);
4160
4161         if (rdev->irq.stat_regs.r600.d1grph_int & DxGRPH_PFLIP_INT_OCCURRED)
4162                 WREG32(D1GRPH_INTERRUPT_STATUS, DxGRPH_PFLIP_INT_CLEAR);
4163         if (rdev->irq.stat_regs.r600.d2grph_int & DxGRPH_PFLIP_INT_OCCURRED)
4164                 WREG32(D2GRPH_INTERRUPT_STATUS, DxGRPH_PFLIP_INT_CLEAR);
4165         if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT)
4166                 WREG32(D1MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
4167         if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT)
4168                 WREG32(D1MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
4169         if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT)
4170                 WREG32(D2MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
4171         if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT)
4172                 WREG32(D2MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
4173         if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) {
4174                 if (ASIC_IS_DCE3(rdev)) {
4175                         tmp = RREG32(DC_HPD1_INT_CONTROL);
4176                         tmp |= DC_HPDx_INT_ACK;
4177                         WREG32(DC_HPD1_INT_CONTROL, tmp);
4178                 } else {
4179                         tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
4180                         tmp |= DC_HPDx_INT_ACK;
4181                         WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
4182                 }
4183         }
4184         if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) {
4185                 if (ASIC_IS_DCE3(rdev)) {
4186                         tmp = RREG32(DC_HPD2_INT_CONTROL);
4187                         tmp |= DC_HPDx_INT_ACK;
4188                         WREG32(DC_HPD2_INT_CONTROL, tmp);
4189                 } else {
4190                         tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
4191                         tmp |= DC_HPDx_INT_ACK;
4192                         WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
4193                 }
4194         }
4195         if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) {
4196                 if (ASIC_IS_DCE3(rdev)) {
4197                         tmp = RREG32(DC_HPD3_INT_CONTROL);
4198                         tmp |= DC_HPDx_INT_ACK;
4199                         WREG32(DC_HPD3_INT_CONTROL, tmp);
4200                 } else {
4201                         tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
4202                         tmp |= DC_HPDx_INT_ACK;
4203                         WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
4204                 }
4205         }
4206         if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) {
4207                 tmp = RREG32(DC_HPD4_INT_CONTROL);
4208                 tmp |= DC_HPDx_INT_ACK;
4209                 WREG32(DC_HPD4_INT_CONTROL, tmp);
4210         }
4211         if (ASIC_IS_DCE32(rdev)) {
4212                 if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) {
4213                         tmp = RREG32(DC_HPD5_INT_CONTROL);
4214                         tmp |= DC_HPDx_INT_ACK;
4215                         WREG32(DC_HPD5_INT_CONTROL, tmp);
4216                 }
4217                 if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) {
4218                         tmp = RREG32(DC_HPD5_INT_CONTROL);
4219                         tmp |= DC_HPDx_INT_ACK;
4220                         WREG32(DC_HPD6_INT_CONTROL, tmp);
4221                 }
4222                 if (rdev->irq.stat_regs.r600.hdmi0_status & AFMT_AZ_FORMAT_WTRIG) {
4223                         tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0);
4224                         tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4225                         WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, tmp);
4226                 }
4227                 if (rdev->irq.stat_regs.r600.hdmi1_status & AFMT_AZ_FORMAT_WTRIG) {
4228                         tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1);
4229                         tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4230                         WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, tmp);
4231                 }
4232         } else {
4233                 if (rdev->irq.stat_regs.r600.hdmi0_status & HDMI0_AZ_FORMAT_WTRIG) {
4234                         tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL);
4235                         tmp |= HDMI0_AZ_FORMAT_WTRIG_ACK;
4236                         WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
4237                 }
4238                 if (rdev->irq.stat_regs.r600.hdmi1_status & HDMI0_AZ_FORMAT_WTRIG) {
4239                         if (ASIC_IS_DCE3(rdev)) {
4240                                 tmp = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL);
4241                                 tmp |= HDMI0_AZ_FORMAT_WTRIG_ACK;
4242                                 WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, tmp);
4243                         } else {
4244                                 tmp = RREG32(HDMI1_AUDIO_PACKET_CONTROL);
4245                                 tmp |= HDMI0_AZ_FORMAT_WTRIG_ACK;
4246                                 WREG32(HDMI1_AUDIO_PACKET_CONTROL, tmp);
4247                         }
4248                 }
4249         }
4250 }
4251
4252 void r600_irq_disable(struct radeon_device *rdev)
4253 {
4254         r600_disable_interrupts(rdev);
4255         /* Wait and acknowledge irq */
4256         mdelay(1);
4257         r600_irq_ack(rdev);
4258         r600_disable_interrupt_state(rdev);
4259 }
4260
4261 static u32 r600_get_ih_wptr(struct radeon_device *rdev)
4262 {
4263         u32 wptr, tmp;
4264
4265         if (rdev->wb.enabled)
4266                 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4267         else
4268                 wptr = RREG32(IH_RB_WPTR);
4269
4270         if (wptr & RB_OVERFLOW) {
4271                 /* When a ring buffer overflow happen start parsing interrupt
4272                  * from the last not overwritten vector (wptr + 16). Hopefully
4273                  * this should allow us to catchup.
4274                  */
4275                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
4276                         wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
4277                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4278                 tmp = RREG32(IH_RB_CNTL);
4279                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4280                 WREG32(IH_RB_CNTL, tmp);
4281         }
4282         return (wptr & rdev->ih.ptr_mask);
4283 }
4284
4285 /*        r600 IV Ring
4286  * Each IV ring entry is 128 bits:
4287  * [7:0]    - interrupt source id
4288  * [31:8]   - reserved
4289  * [59:32]  - interrupt source data
4290  * [127:60]  - reserved
4291  *
4292  * The basic interrupt vector entries
4293  * are decoded as follows:
4294  * src_id  src_data  description
4295  *      1         0  D1 Vblank
4296  *      1         1  D1 Vline
4297  *      5         0  D2 Vblank
4298  *      5         1  D2 Vline
4299  *     19         0  FP Hot plug detection A
4300  *     19         1  FP Hot plug detection B
4301  *     19         2  DAC A auto-detection
4302  *     19         3  DAC B auto-detection
4303  *     21         4  HDMI block A
4304  *     21         5  HDMI block B
4305  *    176         -  CP_INT RB
4306  *    177         -  CP_INT IB1
4307  *    178         -  CP_INT IB2
4308  *    181         -  EOP Interrupt
4309  *    233         -  GUI Idle
4310  *
4311  * Note, these are based on r600 and may need to be
4312  * adjusted or added to on newer asics
4313  */
4314
4315 int r600_irq_process(struct radeon_device *rdev)
4316 {
4317         u32 wptr;
4318         u32 rptr;
4319         u32 src_id, src_data;
4320         u32 ring_index;
4321         bool queue_hotplug = false;
4322         bool queue_hdmi = false;
4323         bool queue_thermal = false;
4324
4325         if (!rdev->ih.enabled || rdev->shutdown)
4326                 return IRQ_NONE;
4327
4328         /* No MSIs, need a dummy read to flush PCI DMAs */
4329         if (!rdev->msi_enabled)
4330                 RREG32(IH_RB_WPTR);
4331
4332         wptr = r600_get_ih_wptr(rdev);
4333
4334 restart_ih:
4335         /* is somebody else already processing irqs? */
4336         if (atomic_xchg(&rdev->ih.lock, 1))
4337                 return IRQ_NONE;
4338
4339         rptr = rdev->ih.rptr;
4340         DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4341
4342         /* Order reading of wptr vs. reading of IH ring data */
4343         rmb();
4344
4345         /* display interrupts */
4346         r600_irq_ack(rdev);
4347
4348         while (rptr != wptr) {
4349                 /* wptr/rptr are in bytes! */
4350                 ring_index = rptr / 4;
4351                 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4352                 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4353
4354                 switch (src_id) {
4355                 case 1: /* D1 vblank/vline */
4356                         switch (src_data) {
4357                         case 0: /* D1 vblank */
4358                                 if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT) {
4359                                         if (rdev->irq.crtc_vblank_int[0]) {
4360                                                 drm_handle_vblank(rdev->ddev, 0);
4361                                                 rdev->pm.vblank_sync = true;
4362                                                 wake_up(&rdev->irq.vblank_queue);
4363                                         }
4364                                         if (atomic_read(&rdev->irq.pflip[0]))
4365                                                 radeon_crtc_handle_flip(rdev, 0);
4366                                         rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
4367                                         DRM_DEBUG("IH: D1 vblank\n");
4368                                 }
4369                                 break;
4370                         case 1: /* D1 vline */
4371                                 if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT) {
4372                                         rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VLINE_INTERRUPT;
4373                                         DRM_DEBUG("IH: D1 vline\n");
4374                                 }
4375                                 break;
4376                         default:
4377                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4378                                 break;
4379                         }
4380                         break;
4381                 case 5: /* D2 vblank/vline */
4382                         switch (src_data) {
4383                         case 0: /* D2 vblank */
4384                                 if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT) {
4385                                         if (rdev->irq.crtc_vblank_int[1]) {
4386                                                 drm_handle_vblank(rdev->ddev, 1);
4387                                                 rdev->pm.vblank_sync = true;
4388                                                 wake_up(&rdev->irq.vblank_queue);
4389                                         }
4390                                         if (atomic_read(&rdev->irq.pflip[1]))
4391                                                 radeon_crtc_handle_flip(rdev, 1);
4392                                         rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VBLANK_INTERRUPT;
4393                                         DRM_DEBUG("IH: D2 vblank\n");
4394                                 }
4395                                 break;
4396                         case 1: /* D1 vline */
4397                                 if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT) {
4398                                         rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VLINE_INTERRUPT;
4399                                         DRM_DEBUG("IH: D2 vline\n");
4400                                 }
4401                                 break;
4402                         default:
4403                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4404                                 break;
4405                         }
4406                         break;
4407                 case 19: /* HPD/DAC hotplug */
4408                         switch (src_data) {
4409                         case 0:
4410                                 if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) {
4411                                         rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD1_INTERRUPT;
4412                                         queue_hotplug = true;
4413                                         DRM_DEBUG("IH: HPD1\n");
4414                                 }
4415                                 break;
4416                         case 1:
4417                                 if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) {
4418                                         rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD2_INTERRUPT;
4419                                         queue_hotplug = true;
4420                                         DRM_DEBUG("IH: HPD2\n");
4421                                 }
4422                                 break;
4423                         case 4:
4424                                 if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) {
4425                                         rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD3_INTERRUPT;
4426                                         queue_hotplug = true;
4427                                         DRM_DEBUG("IH: HPD3\n");
4428                                 }
4429                                 break;
4430                         case 5:
4431                                 if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) {
4432                                         rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD4_INTERRUPT;
4433                                         queue_hotplug = true;
4434                                         DRM_DEBUG("IH: HPD4\n");
4435                                 }
4436                                 break;
4437                         case 10:
4438                                 if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) {
4439                                         rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD5_INTERRUPT;
4440                                         queue_hotplug = true;
4441                                         DRM_DEBUG("IH: HPD5\n");
4442                                 }
4443                                 break;
4444                         case 12:
4445                                 if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) {
4446                                         rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD6_INTERRUPT;
4447                                         queue_hotplug = true;
4448                                         DRM_DEBUG("IH: HPD6\n");
4449                                 }
4450                                 break;
4451                         default:
4452                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4453                                 break;
4454                         }
4455                         break;
4456                 case 21: /* hdmi */
4457                         switch (src_data) {
4458                         case 4:
4459                                 if (rdev->irq.stat_regs.r600.hdmi0_status & HDMI0_AZ_FORMAT_WTRIG) {
4460                                         rdev->irq.stat_regs.r600.hdmi0_status &= ~HDMI0_AZ_FORMAT_WTRIG;
4461                                         queue_hdmi = true;
4462                                         DRM_DEBUG("IH: HDMI0\n");
4463                                 }
4464                                 break;
4465                         case 5:
4466                                 if (rdev->irq.stat_regs.r600.hdmi1_status & HDMI0_AZ_FORMAT_WTRIG) {
4467                                         rdev->irq.stat_regs.r600.hdmi1_status &= ~HDMI0_AZ_FORMAT_WTRIG;
4468                                         queue_hdmi = true;
4469                                         DRM_DEBUG("IH: HDMI1\n");
4470                                 }
4471                                 break;
4472                         default:
4473                                 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
4474                                 break;
4475                         }
4476                         break;
4477                 case 176: /* CP_INT in ring buffer */
4478                 case 177: /* CP_INT in IB1 */
4479                 case 178: /* CP_INT in IB2 */
4480                         DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
4481                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4482                         break;
4483                 case 181: /* CP EOP event */
4484                         DRM_DEBUG("IH: CP EOP\n");
4485                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4486                         break;
4487                 case 224: /* DMA trap event */
4488                         DRM_DEBUG("IH: DMA trap\n");
4489                         radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4490                         break;
4491                 case 230: /* thermal low to high */
4492                         DRM_DEBUG("IH: thermal low to high\n");
4493                         rdev->pm.dpm.thermal.high_to_low = false;
4494                         queue_thermal = true;
4495                         break;
4496                 case 231: /* thermal high to low */
4497                         DRM_DEBUG("IH: thermal high to low\n");
4498                         rdev->pm.dpm.thermal.high_to_low = true;
4499                         queue_thermal = true;
4500                         break;
4501                 case 233: /* GUI IDLE */
4502                         DRM_DEBUG("IH: GUI idle\n");
4503                         break;
4504                 default:
4505                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4506                         break;
4507                 }
4508
4509                 /* wptr/rptr are in bytes! */
4510                 rptr += 16;
4511                 rptr &= rdev->ih.ptr_mask;
4512         }
4513         if (queue_hotplug)
4514                 schedule_work(&rdev->hotplug_work);
4515         if (queue_hdmi)
4516                 schedule_work(&rdev->audio_work);
4517         if (queue_thermal && rdev->pm.dpm_enabled)
4518                 schedule_work(&rdev->pm.dpm.thermal.work);
4519         rdev->ih.rptr = rptr;
4520         WREG32(IH_RB_RPTR, rdev->ih.rptr);
4521         atomic_set(&rdev->ih.lock, 0);
4522
4523         /* make sure wptr hasn't changed while processing */
4524         wptr = r600_get_ih_wptr(rdev);
4525         if (wptr != rptr)
4526                 goto restart_ih;
4527
4528         return IRQ_HANDLED;
4529 }
4530
4531 /*
4532  * Debugfs info
4533  */
4534 #if defined(CONFIG_DEBUG_FS)
4535
4536 static int r600_debugfs_mc_info(struct seq_file *m, void *data)
4537 {
4538         struct drm_info_node *node = (struct drm_info_node *) m->private;
4539         struct drm_device *dev = node->minor->dev;
4540         struct radeon_device *rdev = dev->dev_private;
4541
4542         DREG32_SYS(m, rdev, R_000E50_SRBM_STATUS);
4543         DREG32_SYS(m, rdev, VM_L2_STATUS);
4544         return 0;
4545 }
4546
4547 static struct drm_info_list r600_mc_info_list[] = {
4548         {"r600_mc_info", r600_debugfs_mc_info, 0, NULL},
4549 };
4550 #endif
4551
4552 int r600_debugfs_mc_info_init(struct radeon_device *rdev)
4553 {
4554 #if defined(CONFIG_DEBUG_FS)
4555         return radeon_debugfs_add_files(rdev, r600_mc_info_list, ARRAY_SIZE(r600_mc_info_list));
4556 #else
4557         return 0;
4558 #endif
4559 }
4560
4561 /**
4562  * r600_ioctl_wait_idle - flush host path cache on wait idle ioctl
4563  * rdev: radeon device structure
4564  * bo: buffer object struct which userspace is waiting for idle
4565  *
4566  * Some R6XX/R7XX doesn't seems to take into account HDP flush performed
4567  * through ring buffer, this leads to corruption in rendering, see
4568  * http://bugzilla.kernel.org/show_bug.cgi?id=15186 to avoid this we
4569  * directly perform HDP flush by writing register through MMIO.
4570  */
4571 void r600_ioctl_wait_idle(struct radeon_device *rdev, struct radeon_bo *bo)
4572 {
4573         /* r7xx hw bug.  write to HDP_DEBUG1 followed by fb read
4574          * rather than write to HDP_REG_COHERENCY_FLUSH_CNTL.
4575          * This seems to cause problems on some AGP cards. Just use the old
4576          * method for them.
4577          */
4578         if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&
4579             rdev->vram_scratch.ptr && !(rdev->flags & RADEON_IS_AGP)) {
4580                 void __iomem *ptr = (void *)rdev->vram_scratch.ptr;
4581                 u32 tmp;
4582
4583                 WREG32(HDP_DEBUG1, 0);
4584                 tmp = readl((void __iomem *)ptr);
4585         } else
4586                 WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
4587 }
4588
4589 void r600_set_pcie_lanes(struct radeon_device *rdev, int lanes)
4590 {
4591         u32 link_width_cntl, mask;
4592
4593         if (rdev->flags & RADEON_IS_IGP)
4594                 return;
4595
4596         if (!(rdev->flags & RADEON_IS_PCIE))
4597                 return;
4598
4599         /* x2 cards have a special sequence */
4600         if (ASIC_IS_X2(rdev))
4601                 return;
4602
4603         radeon_gui_idle(rdev);
4604
4605         switch (lanes) {
4606         case 0:
4607                 mask = RADEON_PCIE_LC_LINK_WIDTH_X0;
4608                 break;
4609         case 1:
4610                 mask = RADEON_PCIE_LC_LINK_WIDTH_X1;
4611                 break;
4612         case 2:
4613                 mask = RADEON_PCIE_LC_LINK_WIDTH_X2;
4614                 break;
4615         case 4:
4616                 mask = RADEON_PCIE_LC_LINK_WIDTH_X4;
4617                 break;
4618         case 8:
4619                 mask = RADEON_PCIE_LC_LINK_WIDTH_X8;
4620                 break;
4621         case 12:
4622                 /* not actually supported */
4623                 mask = RADEON_PCIE_LC_LINK_WIDTH_X12;
4624                 break;
4625         case 16:
4626                 mask = RADEON_PCIE_LC_LINK_WIDTH_X16;
4627                 break;
4628         default:
4629                 DRM_ERROR("invalid pcie lane request: %d\n", lanes);
4630                 return;
4631         }
4632
4633         link_width_cntl = RREG32_PCIE_PORT(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
4634         link_width_cntl &= ~RADEON_PCIE_LC_LINK_WIDTH_MASK;
4635         link_width_cntl |= mask << RADEON_PCIE_LC_LINK_WIDTH_SHIFT;
4636         link_width_cntl |= (RADEON_PCIE_LC_RECONFIG_NOW |
4637                             R600_PCIE_LC_RECONFIG_ARC_MISSING_ESCAPE);
4638
4639         WREG32_PCIE_PORT(RADEON_PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4640 }
4641
4642 int r600_get_pcie_lanes(struct radeon_device *rdev)
4643 {
4644         u32 link_width_cntl;
4645
4646         if (rdev->flags & RADEON_IS_IGP)
4647                 return 0;
4648
4649         if (!(rdev->flags & RADEON_IS_PCIE))
4650                 return 0;
4651
4652         /* x2 cards have a special sequence */
4653         if (ASIC_IS_X2(rdev))
4654                 return 0;
4655
4656         radeon_gui_idle(rdev);
4657
4658         link_width_cntl = RREG32_PCIE_PORT(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
4659
4660         switch ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) >> RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT) {
4661         case RADEON_PCIE_LC_LINK_WIDTH_X1:
4662                 return 1;
4663         case RADEON_PCIE_LC_LINK_WIDTH_X2:
4664                 return 2;
4665         case RADEON_PCIE_LC_LINK_WIDTH_X4:
4666                 return 4;
4667         case RADEON_PCIE_LC_LINK_WIDTH_X8:
4668                 return 8;
4669         case RADEON_PCIE_LC_LINK_WIDTH_X12:
4670                 /* not actually supported */
4671                 return 12;
4672         case RADEON_PCIE_LC_LINK_WIDTH_X0:
4673         case RADEON_PCIE_LC_LINK_WIDTH_X16:
4674         default:
4675                 return 16;
4676         }
4677 }
4678
4679 static void r600_pcie_gen2_enable(struct radeon_device *rdev)
4680 {
4681         u32 link_width_cntl, lanes, speed_cntl, training_cntl, tmp;
4682         u16 link_cntl2;
4683
4684         if (radeon_pcie_gen2 == 0)
4685                 return;
4686
4687         if (rdev->flags & RADEON_IS_IGP)
4688                 return;
4689
4690         if (!(rdev->flags & RADEON_IS_PCIE))
4691                 return;
4692
4693         /* x2 cards have a special sequence */
4694         if (ASIC_IS_X2(rdev))
4695                 return;
4696
4697         /* only RV6xx+ chips are supported */
4698         if (rdev->family <= CHIP_R600)
4699                 return;
4700
4701         if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
4702                 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
4703                 return;
4704
4705         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
4706         if (speed_cntl & LC_CURRENT_DATA_RATE) {
4707                 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
4708                 return;
4709         }
4710
4711         DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
4712
4713         /* 55 nm r6xx asics */
4714         if ((rdev->family == CHIP_RV670) ||
4715             (rdev->family == CHIP_RV620) ||
4716             (rdev->family == CHIP_RV635)) {
4717                 /* advertise upconfig capability */
4718                 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
4719                 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
4720                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4721                 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
4722                 if (link_width_cntl & LC_RENEGOTIATION_SUPPORT) {
4723                         lanes = (link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT;
4724                         link_width_cntl &= ~(LC_LINK_WIDTH_MASK |
4725                                              LC_RECONFIG_ARC_MISSING_ESCAPE);
4726                         link_width_cntl |= lanes | LC_RECONFIG_NOW | LC_RENEGOTIATE_EN;
4727                         WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4728                 } else {
4729                         link_width_cntl |= LC_UPCONFIGURE_DIS;
4730                         WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4731                 }
4732         }
4733
4734         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
4735         if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
4736             (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
4737
4738                 /* 55 nm r6xx asics */
4739                 if ((rdev->family == CHIP_RV670) ||
4740                     (rdev->family == CHIP_RV620) ||
4741                     (rdev->family == CHIP_RV635)) {
4742                         WREG32(MM_CFGREGS_CNTL, 0x8);
4743                         link_cntl2 = RREG32(0x4088);
4744                         WREG32(MM_CFGREGS_CNTL, 0);
4745                         /* not supported yet */
4746                         if (link_cntl2 & SELECTABLE_DEEMPHASIS)
4747                                 return;
4748                 }
4749
4750                 speed_cntl &= ~LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK;
4751                 speed_cntl |= (0x3 << LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT);
4752                 speed_cntl &= ~LC_VOLTAGE_TIMER_SEL_MASK;
4753                 speed_cntl &= ~LC_FORCE_DIS_HW_SPEED_CHANGE;
4754                 speed_cntl |= LC_FORCE_EN_HW_SPEED_CHANGE;
4755                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
4756
4757                 tmp = RREG32(0x541c);
4758                 WREG32(0x541c, tmp | 0x8);
4759                 WREG32(MM_CFGREGS_CNTL, MM_WR_TO_CFG_EN);
4760                 link_cntl2 = RREG16(0x4088);
4761                 link_cntl2 &= ~TARGET_LINK_SPEED_MASK;
4762                 link_cntl2 |= 0x2;
4763                 WREG16(0x4088, link_cntl2);
4764                 WREG32(MM_CFGREGS_CNTL, 0);
4765
4766                 if ((rdev->family == CHIP_RV670) ||
4767                     (rdev->family == CHIP_RV620) ||
4768                     (rdev->family == CHIP_RV635)) {
4769                         training_cntl = RREG32_PCIE_PORT(PCIE_LC_TRAINING_CNTL);
4770                         training_cntl &= ~LC_POINT_7_PLUS_EN;
4771                         WREG32_PCIE_PORT(PCIE_LC_TRAINING_CNTL, training_cntl);
4772                 } else {
4773                         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
4774                         speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
4775                         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
4776                 }
4777
4778                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
4779                 speed_cntl |= LC_GEN2_EN_STRAP;
4780                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
4781
4782         } else {
4783                 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
4784                 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
4785                 if (1)
4786                         link_width_cntl |= LC_UPCONFIGURE_DIS;
4787                 else
4788                         link_width_cntl &= ~LC_UPCONFIGURE_DIS;
4789                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4790         }
4791 }
4792
4793 /**
4794  * r600_get_gpu_clock_counter - return GPU clock counter snapshot
4795  *
4796  * @rdev: radeon_device pointer
4797  *
4798  * Fetches a GPU clock counter snapshot (R6xx-cayman).
4799  * Returns the 64 bit clock counter snapshot.
4800  */
4801 uint64_t r600_get_gpu_clock_counter(struct radeon_device *rdev)
4802 {
4803         uint64_t clock;
4804
4805         mutex_lock(&rdev->gpu_clock_mutex);
4806         WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
4807         clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
4808                 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
4809         mutex_unlock(&rdev->gpu_clock_mutex);
4810         return clock;
4811 }