drivers/gpu/drm/i915/i915_irq.c

   1 /* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
   2  */
   3 /*
   4  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sub license, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the
  16  * next paragraph) shall be included in all copies or substantial portions
  17  * of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  20  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  21  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  22  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  23  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  24  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  25  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  */
  28
  29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  30
  31 #include <linux/sysrq.h>
  32 #include <linux/slab.h>
  33 #include <drm/drmP.h>
  34 #include <drm/i915_drm.h>
  35 #include "i915_drv.h"
  36 #include "i915_trace.h"
  37 #include "intel_drv.h"
  38
  39 static const u32 hpd_ibx[] = {
  40         [HPD_CRT] = SDE_CRT_HOTPLUG,
  41         [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
  42         [HPD_PORT_B] = SDE_PORTB_HOTPLUG,
  43         [HPD_PORT_C] = SDE_PORTC_HOTPLUG,
  44         [HPD_PORT_D] = SDE_PORTD_HOTPLUG
  45 };
  46
  47 static const u32 hpd_cpt[] = {
  48         [HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
  49         [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
  50         [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
  51         [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
  52         [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT
  53 };
  54
  55 static const u32 hpd_mask_i915[] = {
  56         [HPD_CRT] = CRT_HOTPLUG_INT_EN,
  57         [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
  58         [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
  59         [HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
  60         [HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
  61         [HPD_PORT_D] = PORTD_HOTPLUG_INT_EN
  62 };
  63
  64 static const u32 hpd_status_gen4[] = {
  65         [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
  66         [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
  67         [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
  68         [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
  69         [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
  70         [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
  71 };
  72
  73 static const u32 hpd_status_i965[] = {
  74          [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
  75          [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I965,
  76          [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I965,
  77          [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
  78          [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
  79          [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
  80 };
  81
  82 static const u32 hpd_status_i915[] = { /* i915 and valleyview are the same */
  83         [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
  84         [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
  85         [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
  86         [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
  87         [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
  88         [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
  89 };
  90
  91 static void ibx_hpd_irq_setup(struct drm_device *dev);
  92 static void i915_hpd_irq_setup(struct drm_device *dev);
  93
  94 /* For display hotplug interrupt */
  95 static void
  96 ironlake_enable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
  97 {
  98         if ((dev_priv->irq_mask & mask) != 0) {
  99                 dev_priv->irq_mask &= ~mask;
 100                 I915_WRITE(DEIMR, dev_priv->irq_mask);
 101                 POSTING_READ(DEIMR);
 102         }
 103 }
 104
 105 static void
 106 ironlake_disable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
 107 {
 108         if ((dev_priv->irq_mask & mask) != mask) {
 109                 dev_priv->irq_mask |= mask;
 110                 I915_WRITE(DEIMR, dev_priv->irq_mask);
 111                 POSTING_READ(DEIMR);
 112         }
 113 }
 114
 115 static bool ivb_can_enable_err_int(struct drm_device *dev)
 116 {
 117         struct drm_i915_private *dev_priv = dev->dev_private;
 118         struct intel_crtc *crtc;
 119         enum pipe pipe;
 120
 121         for_each_pipe(pipe) {
 122                 crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
 123
 124                 if (crtc->cpu_fifo_underrun_disabled)
 125                         return false;
 126         }
 127
 128         return true;
 129 }
 130
 131 static bool cpt_can_enable_serr_int(struct drm_device *dev)
 132 {
 133         struct drm_i915_private *dev_priv = dev->dev_private;
 134         enum pipe pipe;
 135         struct intel_crtc *crtc;
 136
 137         for_each_pipe(pipe) {
 138                 crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
 139
 140                 if (crtc->pch_fifo_underrun_disabled)
 141                         return false;
 142         }
 143
 144         return true;
 145 }
 146
 147 static void ironlake_set_fifo_underrun_reporting(struct drm_device *dev,
 148                                                  enum pipe pipe, bool enable)
 149 {
 150         struct drm_i915_private *dev_priv = dev->dev_private;
 151         uint32_t bit = (pipe == PIPE_A) ? DE_PIPEA_FIFO_UNDERRUN :
 152                                           DE_PIPEB_FIFO_UNDERRUN;
 153
 154         if (enable)
 155                 ironlake_enable_display_irq(dev_priv, bit);
 156         else
 157                 ironlake_disable_display_irq(dev_priv, bit);
 158 }
 159
 160 static void ivybridge_set_fifo_underrun_reporting(struct drm_device *dev,
 161                                                   bool enable)
 162 {
 163         struct drm_i915_private *dev_priv = dev->dev_private;
 164
 165         if (enable) {
 166                 if (!ivb_can_enable_err_int(dev))
 167                         return;
 168
 169                 I915_WRITE(GEN7_ERR_INT, ERR_INT_FIFO_UNDERRUN_A |
 170                                          ERR_INT_FIFO_UNDERRUN_B |
 171                                          ERR_INT_FIFO_UNDERRUN_C);
 172
 173                 ironlake_enable_display_irq(dev_priv, DE_ERR_INT_IVB);
 174         } else {
 175                 ironlake_disable_display_irq(dev_priv, DE_ERR_INT_IVB);
 176         }
 177 }
 178
 179 static void ibx_set_fifo_underrun_reporting(struct intel_crtc *crtc,
 180                                             bool enable)
 181 {
 182         struct drm_device *dev = crtc->base.dev;
 183         struct drm_i915_private *dev_priv = dev->dev_private;
 184         uint32_t bit = (crtc->pipe == PIPE_A) ? SDE_TRANSA_FIFO_UNDER :
 185                                                 SDE_TRANSB_FIFO_UNDER;
 186
 187         if (enable)
 188                 I915_WRITE(SDEIMR, I915_READ(SDEIMR) & ~bit);
 189         else
 190                 I915_WRITE(SDEIMR, I915_READ(SDEIMR) | bit);
 191
 192         POSTING_READ(SDEIMR);
 193 }
 194
 195 static void cpt_set_fifo_underrun_reporting(struct drm_device *dev,
 196                                             enum transcoder pch_transcoder,
 197                                             bool enable)
 198 {
 199         struct drm_i915_private *dev_priv = dev->dev_private;
 200
 201         if (enable) {
 202                 if (!cpt_can_enable_serr_int(dev))
 203                         return;
 204
 205                 I915_WRITE(SERR_INT, SERR_INT_TRANS_A_FIFO_UNDERRUN |
 206                                      SERR_INT_TRANS_B_FIFO_UNDERRUN |
 207                                      SERR_INT_TRANS_C_FIFO_UNDERRUN);
 208
 209                 I915_WRITE(SDEIMR, I915_READ(SDEIMR) & ~SDE_ERROR_CPT);
 210         } else {
 211                 I915_WRITE(SDEIMR, I915_READ(SDEIMR) | SDE_ERROR_CPT);
 212         }
 213
 214         POSTING_READ(SDEIMR);
 215 }
 216
 217 /**
 218  * intel_set_cpu_fifo_underrun_reporting - enable/disable FIFO underrun messages
 219  * @dev: drm device
 220  * @pipe: pipe
 221  * @enable: true if we want to report FIFO underrun errors, false otherwise
 222  *
 223  * This function makes us disable or enable CPU fifo underruns for a specific
 224  * pipe. Notice that on some Gens (e.g. IVB, HSW), disabling FIFO underrun
 225  * reporting for one pipe may also disable all the other CPU error interruts for
 226  * the other pipes, due to the fact that there's just one interrupt mask/enable
 227  * bit for all the pipes.
 228  *
 229  * Returns the previous state of underrun reporting.
 230  */
 231 bool intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev,
 232                                            enum pipe pipe, bool enable)
 233 {
 234         struct drm_i915_private *dev_priv = dev->dev_private;
 235         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
 236         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 237         unsigned long flags;
 238         bool ret;
 239
 240         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 241
 242         ret = !intel_crtc->cpu_fifo_underrun_disabled;
 243
 244         if (enable == ret)
 245                 goto done;
 246
 247         intel_crtc->cpu_fifo_underrun_disabled = !enable;
 248
 249         if (IS_GEN5(dev) || IS_GEN6(dev))
 250                 ironlake_set_fifo_underrun_reporting(dev, pipe, enable);
 251         else if (IS_GEN7(dev))
 252                 ivybridge_set_fifo_underrun_reporting(dev, enable);
 253
 254 done:
 255         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 256         return ret;
 257 }
 258
 259 /**
 260  * intel_set_pch_fifo_underrun_reporting - enable/disable FIFO underrun messages
 261  * @dev: drm device
 262  * @pch_transcoder: the PCH transcoder (same as pipe on IVB and older)
 263  * @enable: true if we want to report FIFO underrun errors, false otherwise
 264  *
 265  * This function makes us disable or enable PCH fifo underruns for a specific
 266  * PCH transcoder. Notice that on some PCHs (e.g. CPT/PPT), disabling FIFO
 267  * underrun reporting for one transcoder may also disable all the other PCH
 268  * error interruts for the other transcoders, due to the fact that there's just
 269  * one interrupt mask/enable bit for all the transcoders.
 270  *
 271  * Returns the previous state of underrun reporting.
 272  */
 273 bool intel_set_pch_fifo_underrun_reporting(struct drm_device *dev,
 274                                            enum transcoder pch_transcoder,
 275                                            bool enable)
 276 {
 277         struct drm_i915_private *dev_priv = dev->dev_private;
 278         enum pipe p;
 279         struct drm_crtc *crtc;
 280         struct intel_crtc *intel_crtc;
 281         unsigned long flags;
 282         bool ret;
 283
 284         if (HAS_PCH_LPT(dev)) {
 285                 crtc = NULL;
 286                 for_each_pipe(p) {
 287                         struct drm_crtc *c = dev_priv->pipe_to_crtc_mapping[p];
 288                         if (intel_pipe_has_type(c, INTEL_OUTPUT_ANALOG)) {
 289                                 crtc = c;
 290                                 break;
 291                         }
 292                 }
 293                 if (!crtc) {
 294                         DRM_ERROR("PCH FIFO underrun, but no CRTC using the PCH found\n");
 295                         return false;
 296                 }
 297         } else {
 298                 crtc = dev_priv->pipe_to_crtc_mapping[pch_transcoder];
 299         }
 300         intel_crtc = to_intel_crtc(crtc);
 301
 302         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 303
 304         ret = !intel_crtc->pch_fifo_underrun_disabled;
 305
 306         if (enable == ret)
 307                 goto done;
 308
 309         intel_crtc->pch_fifo_underrun_disabled = !enable;
 310
 311         if (HAS_PCH_IBX(dev))
 312                 ibx_set_fifo_underrun_reporting(intel_crtc, enable);
 313         else
 314                 cpt_set_fifo_underrun_reporting(dev, pch_transcoder, enable);
 315
 316 done:
 317         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 318         return ret;
 319 }
 320
 321
 322 void
 323 i915_enable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
 324 {
 325         u32 reg = PIPESTAT(pipe);
 326         u32 pipestat = I915_READ(reg) & 0x7fff0000;
 327
 328         if ((pipestat & mask) == mask)
 329                 return;
 330
 331         /* Enable the interrupt, clear any pending status */
 332         pipestat |= mask | (mask >> 16);
 333         I915_WRITE(reg, pipestat);
 334         POSTING_READ(reg);
 335 }
 336
 337 void
 338 i915_disable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
 339 {
 340         u32 reg = PIPESTAT(pipe);
 341         u32 pipestat = I915_READ(reg) & 0x7fff0000;
 342
 343         if ((pipestat & mask) == 0)
 344                 return;
 345
 346         pipestat &= ~mask;
 347         I915_WRITE(reg, pipestat);
 348         POSTING_READ(reg);
 349 }
 350
 351 /**
 352  * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
 353  */
 354 static void i915_enable_asle_pipestat(struct drm_device *dev)
 355 {
 356         drm_i915_private_t *dev_priv = dev->dev_private;
 357         unsigned long irqflags;
 358
 359         if (!dev_priv->opregion.asle || !IS_MOBILE(dev))
 360                 return;
 361
 362         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
 363
 364         i915_enable_pipestat(dev_priv, 1, PIPE_LEGACY_BLC_EVENT_ENABLE);
 365         if (INTEL_INFO(dev)->gen >= 4)
 366                 i915_enable_pipestat(dev_priv, 0, PIPE_LEGACY_BLC_EVENT_ENABLE);
 367
 368         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 369 }
 370
 371 /**
 372  * i915_pipe_enabled - check if a pipe is enabled
 373  * @dev: DRM device
 374  * @pipe: pipe to check
 375  *
 376  * Reading certain registers when the pipe is disabled can hang the chip.
 377  * Use this routine to make sure the PLL is running and the pipe is active
 378  * before reading such registers if unsure.
 379  */
 380 static int
 381 i915_pipe_enabled(struct drm_device *dev, int pipe)
 382 {
 383         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 384
 385         if (drm_core_check_feature(dev, DRIVER_MODESET)) {
 386                 /* Locking is horribly broken here, but whatever. */
 387                 struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
 388                 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 389
 390                 return intel_crtc->active;
 391         } else {
 392                 return I915_READ(PIPECONF(pipe)) & PIPECONF_ENABLE;
 393         }
 394 }
 395
 396 /* Called from drm generic code, passed a 'crtc', which
 397  * we use as a pipe index
 398  */
 399 static u32 i915_get_vblank_counter(struct drm_device *dev, int pipe)
 400 {
 401         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 402         unsigned long high_frame;
 403         unsigned long low_frame;
 404         u32 high1, high2, low;
 405
 406         if (!i915_pipe_enabled(dev, pipe)) {
 407                 DRM_DEBUG_DRIVER("trying to get vblank count for disabled "
 408                                 "pipe %c\n", pipe_name(pipe));
 409                 return 0;
 410         }
 411
 412         high_frame = PIPEFRAME(pipe);
 413         low_frame = PIPEFRAMEPIXEL(pipe);
 414
 415         /*
 416          * High & low register fields aren't synchronized, so make sure
 417          * we get a low value that's stable across two reads of the high
 418          * register.
 419          */
 420         do {
 421                 high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
 422                 low   = I915_READ(low_frame)  & PIPE_FRAME_LOW_MASK;
 423                 high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
 424         } while (high1 != high2);
 425
 426         high1 >>= PIPE_FRAME_HIGH_SHIFT;
 427         low >>= PIPE_FRAME_LOW_SHIFT;
 428         return (high1 << 8) | low;
 429 }
 430
 431 static u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe)
 432 {
 433         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 434         int reg = PIPE_FRMCOUNT_GM45(pipe);
 435
 436         if (!i915_pipe_enabled(dev, pipe)) {
 437                 DRM_DEBUG_DRIVER("trying to get vblank count for disabled "
 438                                  "pipe %c\n", pipe_name(pipe));
 439                 return 0;
 440         }
 441
 442         return I915_READ(reg);
 443 }
 444
 445 static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe,
 446                              int *vpos, int *hpos)
 447 {
 448         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 449         u32 vbl = 0, position = 0;
 450         int vbl_start, vbl_end, htotal, vtotal;
 451         bool in_vbl = true;
 452         int ret = 0;
 453         enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
 454                                                                       pipe);
 455
 456         if (!i915_pipe_enabled(dev, pipe)) {
 457                 DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
 458                                  "pipe %c\n", pipe_name(pipe));
 459                 return 0;
 460         }
 461
 462         /* Get vtotal. */
 463         vtotal = 1 + ((I915_READ(VTOTAL(cpu_transcoder)) >> 16) & 0x1fff);
 464
 465         if (INTEL_INFO(dev)->gen >= 4) {
 466                 /* No obvious pixelcount register. Only query vertical
 467                  * scanout position from Display scan line register.
 468                  */
 469                 position = I915_READ(PIPEDSL(pipe));
 470
 471                 /* Decode into vertical scanout position. Don't have
 472                  * horizontal scanout position.
 473                  */
 474                 *vpos = position & 0x1fff;
 475                 *hpos = 0;
 476         } else {
 477                 /* Have access to pixelcount since start of frame.
 478                  * We can split this into vertical and horizontal
 479                  * scanout position.
 480                  */
 481                 position = (I915_READ(PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
 482
 483                 htotal = 1 + ((I915_READ(HTOTAL(cpu_transcoder)) >> 16) & 0x1fff);
 484                 *vpos = position / htotal;
 485                 *hpos = position - (*vpos * htotal);
 486         }
 487
 488         /* Query vblank area. */
 489         vbl = I915_READ(VBLANK(cpu_transcoder));
 490
 491         /* Test position against vblank region. */
 492         vbl_start = vbl & 0x1fff;
 493         vbl_end = (vbl >> 16) & 0x1fff;
 494
 495         if ((*vpos < vbl_start) || (*vpos > vbl_end))
 496                 in_vbl = false;
 497
 498         /* Inside "upper part" of vblank area? Apply corrective offset: */
 499         if (in_vbl && (*vpos >= vbl_start))
 500                 *vpos = *vpos - vtotal;
 501
 502         /* Readouts valid? */
 503         if (vbl > 0)
 504                 ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;
 505
 506         /* In vblank? */
 507         if (in_vbl)
 508                 ret |= DRM_SCANOUTPOS_INVBL;
 509
 510         return ret;
 511 }
 512
 513 static int i915_get_vblank_timestamp(struct drm_device *dev, int pipe,
 514                               int *max_error,
 515                               struct timeval *vblank_time,
 516                               unsigned flags)
 517 {
 518         struct drm_crtc *crtc;
 519
 520         if (pipe < 0 || pipe >= INTEL_INFO(dev)->num_pipes) {
 521                 DRM_ERROR("Invalid crtc %d\n", pipe);
 522                 return -EINVAL;
 523         }
 524
 525         /* Get drm_crtc to timestamp: */
 526         crtc = intel_get_crtc_for_pipe(dev, pipe);
 527         if (crtc == NULL) {
 528                 DRM_ERROR("Invalid crtc %d\n", pipe);
 529                 return -EINVAL;
 530         }
 531
 532         if (!crtc->enabled) {
 533                 DRM_DEBUG_KMS("crtc %d is disabled\n", pipe);
 534                 return -EBUSY;
 535         }
 536
 537         /* Helper routine in DRM core does all the work: */
 538         return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
 539                                                      vblank_time, flags,
 540                                                      crtc);
 541 }
 542
 543 static int intel_hpd_irq_event(struct drm_device *dev, struct drm_connector *connector)
 544 {
 545         enum drm_connector_status old_status;
 546
 547         WARN_ON(!mutex_is_locked(&dev->mode_config.mutex));
 548         old_status = connector->status;
 549
 550         connector->status = connector->funcs->detect(connector, false);
 551         DRM_DEBUG_KMS("[CONNECTOR:%d:%s] status updated from %d to %d\n",
 552                       connector->base.id,
 553                       drm_get_connector_name(connector),
 554                       old_status, connector->status);
 555         return (old_status != connector->status);
 556 }
 557
 558 /*
 559  * Handle hotplug events outside the interrupt handler proper.
 560  */
 561 #define I915_REENABLE_HOTPLUG_DELAY (2*60*1000)
 562
 563 static void i915_hotplug_work_func(struct work_struct *work)
 564 {
 565         drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
 566                                                     hotplug_work);
 567         struct drm_device *dev = dev_priv->dev;
 568         struct drm_mode_config *mode_config = &dev->mode_config;
 569         struct intel_connector *intel_connector;
 570         struct intel_encoder *intel_encoder;
 571         struct drm_connector *connector;
 572         unsigned long irqflags;
 573         bool hpd_disabled = false;
 574         bool changed = false;
 575         u32 hpd_event_bits;
 576
 577         /* HPD irq before everything is fully set up. */
 578         if (!dev_priv->enable_hotplug_processing)
 579                 return;
 580
 581         mutex_lock(&mode_config->mutex);
 582         DRM_DEBUG_KMS("running encoder hotplug functions\n");
 583
 584         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
 585
 586         hpd_event_bits = dev_priv->hpd_event_bits;
 587         dev_priv->hpd_event_bits = 0;
 588         list_for_each_entry(connector, &mode_config->connector_list, head) {
 589                 intel_connector = to_intel_connector(connector);
 590                 intel_encoder = intel_connector->encoder;
 591                 if (intel_encoder->hpd_pin > HPD_NONE &&
 592                     dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_MARK_DISABLED &&
 593                     connector->polled == DRM_CONNECTOR_POLL_HPD) {
 594                         DRM_INFO("HPD interrupt storm detected on connector %s: "
 595                                  "switching from hotplug detection to polling\n",
 596                                 drm_get_connector_name(connector));
 597                         dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark = HPD_DISABLED;
 598                         connector->polled = DRM_CONNECTOR_POLL_CONNECT
 599                                 | DRM_CONNECTOR_POLL_DISCONNECT;
 600                         hpd_disabled = true;
 601                 }
 602                 if (hpd_event_bits & (1 << intel_encoder->hpd_pin)) {
 603                         DRM_DEBUG_KMS("Connector %s (pin %i) received hotplug event.\n",
 604                                       drm_get_connector_name(connector), intel_encoder->hpd_pin);
 605                 }
 606         }
 607          /* if there were no outputs to poll, poll was disabled,
 608           * therefore make sure it's enabled when disabling HPD on
 609           * some connectors */
 610         if (hpd_disabled) {
 611                 drm_kms_helper_poll_enable(dev);
 612                 mod_timer(&dev_priv->hotplug_reenable_timer,
 613                           jiffies + msecs_to_jiffies(I915_REENABLE_HOTPLUG_DELAY));
 614         }
 615
 616         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 617
 618         list_for_each_entry(connector, &mode_config->connector_list, head) {
 619                 intel_connector = to_intel_connector(connector);
 620                 intel_encoder = intel_connector->encoder;
 621                 if (hpd_event_bits & (1 << intel_encoder->hpd_pin)) {
 622                         if (intel_encoder->hot_plug)
 623                                 intel_encoder->hot_plug(intel_encoder);
 624                         if (intel_hpd_irq_event(dev, connector))
 625                                 changed = true;
 626                 }
 627         }
 628         mutex_unlock(&mode_config->mutex);
 629
 630         if (changed)
 631                 drm_kms_helper_hotplug_event(dev);
 632 }
 633
 634 static void ironlake_handle_rps_change(struct drm_device *dev)
 635 {
 636         drm_i915_private_t *dev_priv = dev->dev_private;
 637         u32 busy_up, busy_down, max_avg, min_avg;
 638         u8 new_delay;
 639         unsigned long flags;
 640
 641         spin_lock_irqsave(&mchdev_lock, flags);
 642
 643         I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));
 644
 645         new_delay = dev_priv->ips.cur_delay;
 646
 647         I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
 648         busy_up = I915_READ(RCPREVBSYTUPAVG);
 649         busy_down = I915_READ(RCPREVBSYTDNAVG);
 650         max_avg = I915_READ(RCBMAXAVG);
 651         min_avg = I915_READ(RCBMINAVG);
 652
 653         /* Handle RCS change request from hw */
 654         if (busy_up > max_avg) {
 655                 if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
 656                         new_delay = dev_priv->ips.cur_delay - 1;
 657                 if (new_delay < dev_priv->ips.max_delay)
 658                         new_delay = dev_priv->ips.max_delay;
 659         } else if (busy_down < min_avg) {
 660                 if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
 661                         new_delay = dev_priv->ips.cur_delay + 1;
 662                 if (new_delay > dev_priv->ips.min_delay)
 663                         new_delay = dev_priv->ips.min_delay;
 664         }
 665
 666         if (ironlake_set_drps(dev, new_delay))
 667                 dev_priv->ips.cur_delay = new_delay;
 668
 669         spin_unlock_irqrestore(&mchdev_lock, flags);
 670
 671         return;
 672 }
 673
 674 static void notify_ring(struct drm_device *dev,
 675                         struct intel_ring_buffer *ring)
 676 {
 677         struct drm_i915_private *dev_priv = dev->dev_private;
 678
 679         if (ring->obj == NULL)
 680                 return;
 681
 682         trace_i915_gem_request_complete(ring, ring->get_seqno(ring, false));
 683
 684         wake_up_all(&ring->irq_queue);
 685         if (i915_enable_hangcheck) {
 686                 dev_priv->gpu_error.hangcheck_count = 0;
 687                 mod_timer(&dev_priv->gpu_error.hangcheck_timer,
 688                           round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES));
 689         }
 690 }
 691
 692 static void gen6_pm_rps_work(struct work_struct *work)
 693 {
 694         drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
 695                                                     rps.work);
 696         u32 pm_iir, pm_imr;
 697         u8 new_delay;
 698
 699         spin_lock_irq(&dev_priv->rps.lock);
 700         pm_iir = dev_priv->rps.pm_iir;
 701         dev_priv->rps.pm_iir = 0;
 702         pm_imr = I915_READ(GEN6_PMIMR);
 703         I915_WRITE(GEN6_PMIMR, 0);
 704         spin_unlock_irq(&dev_priv->rps.lock);
 705
 706         if ((pm_iir & GEN6_PM_DEFERRED_EVENTS) == 0)
 707                 return;
 708
 709         mutex_lock(&dev_priv->rps.hw_lock);
 710
 711         if (pm_iir & GEN6_PM_RP_UP_THRESHOLD)
 712                 new_delay = dev_priv->rps.cur_delay + 1;
 713         else
 714                 new_delay = dev_priv->rps.cur_delay - 1;
 715
 716         /* sysfs frequency interfaces may have snuck in while servicing the
 717          * interrupt
 718          */
 719         if (!(new_delay > dev_priv->rps.max_delay ||
 720               new_delay < dev_priv->rps.min_delay)) {
 721                 if (IS_VALLEYVIEW(dev_priv->dev))
 722                         valleyview_set_rps(dev_priv->dev, new_delay);
 723                 else
 724                         gen6_set_rps(dev_priv->dev, new_delay);
 725         }
 726
 727         if (IS_VALLEYVIEW(dev_priv->dev)) {
 728                 /*
 729                  * On VLV, when we enter RC6 we may not be at the minimum
 730                  * voltage level, so arm a timer to check.  It should only
 731                  * fire when there's activity or once after we've entered
 732                  * RC6, and then won't be re-armed until the next RPS interrupt.
 733                  */
 734                 mod_delayed_work(dev_priv->wq, &dev_priv->rps.vlv_work,
 735                                  msecs_to_jiffies(100));
 736         }
 737
 738         mutex_unlock(&dev_priv->rps.hw_lock);
 739 }
 740
 741
 742 /**
 743  * ivybridge_parity_work - Workqueue called when a parity error interrupt
 744  * occurred.
 745  * @work: workqueue struct
 746  *
 747  * Doesn't actually do anything except notify userspace. As a consequence of
 748  * this event, userspace should try to remap the bad rows since statistically
 749  * it is likely the same row is more likely to go bad again.
 750  */
 751 static void ivybridge_parity_work(struct work_struct *work)
 752 {
 753         drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
 754                                                     l3_parity.error_work);
 755         u32 error_status, row, bank, subbank;
 756         char *parity_event[5];
 757         uint32_t misccpctl;
 758         unsigned long flags;
 759
 760         /* We must turn off DOP level clock gating to access the L3 registers.
 761          * In order to prevent a get/put style interface, acquire struct mutex
 762          * any time we access those registers.
 763          */
 764         mutex_lock(&dev_priv->dev->struct_mutex);
 765
 766         misccpctl = I915_READ(GEN7_MISCCPCTL);
 767         I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
 768         POSTING_READ(GEN7_MISCCPCTL);
 769
 770         error_status = I915_READ(GEN7_L3CDERRST1);
 771         row = GEN7_PARITY_ERROR_ROW(error_status);
 772         bank = GEN7_PARITY_ERROR_BANK(error_status);
 773         subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);
 774
 775         I915_WRITE(GEN7_L3CDERRST1, GEN7_PARITY_ERROR_VALID |
 776                                     GEN7_L3CDERRST1_ENABLE);
 777         POSTING_READ(GEN7_L3CDERRST1);
 778
 779         I915_WRITE(GEN7_MISCCPCTL, misccpctl);
 780
 781         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 782         dev_priv->gt_irq_mask &= ~GT_GEN7_L3_PARITY_ERROR_INTERRUPT;
 783         I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
 784         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 785
 786         mutex_unlock(&dev_priv->dev->struct_mutex);
 787
 788         parity_event[0] = "L3_PARITY_ERROR=1";
 789         parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
 790         parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
 791         parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
 792         parity_event[4] = NULL;
 793
 794         kobject_uevent_env(&dev_priv->dev->primary->kdev.kobj,
 795                            KOBJ_CHANGE, parity_event);
 796
 797         DRM_DEBUG("Parity error: Row = %d, Bank = %d, Sub bank = %d.\n",
 798                   row, bank, subbank);
 799
 800         kfree(parity_event[3]);
 801         kfree(parity_event[2]);
 802         kfree(parity_event[1]);
 803 }
 804
 805 static void ivybridge_handle_parity_error(struct drm_device *dev)
 806 {
 807         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 808         unsigned long flags;
 809
 810         if (!HAS_L3_GPU_CACHE(dev))
 811                 return;
 812
 813         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 814         dev_priv->gt_irq_mask |= GT_GEN7_L3_PARITY_ERROR_INTERRUPT;
 815         I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
 816         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 817
 818         queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work);
 819 }
 820
 821 static void snb_gt_irq_handler(struct drm_device *dev,
 822                                struct drm_i915_private *dev_priv,
 823                                u32 gt_iir)
 824 {
 825
 826         if (gt_iir & (GEN6_RENDER_USER_INTERRUPT |
 827                       GEN6_RENDER_PIPE_CONTROL_NOTIFY_INTERRUPT))
 828                 notify_ring(dev, &dev_priv->ring[RCS]);
 829         if (gt_iir & GEN6_BSD_USER_INTERRUPT)
 830                 notify_ring(dev, &dev_priv->ring[VCS]);
 831         if (gt_iir & GEN6_BLITTER_USER_INTERRUPT)
 832                 notify_ring(dev, &dev_priv->ring[BCS]);
 833
 834         if (gt_iir & (GT_GEN6_BLT_CS_ERROR_INTERRUPT |
 835                       GT_GEN6_BSD_CS_ERROR_INTERRUPT |
 836                       GT_RENDER_CS_ERROR_INTERRUPT)) {
 837                 DRM_ERROR("GT error interrupt 0x%08x\n", gt_iir);
 838                 i915_handle_error(dev, false);
 839         }
 840
 841         if (gt_iir & GT_GEN7_L3_PARITY_ERROR_INTERRUPT)
 842                 ivybridge_handle_parity_error(dev);
 843 }
 844
 845 static void gen6_queue_rps_work(struct drm_i915_private *dev_priv,
 846                                 u32 pm_iir)
 847 {
 848         unsigned long flags;
 849
 850         /*
 851          * IIR bits should never already be set because IMR should
 852          * prevent an interrupt from being shown in IIR. The warning
 853          * displays a case where we've unsafely cleared
 854          * dev_priv->rps.pm_iir. Although missing an interrupt of the same
 855          * type is not a problem, it displays a problem in the logic.
 856          *
 857          * The mask bit in IMR is cleared by dev_priv->rps.work.
 858          */
 859
 860         spin_lock_irqsave(&dev_priv->rps.lock, flags);
 861         dev_priv->rps.pm_iir |= pm_iir;
 862         I915_WRITE(GEN6_PMIMR, dev_priv->rps.pm_iir);
 863         POSTING_READ(GEN6_PMIMR);
 864         spin_unlock_irqrestore(&dev_priv->rps.lock, flags);
 865
 866         queue_work(dev_priv->wq, &dev_priv->rps.work);
 867 }
 868
 869 #define HPD_STORM_DETECT_PERIOD 1000
 870 #define HPD_STORM_THRESHOLD 5
 871
 872 static inline bool hotplug_irq_storm_detect(struct drm_device *dev,
 873                                             u32 hotplug_trigger,
 874                                             const u32 *hpd)
 875 {
 876         drm_i915_private_t *dev_priv = dev->dev_private;
 877         unsigned long irqflags;
 878         int i;
 879         bool ret = false;
 880
 881         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
 882
 883         for (i = 1; i < HPD_NUM_PINS; i++) {
 884
 885                 if (!(hpd[i] & hotplug_trigger) ||
 886                     dev_priv->hpd_stats[i].hpd_mark != HPD_ENABLED)
 887                         continue;
 888
 889                 dev_priv->hpd_event_bits |= (1 << i);
 890                 if (!time_in_range(jiffies, dev_priv->hpd_stats[i].hpd_last_jiffies,
 891                                    dev_priv->hpd_stats[i].hpd_last_jiffies
 892                                    + msecs_to_jiffies(HPD_STORM_DETECT_PERIOD))) {
 893                         dev_priv->hpd_stats[i].hpd_last_jiffies = jiffies;
 894                         dev_priv->hpd_stats[i].hpd_cnt = 0;
 895                 } else if (dev_priv->hpd_stats[i].hpd_cnt > HPD_STORM_THRESHOLD) {
 896                         dev_priv->hpd_stats[i].hpd_mark = HPD_MARK_DISABLED;
 897                         dev_priv->hpd_event_bits &= ~(1 << i);
 898                         DRM_DEBUG_KMS("HPD interrupt storm detected on PIN %d\n", i);
 899                         ret = true;
 900                 } else {
 901                         dev_priv->hpd_stats[i].hpd_cnt++;
 902                 }
 903         }
 904
 905         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 906
 907         return ret;
 908 }
 909
 910 static void gmbus_irq_handler(struct drm_device *dev)
 911 {
 912         struct drm_i915_private *dev_priv = (drm_i915_private_t *) dev->dev_private;
 913
 914         wake_up_all(&dev_priv->gmbus_wait_queue);
 915 }
 916
 917 static void dp_aux_irq_handler(struct drm_device *dev)
 918 {
 919         struct drm_i915_private *dev_priv = (drm_i915_private_t *) dev->dev_private;
 920
 921         wake_up_all(&dev_priv->gmbus_wait_queue);
 922 }
 923
 924 static irqreturn_t valleyview_irq_handler(int irq, void *arg)
 925 {
 926         struct drm_device *dev = (struct drm_device *) arg;
 927         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 928         u32 iir, gt_iir, pm_iir;
 929         irqreturn_t ret = IRQ_NONE;
 930         unsigned long irqflags;
 931         int pipe;
 932         u32 pipe_stats[I915_MAX_PIPES];
 933
 934         atomic_inc(&dev_priv->irq_received);
 935
 936         while (true) {
 937                 iir = I915_READ(VLV_IIR);
 938                 gt_iir = I915_READ(GTIIR);
 939                 pm_iir = I915_READ(GEN6_PMIIR);
 940
 941                 if (gt_iir == 0 && pm_iir == 0 && iir == 0)
 942                         goto out;
 943
 944                 ret = IRQ_HANDLED;
 945
 946                 snb_gt_irq_handler(dev, dev_priv, gt_iir);
 947
 948                 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
 949                 for_each_pipe(pipe) {
 950                         int reg = PIPESTAT(pipe);
 951                         pipe_stats[pipe] = I915_READ(reg);
 952
 953                         /*
 954                          * Clear the PIPE*STAT regs before the IIR
 955                          */
 956                         if (pipe_stats[pipe] & 0x8000ffff) {
 957                                 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
 958                                         DRM_DEBUG_DRIVER("pipe %c underrun\n",
 959                                                          pipe_name(pipe));
 960                                 I915_WRITE(reg, pipe_stats[pipe]);
 961                         }
 962                 }
 963                 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 964
 965                 for_each_pipe(pipe) {
 966                         if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS)
 967                                 drm_handle_vblank(dev, pipe);
 968
 969                         if (pipe_stats[pipe] & PLANE_FLIPDONE_INT_STATUS_VLV) {
 970                                 intel_prepare_page_flip(dev, pipe);
 971                                 intel_finish_page_flip(dev, pipe);
 972                         }
 973                 }
 974
 975                 /* Consume port.  Then clear IIR or we'll miss events */
 976                 if (iir & I915_DISPLAY_PORT_INTERRUPT) {
 977                         u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
 978                         u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
 979
 980                         DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n",
 981                                          hotplug_status);
 982                         if (hotplug_trigger) {
 983                                 if (hotplug_irq_storm_detect(dev, hotplug_trigger, hpd_status_i915))
 984                                         i915_hpd_irq_setup(dev);
 985                                 queue_work(dev_priv->wq,
 986                                            &dev_priv->hotplug_work);
 987                         }
 988                         I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
 989                         I915_READ(PORT_HOTPLUG_STAT);
 990                 }
 991
 992                 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
 993                         gmbus_irq_handler(dev);
 994
 995                 if (pm_iir & GEN6_PM_DEFERRED_EVENTS)
 996                         gen6_queue_rps_work(dev_priv, pm_iir);
 997
 998                 I915_WRITE(GTIIR, gt_iir);
 999                 I915_WRITE(GEN6_PMIIR, pm_iir);
1000                 I915_WRITE(VLV_IIR, iir);
1001         }
1002
1003 out:
1004         return ret;
1005 }
1006
1007 static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
1008 {
1009         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1010         int pipe;
1011         u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
1012
1013         if (hotplug_trigger) {
1014                 if (hotplug_irq_storm_detect(dev, hotplug_trigger, hpd_ibx))
1015                         ibx_hpd_irq_setup(dev);
1016                 queue_work(dev_priv->wq, &dev_priv->hotplug_work);
1017         }
1018         if (pch_iir & SDE_AUDIO_POWER_MASK) {
1019                 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
1020                                SDE_AUDIO_POWER_SHIFT);
1021                 DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
1022                                  port_name(port));
1023         }
1024
1025         if (pch_iir & SDE_AUX_MASK)
1026                 dp_aux_irq_handler(dev);
1027
1028         if (pch_iir & SDE_GMBUS)
1029                 gmbus_irq_handler(dev);
1030
1031         if (pch_iir & SDE_AUDIO_HDCP_MASK)
1032                 DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");
1033
1034         if (pch_iir & SDE_AUDIO_TRANS_MASK)
1035                 DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");
1036
1037         if (pch_iir & SDE_POISON)
1038                 DRM_ERROR("PCH poison interrupt\n");
1039
1040         if (pch_iir & SDE_FDI_MASK)
1041                 for_each_pipe(pipe)
1042                         DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
1043                                          pipe_name(pipe),
1044                                          I915_READ(FDI_RX_IIR(pipe)));
1045
1046         if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
1047                 DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");
1048
1049         if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
1050                 DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");
1051
1052         if (pch_iir & SDE_TRANSA_FIFO_UNDER)
1053                 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_A,
1054                                                           false))
1055                         DRM_DEBUG_DRIVER("PCH transcoder A FIFO underrun\n");
1056
1057         if (pch_iir & SDE_TRANSB_FIFO_UNDER)
1058                 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_B,
1059                                                           false))
1060                         DRM_DEBUG_DRIVER("PCH transcoder B FIFO underrun\n");
1061 }
1062
1063 static void ivb_err_int_handler(struct drm_device *dev)
1064 {
1065         struct drm_i915_private *dev_priv = dev->dev_private;
1066         u32 err_int = I915_READ(GEN7_ERR_INT);
1067
1068         if (err_int & ERR_INT_POISON)
1069                 DRM_ERROR("Poison interrupt\n");
1070
1071         if (err_int & ERR_INT_FIFO_UNDERRUN_A)
1072                 if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_A, false))
1073                         DRM_DEBUG_DRIVER("Pipe A FIFO underrun\n");
1074
1075         if (err_int & ERR_INT_FIFO_UNDERRUN_B)
1076                 if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_B, false))
1077                         DRM_DEBUG_DRIVER("Pipe B FIFO underrun\n");
1078
1079         if (err_int & ERR_INT_FIFO_UNDERRUN_C)
1080                 if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_C, false))
1081                         DRM_DEBUG_DRIVER("Pipe C FIFO underrun\n");
1082
1083         I915_WRITE(GEN7_ERR_INT, err_int);
1084 }
1085
1086 static void cpt_serr_int_handler(struct drm_device *dev)
1087 {
1088         struct drm_i915_private *dev_priv = dev->dev_private;
1089         u32 serr_int = I915_READ(SERR_INT);
1090
1091         if (serr_int & SERR_INT_POISON)
1092                 DRM_ERROR("PCH poison interrupt\n");
1093
1094         if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN)
1095                 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_A,
1096                                                           false))
1097                         DRM_DEBUG_DRIVER("PCH transcoder A FIFO underrun\n");
1098
1099         if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN)
1100                 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_B,
1101                                                           false))
1102                         DRM_DEBUG_DRIVER("PCH transcoder B FIFO underrun\n");
1103
1104         if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN)
1105                 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_C,
1106                                                           false))
1107                         DRM_DEBUG_DRIVER("PCH transcoder C FIFO underrun\n");
1108
1109         I915_WRITE(SERR_INT, serr_int);
1110 }
1111
1112 static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
1113 {
1114         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1115         int pipe;
1116         u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
1117
1118         if (hotplug_trigger) {
1119                 if (hotplug_irq_storm_detect(dev, hotplug_trigger, hpd_cpt))
1120                         ibx_hpd_irq_setup(dev);
1121                 queue_work(dev_priv->wq, &dev_priv->hotplug_work);
1122         }
1123         if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
1124                 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
1125                                SDE_AUDIO_POWER_SHIFT_CPT);
1126                 DRM_DEBUG_DRIVER("PCH audio power change on port %c\n",
1127                                  port_name(port));
1128         }
1129
1130         if (pch_iir & SDE_AUX_MASK_CPT)
1131                 dp_aux_irq_handler(dev);
1132
1133         if (pch_iir & SDE_GMBUS_CPT)
1134                 gmbus_irq_handler(dev);
1135
1136         if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
1137                 DRM_DEBUG_DRIVER("Audio CP request interrupt\n");
1138
1139         if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
1140                 DRM_DEBUG_DRIVER("Audio CP change interrupt\n");
1141
1142         if (pch_iir & SDE_FDI_MASK_CPT)
1143                 for_each_pipe(pipe)
1144                         DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
1145                                          pipe_name(pipe),
1146                                          I915_READ(FDI_RX_IIR(pipe)));
1147
1148         if (pch_iir & SDE_ERROR_CPT)
1149                 cpt_serr_int_handler(dev);
1150 }
1151
1152 static irqreturn_t ivybridge_irq_handler(int irq, void *arg)
1153 {
1154         struct drm_device *dev = (struct drm_device *) arg;
1155         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1156         u32 de_iir, gt_iir, de_ier, pm_iir, sde_ier = 0;
1157         irqreturn_t ret = IRQ_NONE;
1158         int i;
1159
1160         atomic_inc(&dev_priv->irq_received);
1161
1162         /* We get interrupts on unclaimed registers, so check for this before we
1163          * do any I915_{READ,WRITE}. */
1164         if (IS_HASWELL(dev) &&
1165             (I915_READ_NOTRACE(FPGA_DBG) & FPGA_DBG_RM_NOCLAIM)) {
1166                 DRM_ERROR("Unclaimed register before interrupt\n");
1167                 I915_WRITE_NOTRACE(FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
1168         }
1169
1170         /* disable master interrupt before clearing iir  */
1171         de_ier = I915_READ(DEIER);
1172         I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
1173
1174         /* Disable south interrupts. We'll only write to SDEIIR once, so further
1175          * interrupts will will be stored on its back queue, and then we'll be
1176          * able to process them after we restore SDEIER (as soon as we restore
1177          * it, we'll get an interrupt if SDEIIR still has something to process
1178          * due to its back queue). */
1179         if (!HAS_PCH_NOP(dev)) {
1180                 sde_ier = I915_READ(SDEIER);
1181                 I915_WRITE(SDEIER, 0);
1182                 POSTING_READ(SDEIER);
1183         }
1184
1185         /* On Haswell, also mask ERR_INT because we don't want to risk
1186          * generating "unclaimed register" interrupts from inside the interrupt
1187          * handler. */
1188         if (IS_HASWELL(dev))
1189                 ironlake_disable_display_irq(dev_priv, DE_ERR_INT_IVB);
1190
1191         gt_iir = I915_READ(GTIIR);
1192         if (gt_iir) {
1193                 snb_gt_irq_handler(dev, dev_priv, gt_iir);
1194                 I915_WRITE(GTIIR, gt_iir);
1195                 ret = IRQ_HANDLED;
1196         }
1197
1198         de_iir = I915_READ(DEIIR);
1199         if (de_iir) {
1200                 if (de_iir & DE_ERR_INT_IVB)
1201                         ivb_err_int_handler(dev);
1202
1203                 if (de_iir & DE_AUX_CHANNEL_A_IVB)
1204                         dp_aux_irq_handler(dev);
1205
1206                 if (de_iir & DE_GSE_IVB)
1207                         intel_opregion_asle_intr(dev);
1208
1209                 for (i = 0; i < 3; i++) {
1210                         if (de_iir & (DE_PIPEA_VBLANK_IVB << (5 * i)))
1211                                 drm_handle_vblank(dev, i);
1212                         if (de_iir & (DE_PLANEA_FLIP_DONE_IVB << (5 * i))) {
1213                                 intel_prepare_page_flip(dev, i);
1214                                 intel_finish_page_flip_plane(dev, i);
1215                         }
1216                 }
1217
1218                 /* check event from PCH */
1219                 if (!HAS_PCH_NOP(dev) && (de_iir & DE_PCH_EVENT_IVB)) {
1220                         u32 pch_iir = I915_READ(SDEIIR);
1221
1222                         cpt_irq_handler(dev, pch_iir);
1223
1224                         /* clear PCH hotplug event before clear CPU irq */
1225                         I915_WRITE(SDEIIR, pch_iir);
1226                 }
1227
1228                 I915_WRITE(DEIIR, de_iir);
1229                 ret = IRQ_HANDLED;
1230         }
1231
1232         pm_iir = I915_READ(GEN6_PMIIR);
1233         if (pm_iir) {
1234                 if (pm_iir & GEN6_PM_DEFERRED_EVENTS)
1235                         gen6_queue_rps_work(dev_priv, pm_iir);
1236                 I915_WRITE(GEN6_PMIIR, pm_iir);
1237                 ret = IRQ_HANDLED;
1238         }
1239
1240         if (IS_HASWELL(dev) && ivb_can_enable_err_int(dev))
1241                 ironlake_enable_display_irq(dev_priv, DE_ERR_INT_IVB);
1242
1243         I915_WRITE(DEIER, de_ier);
1244         POSTING_READ(DEIER);
1245         if (!HAS_PCH_NOP(dev)) {
1246                 I915_WRITE(SDEIER, sde_ier);
1247                 POSTING_READ(SDEIER);
1248         }
1249
1250         return ret;
1251 }
1252
1253 static void ilk_gt_irq_handler(struct drm_device *dev,
1254                                struct drm_i915_private *dev_priv,
1255                                u32 gt_iir)
1256 {
1257         if (gt_iir & (GT_USER_INTERRUPT | GT_PIPE_NOTIFY))
1258                 notify_ring(dev, &dev_priv->ring[RCS]);
1259         if (gt_iir & GT_BSD_USER_INTERRUPT)
1260                 notify_ring(dev, &dev_priv->ring[VCS]);
1261 }
1262
1263 static irqreturn_t ironlake_irq_handler(int irq, void *arg)
1264 {
1265         struct drm_device *dev = (struct drm_device *) arg;
1266         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1267         int ret = IRQ_NONE;
1268         u32 de_iir, gt_iir, de_ier, pm_iir, sde_ier;
1269
1270         atomic_inc(&dev_priv->irq_received);
1271
1272         /* disable master interrupt before clearing iir  */
1273         de_ier = I915_READ(DEIER);
1274         I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
1275         POSTING_READ(DEIER);
1276
1277         /* Disable south interrupts. We'll only write to SDEIIR once, so further
1278          * interrupts will will be stored on its back queue, and then we'll be
1279          * able to process them after we restore SDEIER (as soon as we restore
1280          * it, we'll get an interrupt if SDEIIR still has something to process
1281          * due to its back queue). */
1282         sde_ier = I915_READ(SDEIER);
1283         I915_WRITE(SDEIER, 0);
1284         POSTING_READ(SDEIER);
1285
1286         de_iir = I915_READ(DEIIR);
1287         gt_iir = I915_READ(GTIIR);
1288         pm_iir = I915_READ(GEN6_PMIIR);
1289
1290         if (de_iir == 0 && gt_iir == 0 && (!IS_GEN6(dev) || pm_iir == 0))
1291                 goto done;
1292
1293         ret = IRQ_HANDLED;
1294
1295         if (IS_GEN5(dev))
1296                 ilk_gt_irq_handler(dev, dev_priv, gt_iir);
1297         else
1298                 snb_gt_irq_handler(dev, dev_priv, gt_iir);
1299
1300         if (de_iir & DE_AUX_CHANNEL_A)
1301                 dp_aux_irq_handler(dev);
1302
1303         if (de_iir & DE_GSE)
1304                 intel_opregion_asle_intr(dev);
1305
1306         if (de_iir & DE_PIPEA_VBLANK)
1307                 drm_handle_vblank(dev, 0);
1308
1309         if (de_iir & DE_PIPEB_VBLANK)
1310                 drm_handle_vblank(dev, 1);
1311
1312         if (de_iir & DE_POISON)
1313                 DRM_ERROR("Poison interrupt\n");
1314
1315         if (de_iir & DE_PIPEA_FIFO_UNDERRUN)
1316                 if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_A, false))
1317                         DRM_DEBUG_DRIVER("Pipe A FIFO underrun\n");
1318
1319         if (de_iir & DE_PIPEB_FIFO_UNDERRUN)
1320                 if (intel_set_cpu_fifo_underrun_reporting(dev, PIPE_B, false))
1321                         DRM_DEBUG_DRIVER("Pipe B FIFO underrun\n");
1322
1323         if (de_iir & DE_PLANEA_FLIP_DONE) {
1324                 intel_prepare_page_flip(dev, 0);
1325                 intel_finish_page_flip_plane(dev, 0);
1326         }
1327
1328         if (de_iir & DE_PLANEB_FLIP_DONE) {
1329                 intel_prepare_page_flip(dev, 1);
1330                 intel_finish_page_flip_plane(dev, 1);
1331         }
1332
1333         /* check event from PCH */
1334         if (de_iir & DE_PCH_EVENT) {
1335                 u32 pch_iir = I915_READ(SDEIIR);
1336
1337                 if (HAS_PCH_CPT(dev))
1338                         cpt_irq_handler(dev, pch_iir);
1339                 else
1340                         ibx_irq_handler(dev, pch_iir);
1341
1342                 /* should clear PCH hotplug event before clear CPU irq */
1343                 I915_WRITE(SDEIIR, pch_iir);
1344         }
1345
1346         if (IS_GEN5(dev) &&  de_iir & DE_PCU_EVENT)
1347                 ironlake_handle_rps_change(dev);
1348
1349         if (IS_GEN6(dev) && pm_iir & GEN6_PM_DEFERRED_EVENTS)
1350                 gen6_queue_rps_work(dev_priv, pm_iir);
1351
1352         I915_WRITE(GTIIR, gt_iir);
1353         I915_WRITE(DEIIR, de_iir);
1354         I915_WRITE(GEN6_PMIIR, pm_iir);
1355
1356 done:
1357         I915_WRITE(DEIER, de_ier);
1358         POSTING_READ(DEIER);
1359         I915_WRITE(SDEIER, sde_ier);
1360         POSTING_READ(SDEIER);
1361
1362         return ret;
1363 }
1364
1365 /**
1366  * i915_error_work_func - do process context error handling work
1367  * @work: work struct
1368  *
1369  * Fire an error uevent so userspace can see that a hang or error
1370  * was detected.
1371  */
1372 static void i915_error_work_func(struct work_struct *work)
1373 {
1374         struct i915_gpu_error *error = container_of(work, struct i915_gpu_error,
1375                                                     work);
1376         drm_i915_private_t *dev_priv = container_of(error, drm_i915_private_t,
1377                                                     gpu_error);
1378         struct drm_device *dev = dev_priv->dev;
1379         struct intel_ring_buffer *ring;
1380         char *error_event[] = { "ERROR=1", NULL };
1381         char *reset_event[] = { "RESET=1", NULL };
1382         char *reset_done_event[] = { "ERROR=0", NULL };
1383         int i, ret;
1384
1385         kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE, error_event);
1386
1387         /*
1388          * Note that there's only one work item which does gpu resets, so we
1389          * need not worry about concurrent gpu resets potentially incrementing
1390          * error->reset_counter twice. We only need to take care of another
1391          * racing irq/hangcheck declaring the gpu dead for a second time. A
1392          * quick check for that is good enough: schedule_work ensures the
1393          * correct ordering between hang detection and this work item, and since
1394          * the reset in-progress bit is only ever set by code outside of this
1395          * work we don't need to worry about any other races.
1396          */
1397         if (i915_reset_in_progress(error) && !i915_terminally_wedged(error)) {
1398                 DRM_DEBUG_DRIVER("resetting chip\n");
1399                 kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE,
1400                                    reset_event);
1401
1402                 ret = i915_reset(dev);
1403
1404                 if (ret == 0) {
1405                         /*
1406                          * After all the gem state is reset, increment the reset
1407                          * counter and wake up everyone waiting for the reset to
1408                          * complete.
1409                          *
1410                          * Since unlock operations are a one-sided barrier only,
1411                          * we need to insert a barrier here to order any seqno
1412                          * updates before
1413                          * the counter increment.
1414                          */
1415                         smp_mb__before_atomic_inc();
1416                         atomic_inc(&dev_priv->gpu_error.reset_counter);
1417
1418                         kobject_uevent_env(&dev->primary->kdev.kobj,
1419                                            KOBJ_CHANGE, reset_done_event);
1420                 } else {
1421                         atomic_set(&error->reset_counter, I915_WEDGED);
1422                 }
1423
1424                 for_each_ring(ring, dev_priv, i)
1425                         wake_up_all(&ring->irq_queue);
1426
1427                 intel_display_handle_reset(dev);
1428
1429                 wake_up_all(&dev_priv->gpu_error.reset_queue);
1430         }
1431 }
1432
1433 /* NB: please notice the memset */
1434 static void i915_get_extra_instdone(struct drm_device *dev,
1435                                     uint32_t *instdone)
1436 {
1437         struct drm_i915_private *dev_priv = dev->dev_private;
1438         memset(instdone, 0, sizeof(*instdone) * I915_NUM_INSTDONE_REG);
1439
1440         switch(INTEL_INFO(dev)->gen) {
1441         case 2:
1442         case 3:
1443                 instdone[0] = I915_READ(INSTDONE);
1444                 break;
1445         case 4:
1446         case 5:
1447         case 6:
1448                 instdone[0] = I915_READ(INSTDONE_I965);
1449                 instdone[1] = I915_READ(INSTDONE1);
1450                 break;
1451         default:
1452                 WARN_ONCE(1, "Unsupported platform\n");
1453         case 7:
1454                 instdone[0] = I915_READ(GEN7_INSTDONE_1);
1455                 instdone[1] = I915_READ(GEN7_SC_INSTDONE);
1456                 instdone[2] = I915_READ(GEN7_SAMPLER_INSTDONE);
1457                 instdone[3] = I915_READ(GEN7_ROW_INSTDONE);
1458                 break;
1459         }
1460 }
1461
1462 #ifdef CONFIG_DEBUG_FS
1463 static struct drm_i915_error_object *
1464 i915_error_object_create_sized(struct drm_i915_private *dev_priv,
1465                                struct drm_i915_gem_object *src,
1466                                const int num_pages)
1467 {
1468         struct drm_i915_error_object *dst;
1469         int i;
1470         u32 reloc_offset;
1471
1472         if (src == NULL || src->pages == NULL)
1473                 return NULL;
1474
1475         dst = kmalloc(sizeof(*dst) + num_pages * sizeof(u32 *), GFP_ATOMIC);
1476         if (dst == NULL)
1477                 return NULL;
1478
1479         reloc_offset = src->gtt_offset;
1480         for (i = 0; i < num_pages; i++) {
1481                 unsigned long flags;
1482                 void *d;
1483
1484                 d = kmalloc(PAGE_SIZE, GFP_ATOMIC);
1485                 if (d == NULL)
1486                         goto unwind;
1487
1488                 local_irq_save(flags);
1489                 if (reloc_offset < dev_priv->gtt.mappable_end &&
1490                     src->has_global_gtt_mapping) {
1491                         void __iomem *s;
1492
1493                         /* Simply ignore tiling or any overlapping fence.
1494                          * It's part of the error state, and this hopefully
1495                          * captures what the GPU read.
1496                          */
1497
1498                         s = io_mapping_map_atomic_wc(dev_priv->gtt.mappable,
1499                                                      reloc_offset);
1500                         memcpy_fromio(d, s, PAGE_SIZE);
1501                         io_mapping_unmap_atomic(s);
1502                 } else if (src->stolen) {
1503                         unsigned long offset;
1504
1505                         offset = dev_priv->mm.stolen_base;
1506                         offset += src->stolen->start;
1507                         offset += i << PAGE_SHIFT;
1508
1509                         memcpy_fromio(d, (void __iomem *) offset, PAGE_SIZE);
1510                 } else {
1511                         struct page *page;
1512                         void *s;
1513
1514                         page = i915_gem_object_get_page(src, i);
1515
1516                         drm_clflush_pages(&page, 1);
1517
1518                         s = kmap_atomic(page);
1519                         memcpy(d, s, PAGE_SIZE);
1520                         kunmap_atomic(s);
1521
1522                         drm_clflush_pages(&page, 1);
1523                 }
1524                 local_irq_restore(flags);
1525
1526                 dst->pages[i] = d;
1527
1528                 reloc_offset += PAGE_SIZE;
1529         }
1530         dst->page_count = num_pages;
1531         dst->gtt_offset = src->gtt_offset;
1532
1533         return dst;
1534
1535 unwind:
1536         while (i--)
1537                 kfree(dst->pages[i]);
1538         kfree(dst);
1539         return NULL;
1540 }
1541 #define i915_error_object_create(dev_priv, src) \
1542         i915_error_object_create_sized((dev_priv), (src), \
1543                                        (src)->base.size>>PAGE_SHIFT)
1544
1545 static void
1546 i915_error_object_free(struct drm_i915_error_object *obj)
1547 {
1548         int page;
1549
1550         if (obj == NULL)
1551                 return;
1552
1553         for (page = 0; page < obj->page_count; page++)
1554                 kfree(obj->pages[page]);
1555
1556         kfree(obj);
1557 }
1558
1559 void
1560 i915_error_state_free(struct kref *error_ref)
1561 {
1562         struct drm_i915_error_state *error = container_of(error_ref,
1563                                                           typeof(*error), ref);
1564         int i;
1565
1566         for (i = 0; i < ARRAY_SIZE(error->ring); i++) {
1567                 i915_error_object_free(error->ring[i].batchbuffer);
1568                 i915_error_object_free(error->ring[i].ringbuffer);
1569                 i915_error_object_free(error->ring[i].ctx);
1570                 kfree(error->ring[i].requests);
1571         }
1572
1573         kfree(error->active_bo);
1574         kfree(error->overlay);
1575         kfree(error->display);
1576         kfree(error);
1577 }
1578 static void capture_bo(struct drm_i915_error_buffer *err,
1579                        struct drm_i915_gem_object *obj)
1580 {
1581         err->size = obj->base.size;
1582         err->name = obj->base.name;
1583         err->rseqno = obj->last_read_seqno;
1584         err->wseqno = obj->last_write_seqno;
1585         err->gtt_offset = obj->gtt_offset;
1586         err->read_domains = obj->base.read_domains;
1587         err->write_domain = obj->base.write_domain;
1588         err->fence_reg = obj->fence_reg;
1589         err->pinned = 0;
1590         if (obj->pin_count > 0)
1591                 err->pinned = 1;
1592         if (obj->user_pin_count > 0)
1593                 err->pinned = -1;
1594         err->tiling = obj->tiling_mode;
1595         err->dirty = obj->dirty;
1596         err->purgeable = obj->madv != I915_MADV_WILLNEED;
1597         err->ring = obj->ring ? obj->ring->id : -1;
1598         err->cache_level = obj->cache_level;
1599 }
1600
1601 static u32 capture_active_bo(struct drm_i915_error_buffer *err,
1602                              int count, struct list_head *head)
1603 {
1604         struct drm_i915_gem_object *obj;
1605         int i = 0;
1606
1607         list_for_each_entry(obj, head, mm_list) {
1608                 capture_bo(err++, obj);
1609                 if (++i == count)
1610                         break;
1611         }
1612
1613         return i;
1614 }
1615
1616 static u32 capture_pinned_bo(struct drm_i915_error_buffer *err,
1617                              int count, struct list_head *head)
1618 {
1619         struct drm_i915_gem_object *obj;
1620         int i = 0;
1621
1622         list_for_each_entry(obj, head, gtt_list) {
1623                 if (obj->pin_count == 0)
1624                         continue;
1625
1626                 capture_bo(err++, obj);
1627                 if (++i == count)
1628                         break;
1629         }
1630
1631         return i;
1632 }
1633
1634 static void i915_gem_record_fences(struct drm_device *dev,
1635                                    struct drm_i915_error_state *error)
1636 {
1637         struct drm_i915_private *dev_priv = dev->dev_private;
1638         int i;
1639
1640         /* Fences */
1641         switch (INTEL_INFO(dev)->gen) {
1642         case 7:
1643         case 6:
1644                 for (i = 0; i < dev_priv->num_fence_regs; i++)
1645                         error->fence[i] = I915_READ64(FENCE_REG_SANDYBRIDGE_0 + (i * 8));
1646                 break;
1647         case 5:
1648         case 4:
1649                 for (i = 0; i < 16; i++)
1650                         error->fence[i] = I915_READ64(FENCE_REG_965_0 + (i * 8));
1651                 break;
1652         case 3:
1653                 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
1654                         for (i = 0; i < 8; i++)
1655                                 error->fence[i+8] = I915_READ(FENCE_REG_945_8 + (i * 4));
1656         case 2:
1657                 for (i = 0; i < 8; i++)
1658                         error->fence[i] = I915_READ(FENCE_REG_830_0 + (i * 4));
1659                 break;
1660
1661         default:
1662                 BUG();
1663         }
1664 }
1665
1666 static struct drm_i915_error_object *
1667 i915_error_first_batchbuffer(struct drm_i915_private *dev_priv,
1668                              struct intel_ring_buffer *ring)
1669 {
1670         struct drm_i915_gem_object *obj;
1671         u32 seqno;
1672
1673         if (!ring->get_seqno)
1674                 return NULL;
1675
1676         if (HAS_BROKEN_CS_TLB(dev_priv->dev)) {
1677                 u32 acthd = I915_READ(ACTHD);
1678
1679                 if (WARN_ON(ring->id != RCS))
1680                         return NULL;
1681
1682                 obj = ring->private;
1683                 if (acthd >= obj->gtt_offset &&
1684                     acthd < obj->gtt_offset + obj->base.size)
1685                         return i915_error_object_create(dev_priv, obj);
1686         }
1687
1688         seqno = ring->get_seqno(ring, false);
1689         list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) {
1690                 if (obj->ring != ring)
1691                         continue;
1692
1693                 if (i915_seqno_passed(seqno, obj->last_read_seqno))
1694                         continue;
1695
1696                 if ((obj->base.read_domains & I915_GEM_DOMAIN_COMMAND) == 0)
1697                         continue;
1698
1699                 /* We need to copy these to an anonymous buffer as the simplest
1700                  * method to avoid being overwritten by userspace.
1701                  */
1702                 return i915_error_object_create(dev_priv, obj);
1703         }
1704
1705         return NULL;
1706 }
1707
1708 static void i915_record_ring_state(struct drm_device *dev,
1709                                    struct drm_i915_error_state *error,
1710                                    struct intel_ring_buffer *ring)
1711 {
1712         struct drm_i915_private *dev_priv = dev->dev_private;
1713
1714         if (INTEL_INFO(dev)->gen >= 6) {
1715                 error->rc_psmi[ring->id] = I915_READ(ring->mmio_base + 0x50);
1716                 error->fault_reg[ring->id] = I915_READ(RING_FAULT_REG(ring));
1717                 error->semaphore_mboxes[ring->id][0]
1718                         = I915_READ(RING_SYNC_0(ring->mmio_base));
1719                 error->semaphore_mboxes[ring->id][1]
1720                         = I915_READ(RING_SYNC_1(ring->mmio_base));
1721                 error->semaphore_seqno[ring->id][0] = ring->sync_seqno[0];
1722                 error->semaphore_seqno[ring->id][1] = ring->sync_seqno[1];
1723         }
1724
1725         if (INTEL_INFO(dev)->gen >= 4) {
1726                 error->faddr[ring->id] = I915_READ(RING_DMA_FADD(ring->mmio_base));
1727                 error->ipeir[ring->id] = I915_READ(RING_IPEIR(ring->mmio_base));
1728                 error->ipehr[ring->id] = I915_READ(RING_IPEHR(ring->mmio_base));
1729                 error->instdone[ring->id] = I915_READ(RING_INSTDONE(ring->mmio_base));
1730                 error->instps[ring->id] = I915_READ(RING_INSTPS(ring->mmio_base));
1731                 if (ring->id == RCS)
1732                         error->bbaddr = I915_READ64(BB_ADDR);
1733         } else {
1734                 error->faddr[ring->id] = I915_READ(DMA_FADD_I8XX);
1735                 error->ipeir[ring->id] = I915_READ(IPEIR);
1736                 error->ipehr[ring->id] = I915_READ(IPEHR);
1737                 error->instdone[ring->id] = I915_READ(INSTDONE);
1738         }
1739
1740         error->waiting[ring->id] = waitqueue_active(&ring->irq_queue);
1741         error->instpm[ring->id] = I915_READ(RING_INSTPM(ring->mmio_base));
1742         error->seqno[ring->id] = ring->get_seqno(ring, false);
1743         error->acthd[ring->id] = intel_ring_get_active_head(ring);
1744         error->head[ring->id] = I915_READ_HEAD(ring);
1745         error->tail[ring->id] = I915_READ_TAIL(ring);
1746         error->ctl[ring->id] = I915_READ_CTL(ring);
1747
1748         error->cpu_ring_head[ring->id] = ring->head;
1749         error->cpu_ring_tail[ring->id] = ring->tail;
1750 }
1751
1752
1753 static void i915_gem_record_active_context(struct intel_ring_buffer *ring,
1754                                            struct drm_i915_error_state *error,
1755                                            struct drm_i915_error_ring *ering)
1756 {
1757         struct drm_i915_private *dev_priv = ring->dev->dev_private;
1758         struct drm_i915_gem_object *obj;
1759
1760         /* Currently render ring is the only HW context user */
1761         if (ring->id != RCS || !error->ccid)
1762                 return;
1763
1764         list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) {
1765                 if ((error->ccid & PAGE_MASK) == obj->gtt_offset) {
1766                         ering->ctx = i915_error_object_create_sized(dev_priv,
1767                                                                     obj, 1);
1768                 }
1769         }
1770 }
1771
1772 static void i915_gem_record_rings(struct drm_device *dev,
1773                                   struct drm_i915_error_state *error)
1774 {
1775         struct drm_i915_private *dev_priv = dev->dev_private;
1776         struct intel_ring_buffer *ring;
1777         struct drm_i915_gem_request *request;
1778         int i, count;
1779
1780         for_each_ring(ring, dev_priv, i) {
1781                 i915_record_ring_state(dev, error, ring);
1782
1783                 error->ring[i].batchbuffer =
1784                         i915_error_first_batchbuffer(dev_priv, ring);
1785
1786                 error->ring[i].ringbuffer =
1787                         i915_error_object_create(dev_priv, ring->obj);
1788
1789
1790                 i915_gem_record_active_context(ring, error, &error->ring[i]);
1791
1792                 count = 0;
1793                 list_for_each_entry(request, &ring->request_list, list)
1794                         count++;
1795
1796                 error->ring[i].num_requests = count;
1797                 error->ring[i].requests =
1798                         kmalloc(count*sizeof(struct drm_i915_error_request),
1799                                 GFP_ATOMIC);
1800                 if (error->ring[i].requests == NULL) {
1801                         error->ring[i].num_requests = 0;
1802                         continue;
1803                 }
1804
1805                 count = 0;
1806                 list_for_each_entry(request, &ring->request_list, list) {
1807                         struct drm_i915_error_request *erq;
1808
1809                         erq = &error->ring[i].requests[count++];
1810                         erq->seqno = request->seqno;
1811                         erq->jiffies = request->emitted_jiffies;
1812                         erq->tail = request->tail;
1813                 }
1814         }
1815 }
1816
1817 /**
1818  * i915_capture_error_state - capture an error record for later analysis
1819  * @dev: drm device
1820  *
1821  * Should be called when an error is detected (either a hang or an error
1822  * interrupt) to capture error state from the time of the error.  Fills
1823  * out a structure which becomes available in debugfs for user level tools
1824  * to pick up.
1825  */
1826 static void i915_capture_error_state(struct drm_device *dev)
1827 {
1828         struct drm_i915_private *dev_priv = dev->dev_private;
1829         struct drm_i915_gem_object *obj;
1830         struct drm_i915_error_state *error;
1831         unsigned long flags;
1832         int i, pipe;
1833
1834         spin_lock_irqsave(&dev_priv->gpu_error.lock, flags);
1835         error = dev_priv->gpu_error.first_error;
1836         spin_unlock_irqrestore(&dev_priv->gpu_error.lock, flags);
1837         if (error)
1838                 return;
1839
1840         /* Account for pipe specific data like PIPE*STAT */
1841         error = kzalloc(sizeof(*error), GFP_ATOMIC);
1842         if (!error) {
1843                 DRM_DEBUG_DRIVER("out of memory, not capturing error state\n");
1844                 return;
1845         }
1846
1847         DRM_INFO("capturing error event; look for more information in "
1848                  "/sys/kernel/debug/dri/%d/i915_error_state\n",
1849                  dev->primary->index);
1850
1851         kref_init(&error->ref);
1852         error->eir = I915_READ(EIR);
1853         error->pgtbl_er = I915_READ(PGTBL_ER);
1854         if (HAS_HW_CONTEXTS(dev))
1855                 error->ccid = I915_READ(CCID);
1856
1857         if (HAS_PCH_SPLIT(dev))
1858                 error->ier = I915_READ(DEIER) | I915_READ(GTIER);
1859         else if (IS_VALLEYVIEW(dev))
1860                 error->ier = I915_READ(GTIER) | I915_READ(VLV_IER);
1861         else if (IS_GEN2(dev))
1862                 error->ier = I915_READ16(IER);
1863         else
1864                 error->ier = I915_READ(IER);
1865
1866         if (INTEL_INFO(dev)->gen >= 6)
1867                 error->derrmr = I915_READ(DERRMR);
1868
1869         if (IS_VALLEYVIEW(dev))
1870                 error->forcewake = I915_READ(FORCEWAKE_VLV);
1871         else if (INTEL_INFO(dev)->gen >= 7)
1872                 error->forcewake = I915_READ(FORCEWAKE_MT);
1873         else if (INTEL_INFO(dev)->gen == 6)
1874                 error->forcewake = I915_READ(FORCEWAKE);
1875
1876         if (!HAS_PCH_SPLIT(dev))
1877                 for_each_pipe(pipe)
1878                         error->pipestat[pipe] = I915_READ(PIPESTAT(pipe));
1879
1880         if (INTEL_INFO(dev)->gen >= 6) {
1881                 error->error = I915_READ(ERROR_GEN6);
1882                 error->done_reg = I915_READ(DONE_REG);
1883         }
1884
1885         if (INTEL_INFO(dev)->gen == 7)
1886                 error->err_int = I915_READ(GEN7_ERR_INT);
1887
1888         i915_get_extra_instdone(dev, error->extra_instdone);
1889
1890         i915_gem_record_fences(dev, error);
1891         i915_gem_record_rings(dev, error);
1892
1893         /* Record buffers on the active and pinned lists. */
1894         error->active_bo = NULL;
1895         error->pinned_bo = NULL;
1896
1897         i = 0;
1898         list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list)
1899                 i++;
1900         error->active_bo_count = i;
1901         list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list)
1902                 if (obj->pin_count)
1903                         i++;
1904         error->pinned_bo_count = i - error->active_bo_count;
1905
1906         error->active_bo = NULL;
1907         error->pinned_bo = NULL;
1908         if (i) {
1909                 error->active_bo = kmalloc(sizeof(*error->active_bo)*i,
1910                                            GFP_ATOMIC);
1911                 if (error->active_bo)
1912                         error->pinned_bo =
1913                                 error->active_bo + error->active_bo_count;
1914         }
1915
1916         if (error->active_bo)
1917                 error->active_bo_count =
1918                         capture_active_bo(error->active_bo,
1919                                           error->active_bo_count,
1920                                           &dev_priv->mm.active_list);
1921
1922         if (error->pinned_bo)
1923                 error->pinned_bo_count =
1924                         capture_pinned_bo(error->pinned_bo,
1925                                           error->pinned_bo_count,
1926                                           &dev_priv->mm.bound_list);
1927
1928         do_gettimeofday(&error->time);
1929
1930         error->overlay = intel_overlay_capture_error_state(dev);
1931         error->display = intel_display_capture_error_state(dev);
1932
1933         spin_lock_irqsave(&dev_priv->gpu_error.lock, flags);
1934         if (dev_priv->gpu_error.first_error == NULL) {
1935                 dev_priv->gpu_error.first_error = error;
1936                 error = NULL;
1937         }
1938         spin_unlock_irqrestore(&dev_priv->gpu_error.lock, flags);
1939
1940         if (error)
1941                 i915_error_state_free(&error->ref);
1942 }
1943
1944 void i915_destroy_error_state(struct drm_device *dev)
1945 {
1946         struct drm_i915_private *dev_priv = dev->dev_private;
1947         struct drm_i915_error_state *error;
1948         unsigned long flags;
1949
1950         spin_lock_irqsave(&dev_priv->gpu_error.lock, flags);
1951         error = dev_priv->gpu_error.first_error;
1952         dev_priv->gpu_error.first_error = NULL;
1953         spin_unlock_irqrestore(&dev_priv->gpu_error.lock, flags);
1954
1955         if (error)
1956                 kref_put(&error->ref, i915_error_state_free);
1957 }
1958 #else
1959 #define i915_capture_error_state(x)
1960 #endif
1961
1962 static void i915_report_and_clear_eir(struct drm_device *dev)
1963 {
1964         struct drm_i915_private *dev_priv = dev->dev_private;
1965         uint32_t instdone[I915_NUM_INSTDONE_REG];
1966         u32 eir = I915_READ(EIR);
1967         int pipe, i;
1968
1969         if (!eir)
1970                 return;
1971
1972         pr_err("render error detected, EIR: 0x%08x\n", eir);
1973
1974         i915_get_extra_instdone(dev, instdone);
1975
1976         if (IS_G4X(dev)) {
1977                 if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
1978                         u32 ipeir = I915_READ(IPEIR_I965);
1979
1980                         pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
1981                         pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
1982                         for (i = 0; i < ARRAY_SIZE(instdone); i++)
1983                                 pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
1984                         pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
1985                         pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
1986                         I915_WRITE(IPEIR_I965, ipeir);
1987                         POSTING_READ(IPEIR_I965);
1988                 }
1989                 if (eir & GM45_ERROR_PAGE_TABLE) {
1990                         u32 pgtbl_err = I915_READ(PGTBL_ER);
1991                         pr_err("page table error\n");
1992                         pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
1993                         I915_WRITE(PGTBL_ER, pgtbl_err);
1994                         POSTING_READ(PGTBL_ER);
1995                 }
1996         }
1997
1998         if (!IS_GEN2(dev)) {
1999                 if (eir & I915_ERROR_PAGE_TABLE) {
2000                         u32 pgtbl_err = I915_READ(PGTBL_ER);
2001                         pr_err("page table error\n");
2002                         pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
2003                         I915_WRITE(PGTBL_ER, pgtbl_err);
2004                         POSTING_READ(PGTBL_ER);
2005                 }
2006         }
2007
2008         if (eir & I915_ERROR_MEMORY_REFRESH) {
2009                 pr_err("memory refresh error:\n");
2010                 for_each_pipe(pipe)
2011                         pr_err("pipe %c stat: 0x%08x\n",
2012                                pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
2013                 /* pipestat has already been acked */
2014         }
2015         if (eir & I915_ERROR_INSTRUCTION) {
2016                 pr_err("instruction error\n");
2017                 pr_err("  INSTPM: 0x%08x\n", I915_READ(INSTPM));
2018                 for (i = 0; i < ARRAY_SIZE(instdone); i++)
2019                         pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
2020                 if (INTEL_INFO(dev)->gen < 4) {
2021                         u32 ipeir = I915_READ(IPEIR);
2022
2023                         pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR));
2024                         pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR));
2025                         pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD));
2026                         I915_WRITE(IPEIR, ipeir);
2027                         POSTING_READ(IPEIR);
2028                 } else {
2029                         u32 ipeir = I915_READ(IPEIR_I965);
2030
2031                         pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
2032                         pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
2033                         pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
2034                         pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
2035                         I915_WRITE(IPEIR_I965, ipeir);
2036                         POSTING_READ(IPEIR_I965);
2037                 }
2038         }
2039
2040         I915_WRITE(EIR, eir);
2041         POSTING_READ(EIR);
2042         eir = I915_READ(EIR);
2043         if (eir) {
2044                 /*
2045                  * some errors might have become stuck,
2046                  * mask them.
2047                  */
2048                 DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir);
2049                 I915_WRITE(EMR, I915_READ(EMR) | eir);
2050                 I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
2051         }
2052 }
2053
2054 /**
2055  * i915_handle_error - handle an error interrupt
2056  * @dev: drm device
2057  *
2058  * Do some basic checking of regsiter state at error interrupt time and
2059  * dump it to the syslog.  Also call i915_capture_error_state() to make
2060  * sure we get a record and make it available in debugfs.  Fire a uevent
2061  * so userspace knows something bad happened (should trigger collection
2062  * of a ring dump etc.).
2063  */
2064 void i915_handle_error(struct drm_device *dev, bool wedged)
2065 {
2066         struct drm_i915_private *dev_priv = dev->dev_private;
2067         struct intel_ring_buffer *ring;
2068         int i;
2069
2070         i915_capture_error_state(dev);
2071         i915_report_and_clear_eir(dev);
2072
2073         if (wedged) {
2074                 atomic_set_mask(I915_RESET_IN_PROGRESS_FLAG,
2075                                 &dev_priv->gpu_error.reset_counter);
2076
2077                 /*
2078                  * Wakeup waiting processes so that the reset work item
2079                  * doesn't deadlock trying to grab various locks.
2080                  */
2081                 for_each_ring(ring, dev_priv, i)
2082                         wake_up_all(&ring->irq_queue);
2083         }
2084
2085         queue_work(dev_priv->wq, &dev_priv->gpu_error.work);
2086 }
2087
2088 static void __always_unused i915_pageflip_stall_check(struct drm_device *dev, int pipe)
2089 {
2090         drm_i915_private_t *dev_priv = dev->dev_private;
2091         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
2092         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2093         struct drm_i915_gem_object *obj;
2094         struct intel_unpin_work *work;
2095         unsigned long flags;
2096         bool stall_detected;
2097
2098         /* Ignore early vblank irqs */
2099         if (intel_crtc == NULL)
2100                 return;
2101
2102         spin_lock_irqsave(&dev->event_lock, flags);
2103         work = intel_crtc->unpin_work;
2104
2105         if (work == NULL ||
2106             atomic_read(&work->pending) >= INTEL_FLIP_COMPLETE ||
2107             !work->enable_stall_check) {
2108                 /* Either the pending flip IRQ arrived, or we're too early. Don't check */
2109                 spin_unlock_irqrestore(&dev->event_lock, flags);
2110                 return;
2111         }
2112
2113         /* Potential stall - if we see that the flip has happened, assume a missed interrupt */
2114         obj = work->pending_flip_obj;
2115         if (INTEL_INFO(dev)->gen >= 4) {
2116                 int dspsurf = DSPSURF(intel_crtc->plane);
2117                 stall_detected = I915_HI_DISPBASE(I915_READ(dspsurf)) ==
2118                                         obj->gtt_offset;
2119         } else {
2120                 int dspaddr = DSPADDR(intel_crtc->plane);
2121                 stall_detected = I915_READ(dspaddr) == (obj->gtt_offset +
2122                                                         crtc->y * crtc->fb->pitches[0] +
2123                                                         crtc->x * crtc->fb->bits_per_pixel/8);
2124         }
2125
2126         spin_unlock_irqrestore(&dev->event_lock, flags);
2127
2128         if (stall_detected) {
2129                 DRM_DEBUG_DRIVER("Pageflip stall detected\n");
2130                 intel_prepare_page_flip(dev, intel_crtc->plane);
2131         }
2132 }
2133
2134 /* Called from drm generic code, passed 'crtc' which
2135  * we use as a pipe index
2136  */
2137 static int i915_enable_vblank(struct drm_device *dev, int pipe)
2138 {
2139         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2140         unsigned long irqflags;
2141
2142         if (!i915_pipe_enabled(dev, pipe))
2143                 return -EINVAL;
2144
2145         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2146         if (INTEL_INFO(dev)->gen >= 4)
2147                 i915_enable_pipestat(dev_priv, pipe,
2148                                      PIPE_START_VBLANK_INTERRUPT_ENABLE);
2149         else
2150                 i915_enable_pipestat(dev_priv, pipe,
2151                                      PIPE_VBLANK_INTERRUPT_ENABLE);
2152
2153         /* maintain vblank delivery even in deep C-states */
2154         if (dev_priv->info->gen == 3)
2155                 I915_WRITE(INSTPM, _MASKED_BIT_DISABLE(INSTPM_AGPBUSY_DIS));
2156         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2157
2158         return 0;
2159 }
2160
2161 static int ironlake_enable_vblank(struct drm_device *dev, int pipe)
2162 {
2163         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2164         unsigned long irqflags;
2165
2166         if (!i915_pipe_enabled(dev, pipe))
2167                 return -EINVAL;
2168
2169         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2170         ironlake_enable_display_irq(dev_priv, (pipe == 0) ?
2171                                     DE_PIPEA_VBLANK : DE_PIPEB_VBLANK);
2172         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2173
2174         return 0;
2175 }
2176
2177 static int ivybridge_enable_vblank(struct drm_device *dev, int pipe)
2178 {
2179         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2180         unsigned long irqflags;
2181
2182         if (!i915_pipe_enabled(dev, pipe))
2183                 return -EINVAL;
2184
2185         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2186         ironlake_enable_display_irq(dev_priv,
2187                                     DE_PIPEA_VBLANK_IVB << (5 * pipe));
2188         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2189
2190         return 0;
2191 }
2192
2193 static int valleyview_enable_vblank(struct drm_device *dev, int pipe)
2194 {
2195         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2196         unsigned long irqflags;
2197         u32 imr;
2198
2199         if (!i915_pipe_enabled(dev, pipe))
2200                 return -EINVAL;
2201
2202         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2203         imr = I915_READ(VLV_IMR);
2204         if (pipe == 0)
2205                 imr &= ~I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT;
2206         else
2207                 imr &= ~I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
2208         I915_WRITE(VLV_IMR, imr);
2209         i915_enable_pipestat(dev_priv, pipe,
2210                              PIPE_START_VBLANK_INTERRUPT_ENABLE);
2211         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2212
2213         return 0;
2214 }
2215
2216 /* Called from drm generic code, passed 'crtc' which
2217  * we use as a pipe index
2218  */
2219 static void i915_disable_vblank(struct drm_device *dev, int pipe)
2220 {
2221         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2222         unsigned long irqflags;
2223
2224         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2225         if (dev_priv->info->gen == 3)
2226                 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_DIS));
2227
2228         i915_disable_pipestat(dev_priv, pipe,
2229                               PIPE_VBLANK_INTERRUPT_ENABLE |
2230                               PIPE_START_VBLANK_INTERRUPT_ENABLE);
2231         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2232 }
2233
2234 static void ironlake_disable_vblank(struct drm_device *dev, int pipe)
2235 {
2236         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2237         unsigned long irqflags;
2238
2239         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2240         ironlake_disable_display_irq(dev_priv, (pipe == 0) ?
2241                                      DE_PIPEA_VBLANK : DE_PIPEB_VBLANK);
2242         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2243 }
2244
2245 static void ivybridge_disable_vblank(struct drm_device *dev, int pipe)
2246 {
2247         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2248         unsigned long irqflags;
2249
2250         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2251         ironlake_disable_display_irq(dev_priv,
2252                                      DE_PIPEA_VBLANK_IVB << (pipe * 5));
2253         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2254 }
2255
2256 static void valleyview_disable_vblank(struct drm_device *dev, int pipe)
2257 {
2258         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2259         unsigned long irqflags;
2260         u32 imr;
2261
2262         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2263         i915_disable_pipestat(dev_priv, pipe,
2264                               PIPE_START_VBLANK_INTERRUPT_ENABLE);
2265         imr = I915_READ(VLV_IMR);
2266         if (pipe == 0)
2267                 imr |= I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT;
2268         else
2269                 imr |= I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
2270         I915_WRITE(VLV_IMR, imr);
2271         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2272 }
2273
2274 static u32
2275 ring_last_seqno(struct intel_ring_buffer *ring)
2276 {
2277         return list_entry(ring->request_list.prev,
2278                           struct drm_i915_gem_request, list)->seqno;
2279 }
2280
2281 static bool i915_hangcheck_ring_idle(struct intel_ring_buffer *ring,
2282                                      u32 ring_seqno, bool *err)
2283 {
2284         if (list_empty(&ring->request_list) ||
2285             i915_seqno_passed(ring_seqno, ring_last_seqno(ring))) {
2286                 /* Issue a wake-up to catch stuck h/w. */
2287                 if (waitqueue_active(&ring->irq_queue)) {
2288                         DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
2289                                   ring->name);
2290                         wake_up_all(&ring->irq_queue);
2291                         *err = true;
2292                 }
2293                 return true;
2294         }
2295         return false;
2296 }
2297
2298 static bool semaphore_passed(struct intel_ring_buffer *ring)
2299 {
2300         struct drm_i915_private *dev_priv = ring->dev->dev_private;
2301         u32 acthd = intel_ring_get_active_head(ring) & HEAD_ADDR;
2302         struct intel_ring_buffer *signaller;
2303         u32 cmd, ipehr, acthd_min;
2304
2305         ipehr = I915_READ(RING_IPEHR(ring->mmio_base));
2306         if ((ipehr & ~(0x3 << 16)) !=
2307             (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE | MI_SEMAPHORE_REGISTER))
2308                 return false;
2309
2310         /* ACTHD is likely pointing to the dword after the actual command,
2311          * so scan backwards until we find the MBOX.
2312          */
2313         acthd_min = max((int)acthd - 3 * 4, 0);
2314         do {
2315                 cmd = ioread32(ring->virtual_start + acthd);
2316                 if (cmd == ipehr)
2317                         break;
2318
2319                 acthd -= 4;
2320                 if (acthd < acthd_min)
2321                         return false;
2322         } while (1);
2323
2324         signaller = &dev_priv->ring[(ring->id + (((ipehr >> 17) & 1) + 1)) % 3];
2325         return i915_seqno_passed(signaller->get_seqno(signaller, false),
2326                                  ioread32(ring->virtual_start+acthd+4)+1);
2327 }
2328
2329 static bool kick_ring(struct intel_ring_buffer *ring)
2330 {
2331         struct drm_device *dev = ring->dev;
2332         struct drm_i915_private *dev_priv = dev->dev_private;
2333         u32 tmp = I915_READ_CTL(ring);
2334         if (tmp & RING_WAIT) {
2335                 DRM_ERROR("Kicking stuck wait on %s\n",
2336                           ring->name);
2337                 I915_WRITE_CTL(ring, tmp);
2338                 return true;
2339         }
2340
2341         if (INTEL_INFO(dev)->gen >= 6 &&
2342             tmp & RING_WAIT_SEMAPHORE &&
2343             semaphore_passed(ring)) {
2344                 DRM_ERROR("Kicking stuck semaphore on %s\n",
2345                           ring->name);
2346                 I915_WRITE_CTL(ring, tmp);
2347                 return true;
2348         }
2349         return false;
2350 }
2351
2352 static bool i915_hangcheck_ring_hung(struct intel_ring_buffer *ring)
2353 {
2354         if (IS_GEN2(ring->dev))
2355                 return false;
2356
2357         /* Is the chip hanging on a WAIT_FOR_EVENT?
2358          * If so we can simply poke the RB_WAIT bit
2359          * and break the hang. This should work on
2360          * all but the second generation chipsets.
2361          */
2362         return !kick_ring(ring);
2363 }
2364
2365 static bool i915_hangcheck_hung(struct drm_device *dev)
2366 {
2367         drm_i915_private_t *dev_priv = dev->dev_private;
2368
2369         if (dev_priv->gpu_error.hangcheck_count++ > 1) {
2370                 bool hung = true;
2371                 struct intel_ring_buffer *ring;
2372                 int i;
2373
2374                 DRM_ERROR("Hangcheck timer elapsed... GPU hung\n");
2375                 i915_handle_error(dev, true);
2376
2377                 for_each_ring(ring, dev_priv, i)
2378                         hung &= i915_hangcheck_ring_hung(ring);
2379
2380                 return hung;
2381         }
2382
2383         return false;
2384 }
2385
2386 /**
2387  * This is called when the chip hasn't reported back with completed
2388  * batchbuffers in a long time. The first time this is called we simply record
2389  * ACTHD. If ACTHD hasn't changed by the time the hangcheck timer elapses
2390  * again, we assume the chip is wedged and try to fix it.
2391  */
2392 void i915_hangcheck_elapsed(unsigned long data)
2393 {
2394         struct drm_device *dev = (struct drm_device *)data;
2395         drm_i915_private_t *dev_priv = dev->dev_private;
2396         struct intel_ring_buffer *ring;
2397         bool err = false, idle;
2398         int i;
2399         u32 seqno[I915_NUM_RINGS];
2400         bool work_done;
2401
2402         if (!i915_enable_hangcheck)
2403                 return;
2404
2405         idle = true;
2406         for_each_ring(ring, dev_priv, i) {
2407                 seqno[i] = ring->get_seqno(ring, false);
2408                 idle &= i915_hangcheck_ring_idle(ring, seqno[i], &err);
2409         }
2410
2411         /* If all work is done then ACTHD clearly hasn't advanced. */
2412         if (idle) {
2413                 if (err) {
2414                         if (i915_hangcheck_hung(dev))
2415                                 return;
2416
2417                         goto repeat;
2418                 }
2419
2420                 dev_priv->gpu_error.hangcheck_count = 0;
2421                 return;
2422         }
2423
2424         work_done = false;
2425         for_each_ring(ring, dev_priv, i) {
2426                 if (ring->hangcheck.seqno != seqno[i]) {
2427                         work_done = true;
2428                         ring->hangcheck.seqno = seqno[i];
2429                 }
2430         }
2431
2432         if (!work_done) {
2433                 if (i915_hangcheck_hung(dev))
2434                         return;
2435         } else {
2436                 dev_priv->gpu_error.hangcheck_count = 0;
2437         }
2438
2439 repeat:
2440         /* Reset timer case chip hangs without another request being added */
2441         mod_timer(&dev_priv->gpu_error.hangcheck_timer,
2442                   round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES));
2443 }
2444
2445 /* drm_dma.h hooks
2446 */
2447 static void ironlake_irq_preinstall(struct drm_device *dev)
2448 {
2449         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2450
2451         atomic_set(&dev_priv->irq_received, 0);
2452
2453         I915_WRITE(HWSTAM, 0xeffe);
2454
2455         /* XXX hotplug from PCH */
2456
2457         I915_WRITE(DEIMR, 0xffffffff);
2458         I915_WRITE(DEIER, 0x0);
2459         POSTING_READ(DEIER);
2460
2461         /* and GT */
2462         I915_WRITE(GTIMR, 0xffffffff);
2463         I915_WRITE(GTIER, 0x0);
2464         POSTING_READ(GTIER);
2465
2466         if (HAS_PCH_NOP(dev))
2467                 return;
2468
2469         /* south display irq */
2470         I915_WRITE(SDEIMR, 0xffffffff);
2471         /*
2472          * SDEIER is also touched by the interrupt handler to work around missed
2473          * PCH interrupts. Hence we can't update it after the interrupt handler
2474          * is enabled - instead we unconditionally enable all PCH interrupt
2475          * sources here, but then only unmask them as needed with SDEIMR.
2476          */
2477         I915_WRITE(SDEIER, 0xffffffff);
2478         POSTING_READ(SDEIER);
2479 }
2480
2481 static void valleyview_irq_preinstall(struct drm_device *dev)
2482 {
2483         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2484         int pipe;
2485
2486         atomic_set(&dev_priv->irq_received, 0);
2487
2488         /* VLV magic */
2489         I915_WRITE(VLV_IMR, 0);
2490         I915_WRITE(RING_IMR(RENDER_RING_BASE), 0);
2491         I915_WRITE(RING_IMR(GEN6_BSD_RING_BASE), 0);
2492         I915_WRITE(RING_IMR(BLT_RING_BASE), 0);
2493
2494         /* and GT */
2495         I915_WRITE(GTIIR, I915_READ(GTIIR));
2496         I915_WRITE(GTIIR, I915_READ(GTIIR));
2497         I915_WRITE(GTIMR, 0xffffffff);
2498         I915_WRITE(GTIER, 0x0);
2499         POSTING_READ(GTIER);
2500
2501         I915_WRITE(DPINVGTT, 0xff);
2502
2503         I915_WRITE(PORT_HOTPLUG_EN, 0);
2504         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2505         for_each_pipe(pipe)
2506                 I915_WRITE(PIPESTAT(pipe), 0xffff);
2507         I915_WRITE(VLV_IIR, 0xffffffff);
2508         I915_WRITE(VLV_IMR, 0xffffffff);
2509         I915_WRITE(VLV_IER, 0x0);
2510         POSTING_READ(VLV_IER);
2511 }
2512
2513 static void ibx_hpd_irq_setup(struct drm_device *dev)
2514 {
2515         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2516         struct drm_mode_config *mode_config = &dev->mode_config;
2517         struct intel_encoder *intel_encoder;
2518         u32 mask = ~I915_READ(SDEIMR);
2519         u32 hotplug;
2520
2521         if (HAS_PCH_IBX(dev)) {
2522                 mask &= ~SDE_HOTPLUG_MASK;
2523                 list_for_each_entry(intel_encoder, &mode_config->encoder_list, base.head)
2524                         if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
2525                                 mask |= hpd_ibx[intel_encoder->hpd_pin];
2526         } else {
2527                 mask &= ~SDE_HOTPLUG_MASK_CPT;
2528                 list_for_each_entry(intel_encoder, &mode_config->encoder_list, base.head)
2529                         if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
2530                                 mask |= hpd_cpt[intel_encoder->hpd_pin];
2531         }
2532
2533         I915_WRITE(SDEIMR, ~mask);
2534
2535         /*
2536          * Enable digital hotplug on the PCH, and configure the DP short pulse
2537          * duration to 2ms (which is the minimum in the Display Port spec)
2538          *
2539          * This register is the same on all known PCH chips.
2540          */
2541         hotplug = I915_READ(PCH_PORT_HOTPLUG);
2542         hotplug &= ~(PORTD_PULSE_DURATION_MASK|PORTC_PULSE_DURATION_MASK|PORTB_PULSE_DURATION_MASK);
2543         hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms;
2544         hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms;
2545         hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms;
2546         I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
2547 }
2548
2549 static void ibx_irq_postinstall(struct drm_device *dev)
2550 {
2551         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2552         u32 mask;
2553
2554         if (HAS_PCH_IBX(dev)) {
2555                 mask = SDE_GMBUS | SDE_AUX_MASK | SDE_TRANSB_FIFO_UNDER |
2556                        SDE_TRANSA_FIFO_UNDER | SDE_POISON;
2557         } else {
2558                 mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT | SDE_ERROR_CPT;
2559
2560                 I915_WRITE(SERR_INT, I915_READ(SERR_INT));
2561         }
2562
2563         if (HAS_PCH_NOP(dev))
2564                 return;
2565
2566         I915_WRITE(SDEIIR, I915_READ(SDEIIR));
2567         I915_WRITE(SDEIMR, ~mask);
2568 }
2569
2570 static int ironlake_irq_postinstall(struct drm_device *dev)
2571 {
2572         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2573         /* enable kind of interrupts always enabled */
2574         u32 display_mask = DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
2575                            DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE |
2576                            DE_AUX_CHANNEL_A | DE_PIPEB_FIFO_UNDERRUN |
2577                            DE_PIPEA_FIFO_UNDERRUN | DE_POISON;
2578         u32 render_irqs;
2579
2580         dev_priv->irq_mask = ~display_mask;
2581
2582         /* should always can generate irq */
2583         I915_WRITE(DEIIR, I915_READ(DEIIR));
2584         I915_WRITE(DEIMR, dev_priv->irq_mask);
2585         I915_WRITE(DEIER, display_mask | DE_PIPEA_VBLANK | DE_PIPEB_VBLANK);
2586         POSTING_READ(DEIER);
2587
2588         dev_priv->gt_irq_mask = ~0;
2589
2590         I915_WRITE(GTIIR, I915_READ(GTIIR));
2591         I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
2592
2593         if (IS_GEN6(dev))
2594                 render_irqs =
2595                         GT_USER_INTERRUPT |
2596                         GEN6_BSD_USER_INTERRUPT |
2597                         GEN6_BLITTER_USER_INTERRUPT;
2598         else
2599                 render_irqs =
2600                         GT_USER_INTERRUPT |
2601                         GT_PIPE_NOTIFY |
2602                         GT_BSD_USER_INTERRUPT;
2603         I915_WRITE(GTIER, render_irqs);
2604         POSTING_READ(GTIER);
2605
2606         ibx_irq_postinstall(dev);
2607
2608         if (IS_IRONLAKE_M(dev)) {
2609                 /* Clear & enable PCU event interrupts */
2610                 I915_WRITE(DEIIR, DE_PCU_EVENT);
2611                 I915_WRITE(DEIER, I915_READ(DEIER) | DE_PCU_EVENT);
2612                 ironlake_enable_display_irq(dev_priv, DE_PCU_EVENT);
2613         }
2614
2615         return 0;
2616 }
2617
2618 static int ivybridge_irq_postinstall(struct drm_device *dev)
2619 {
2620         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2621         /* enable kind of interrupts always enabled */
2622         u32 display_mask =
2623                 DE_MASTER_IRQ_CONTROL | DE_GSE_IVB | DE_PCH_EVENT_IVB |
2624                 DE_PLANEC_FLIP_DONE_IVB |
2625                 DE_PLANEB_FLIP_DONE_IVB |
2626                 DE_PLANEA_FLIP_DONE_IVB |
2627                 DE_AUX_CHANNEL_A_IVB |
2628                 DE_ERR_INT_IVB;
2629         u32 render_irqs;
2630
2631         dev_priv->irq_mask = ~display_mask;
2632
2633         /* should always can generate irq */
2634         I915_WRITE(GEN7_ERR_INT, I915_READ(GEN7_ERR_INT));
2635         I915_WRITE(DEIIR, I915_READ(DEIIR));
2636         I915_WRITE(DEIMR, dev_priv->irq_mask);
2637         I915_WRITE(DEIER,
2638                    display_mask |
2639                    DE_PIPEC_VBLANK_IVB |
2640                    DE_PIPEB_VBLANK_IVB |
2641                    DE_PIPEA_VBLANK_IVB);
2642         POSTING_READ(DEIER);
2643
2644         dev_priv->gt_irq_mask = ~GT_GEN7_L3_PARITY_ERROR_INTERRUPT;
2645
2646         I915_WRITE(GTIIR, I915_READ(GTIIR));
2647         I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
2648
2649         render_irqs = GT_USER_INTERRUPT | GEN6_BSD_USER_INTERRUPT |
2650                 GEN6_BLITTER_USER_INTERRUPT | GT_GEN7_L3_PARITY_ERROR_INTERRUPT;
2651         I915_WRITE(GTIER, render_irqs);
2652         POSTING_READ(GTIER);
2653
2654         ibx_irq_postinstall(dev);
2655
2656         return 0;
2657 }
2658
2659 static int valleyview_irq_postinstall(struct drm_device *dev)
2660 {
2661         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2662         u32 enable_mask;
2663         u32 pipestat_enable = PLANE_FLIP_DONE_INT_EN_VLV;
2664         u32 render_irqs;
2665
2666         enable_mask = I915_DISPLAY_PORT_INTERRUPT;
2667         enable_mask |= I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2668                 I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT |
2669                 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2670                 I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
2671
2672         /*
2673          *Leave vblank interrupts masked initially.  enable/disable will
2674          * toggle them based on usage.
2675          */
2676         dev_priv->irq_mask = (~enable_mask) |
2677                 I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT |
2678                 I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
2679
2680         I915_WRITE(PORT_HOTPLUG_EN, 0);
2681         POSTING_READ(PORT_HOTPLUG_EN);
2682
2683         I915_WRITE(VLV_IMR, dev_priv->irq_mask);
2684         I915_WRITE(VLV_IER, enable_mask);
2685         I915_WRITE(VLV_IIR, 0xffffffff);
2686         I915_WRITE(PIPESTAT(0), 0xffff);
2687         I915_WRITE(PIPESTAT(1), 0xffff);
2688         POSTING_READ(VLV_IER);
2689
2690         i915_enable_pipestat(dev_priv, 0, pipestat_enable);
2691         i915_enable_pipestat(dev_priv, 0, PIPE_GMBUS_EVENT_ENABLE);
2692         i915_enable_pipestat(dev_priv, 1, pipestat_enable);
2693
2694         I915_WRITE(VLV_IIR, 0xffffffff);
2695         I915_WRITE(VLV_IIR, 0xffffffff);
2696
2697         I915_WRITE(GTIIR, I915_READ(GTIIR));
2698         I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
2699
2700         render_irqs = GT_USER_INTERRUPT | GEN6_BSD_USER_INTERRUPT |
2701                 GEN6_BLITTER_USER_INTERRUPT;
2702         I915_WRITE(GTIER, render_irqs);
2703         POSTING_READ(GTIER);
2704
2705         /* ack & enable invalid PTE error interrupts */
2706 #if 0 /* FIXME: add support to irq handler for checking these bits */
2707         I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
2708         I915_WRITE(DPINVGTT, DPINVGTT_EN_MASK);
2709 #endif
2710
2711         I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
2712
2713         return 0;
2714 }
2715
2716 static void valleyview_irq_uninstall(struct drm_device *dev)
2717 {
2718         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2719         int pipe;
2720
2721         if (!dev_priv)
2722                 return;
2723
2724         del_timer_sync(&dev_priv->hotplug_reenable_timer);
2725
2726         for_each_pipe(pipe)
2727                 I915_WRITE(PIPESTAT(pipe), 0xffff);
2728
2729         I915_WRITE(HWSTAM, 0xffffffff);
2730         I915_WRITE(PORT_HOTPLUG_EN, 0);
2731         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2732         for_each_pipe(pipe)
2733                 I915_WRITE(PIPESTAT(pipe), 0xffff);
2734         I915_WRITE(VLV_IIR, 0xffffffff);
2735         I915_WRITE(VLV_IMR, 0xffffffff);
2736         I915_WRITE(VLV_IER, 0x0);
2737         POSTING_READ(VLV_IER);
2738 }
2739
2740 static void ironlake_irq_uninstall(struct drm_device *dev)
2741 {
2742         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2743
2744         if (!dev_priv)
2745                 return;
2746
2747         del_timer_sync(&dev_priv->hotplug_reenable_timer);
2748
2749         I915_WRITE(HWSTAM, 0xffffffff);
2750
2751         I915_WRITE(DEIMR, 0xffffffff);
2752         I915_WRITE(DEIER, 0x0);
2753         I915_WRITE(DEIIR, I915_READ(DEIIR));
2754         if (IS_GEN7(dev))
2755                 I915_WRITE(GEN7_ERR_INT, I915_READ(GEN7_ERR_INT));
2756
2757         I915_WRITE(GTIMR, 0xffffffff);
2758         I915_WRITE(GTIER, 0x0);
2759         I915_WRITE(GTIIR, I915_READ(GTIIR));
2760
2761         if (HAS_PCH_NOP(dev))
2762                 return;
2763
2764         I915_WRITE(SDEIMR, 0xffffffff);
2765         I915_WRITE(SDEIER, 0x0);
2766         I915_WRITE(SDEIIR, I915_READ(SDEIIR));
2767         if (HAS_PCH_CPT(dev) || HAS_PCH_LPT(dev))
2768                 I915_WRITE(SERR_INT, I915_READ(SERR_INT));
2769 }
2770
2771 static void i8xx_irq_preinstall(struct drm_device * dev)
2772 {
2773         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2774         int pipe;
2775
2776         atomic_set(&dev_priv->irq_received, 0);
2777
2778         for_each_pipe(pipe)
2779                 I915_WRITE(PIPESTAT(pipe), 0);
2780         I915_WRITE16(IMR, 0xffff);
2781         I915_WRITE16(IER, 0x0);
2782         POSTING_READ16(IER);
2783 }
2784
2785 static int i8xx_irq_postinstall(struct drm_device *dev)
2786 {
2787         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2788
2789         I915_WRITE16(EMR,
2790                      ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
2791
2792         /* Unmask the interrupts that we always want on. */
2793         dev_priv->irq_mask =
2794                 ~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2795                   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2796                   I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2797                   I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
2798                   I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
2799         I915_WRITE16(IMR, dev_priv->irq_mask);
2800
2801         I915_WRITE16(IER,
2802                      I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2803                      I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2804                      I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT |
2805                      I915_USER_INTERRUPT);
2806         POSTING_READ16(IER);
2807
2808         return 0;
2809 }
2810
2811 /*
2812  * Returns true when a page flip has completed.
2813  */
2814 static bool i8xx_handle_vblank(struct drm_device *dev,
2815                                int pipe, u16 iir)
2816 {
2817         drm_i915_private_t *dev_priv = dev->dev_private;
2818         u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(pipe);
2819
2820         if (!drm_handle_vblank(dev, pipe))
2821                 return false;
2822
2823         if ((iir & flip_pending) == 0)
2824                 return false;
2825
2826         intel_prepare_page_flip(dev, pipe);
2827
2828         /* We detect FlipDone by looking for the change in PendingFlip from '1'
2829          * to '0' on the following vblank, i.e. IIR has the Pendingflip
2830          * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
2831          * the flip is completed (no longer pending). Since this doesn't raise
2832          * an interrupt per se, we watch for the change at vblank.
2833          */
2834         if (I915_READ16(ISR) & flip_pending)
2835                 return false;
2836
2837         intel_finish_page_flip(dev, pipe);
2838
2839         return true;
2840 }
2841
2842 static irqreturn_t i8xx_irq_handler(int irq, void *arg)
2843 {
2844         struct drm_device *dev = (struct drm_device *) arg;
2845         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2846         u16 iir, new_iir;
2847         u32 pipe_stats[2];
2848         unsigned long irqflags;
2849         int irq_received;
2850         int pipe;
2851         u16 flip_mask =
2852                 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2853                 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
2854
2855         atomic_inc(&dev_priv->irq_received);
2856
2857         iir = I915_READ16(IIR);
2858         if (iir == 0)
2859                 return IRQ_NONE;
2860
2861         while (iir & ~flip_mask) {
2862                 /* Can't rely on pipestat interrupt bit in iir as it might
2863                  * have been cleared after the pipestat interrupt was received.
2864                  * It doesn't set the bit in iir again, but it still produces
2865                  * interrupts (for non-MSI).
2866                  */
2867                 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2868                 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
2869                         i915_handle_error(dev, false);
2870
2871                 for_each_pipe(pipe) {
2872                         int reg = PIPESTAT(pipe);
2873                         pipe_stats[pipe] = I915_READ(reg);
2874
2875                         /*
2876                          * Clear the PIPE*STAT regs before the IIR
2877                          */
2878                         if (pipe_stats[pipe] & 0x8000ffff) {
2879                                 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
2880                                         DRM_DEBUG_DRIVER("pipe %c underrun\n",
2881                                                          pipe_name(pipe));
2882                                 I915_WRITE(reg, pipe_stats[pipe]);
2883                                 irq_received = 1;
2884                         }
2885                 }
2886                 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2887
2888                 I915_WRITE16(IIR, iir & ~flip_mask);
2889                 new_iir = I915_READ16(IIR); /* Flush posted writes */
2890
2891                 i915_update_dri1_breadcrumb(dev);
2892
2893                 if (iir & I915_USER_INTERRUPT)
2894                         notify_ring(dev, &dev_priv->ring[RCS]);
2895
2896                 if (pipe_stats[0] & PIPE_VBLANK_INTERRUPT_STATUS &&
2897                     i8xx_handle_vblank(dev, 0, iir))
2898                         flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(0);
2899
2900                 if (pipe_stats[1] & PIPE_VBLANK_INTERRUPT_STATUS &&
2901                     i8xx_handle_vblank(dev, 1, iir))
2902                         flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(1);
2903
2904                 iir = new_iir;
2905         }
2906
2907         return IRQ_HANDLED;
2908 }
2909
2910 static void i8xx_irq_uninstall(struct drm_device * dev)
2911 {
2912         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2913         int pipe;
2914
2915         for_each_pipe(pipe) {
2916                 /* Clear enable bits; then clear status bits */
2917                 I915_WRITE(PIPESTAT(pipe), 0);
2918                 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
2919         }
2920         I915_WRITE16(IMR, 0xffff);
2921         I915_WRITE16(IER, 0x0);
2922         I915_WRITE16(IIR, I915_READ16(IIR));
2923 }
2924
2925 static void i915_irq_preinstall(struct drm_device * dev)
2926 {
2927         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2928         int pipe;
2929
2930         atomic_set(&dev_priv->irq_received, 0);
2931
2932         if (I915_HAS_HOTPLUG(dev)) {
2933                 I915_WRITE(PORT_HOTPLUG_EN, 0);
2934                 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2935         }
2936
2937         I915_WRITE16(HWSTAM, 0xeffe);
2938         for_each_pipe(pipe)
2939                 I915_WRITE(PIPESTAT(pipe), 0);
2940         I915_WRITE(IMR, 0xffffffff);
2941         I915_WRITE(IER, 0x0);
2942         POSTING_READ(IER);
2943 }
2944
2945 static int i915_irq_postinstall(struct drm_device *dev)
2946 {
2947         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2948         u32 enable_mask;
2949
2950         I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
2951
2952         /* Unmask the interrupts that we always want on. */
2953         dev_priv->irq_mask =
2954                 ~(I915_ASLE_INTERRUPT |
2955                   I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2956                   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2957                   I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
2958                   I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
2959                   I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
2960
2961         enable_mask =
2962                 I915_ASLE_INTERRUPT |
2963                 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2964                 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2965                 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT |
2966                 I915_USER_INTERRUPT;
2967
2968         if (I915_HAS_HOTPLUG(dev)) {
2969                 I915_WRITE(PORT_HOTPLUG_EN, 0);
2970                 POSTING_READ(PORT_HOTPLUG_EN);
2971
2972                 /* Enable in IER... */
2973                 enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
2974                 /* and unmask in IMR */
2975                 dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
2976         }
2977
2978         I915_WRITE(IMR, dev_priv->irq_mask);
2979         I915_WRITE(IER, enable_mask);
2980         POSTING_READ(IER);
2981
2982         i915_enable_asle_pipestat(dev);
2983
2984         return 0;
2985 }
2986
2987 /*
2988  * Returns true when a page flip has completed.
2989  */
2990 static bool i915_handle_vblank(struct drm_device *dev,
2991                                int plane, int pipe, u32 iir)
2992 {
2993         drm_i915_private_t *dev_priv = dev->dev_private;
2994         u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
2995
2996         if (!drm_handle_vblank(dev, pipe))
2997                 return false;
2998
2999         if ((iir & flip_pending) == 0)
3000                 return false;
3001
3002         intel_prepare_page_flip(dev, plane);
3003
3004         /* We detect FlipDone by looking for the change in PendingFlip from '1'
3005          * to '0' on the following vblank, i.e. IIR has the Pendingflip
3006          * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
3007          * the flip is completed (no longer pending). Since this doesn't raise
3008          * an interrupt per se, we watch for the change at vblank.
3009          */
3010         if (I915_READ(ISR) & flip_pending)
3011                 return false;
3012
3013         intel_finish_page_flip(dev, pipe);
3014
3015         return true;
3016 }
3017
3018 static irqreturn_t i915_irq_handler(int irq, void *arg)
3019 {
3020         struct drm_device *dev = (struct drm_device *) arg;
3021         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
3022         u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
3023         unsigned long irqflags;
3024         u32 flip_mask =
3025                 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3026                 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
3027         int pipe, ret = IRQ_NONE;
3028
3029         atomic_inc(&dev_priv->irq_received);
3030
3031         iir = I915_READ(IIR);
3032         do {
3033                 bool irq_received = (iir & ~flip_mask) != 0;
3034                 bool blc_event = false;
3035
3036                 /* Can't rely on pipestat interrupt bit in iir as it might
3037                  * have been cleared after the pipestat interrupt was received.
3038                  * It doesn't set the bit in iir again, but it still produces
3039                  * interrupts (for non-MSI).
3040                  */
3041                 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3042                 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3043                         i915_handle_error(dev, false);
3044
3045                 for_each_pipe(pipe) {
3046                         int reg = PIPESTAT(pipe);
3047                         pipe_stats[pipe] = I915_READ(reg);
3048
3049                         /* Clear the PIPE*STAT regs before the IIR */
3050                         if (pipe_stats[pipe] & 0x8000ffff) {
3051                                 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
3052                                         DRM_DEBUG_DRIVER("pipe %c underrun\n",
3053                                                          pipe_name(pipe));
3054                                 I915_WRITE(reg, pipe_stats[pipe]);
3055                                 irq_received = true;
3056                         }
3057                 }
3058                 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3059
3060                 if (!irq_received)
3061                         break;
3062
3063                 /* Consume port.  Then clear IIR or we'll miss events */
3064                 if ((I915_HAS_HOTPLUG(dev)) &&
3065                     (iir & I915_DISPLAY_PORT_INTERRUPT)) {
3066                         u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
3067                         u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
3068
3069                         DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n",
3070                                   hotplug_status);
3071                         if (hotplug_trigger) {
3072                                 if (hotplug_irq_storm_detect(dev, hotplug_trigger, hpd_status_i915))
3073                                         i915_hpd_irq_setup(dev);
3074                                 queue_work(dev_priv->wq,
3075                                            &dev_priv->hotplug_work);
3076                         }
3077                         I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
3078                         POSTING_READ(PORT_HOTPLUG_STAT);
3079                 }
3080
3081                 I915_WRITE(IIR, iir & ~flip_mask);
3082                 new_iir = I915_READ(IIR); /* Flush posted writes */
3083
3084                 if (iir & I915_USER_INTERRUPT)
3085                         notify_ring(dev, &dev_priv->ring[RCS]);
3086
3087                 for_each_pipe(pipe) {
3088                         int plane = pipe;
3089                         if (IS_MOBILE(dev))
3090                                 plane = !plane;
3091
3092                         if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
3093                             i915_handle_vblank(dev, plane, pipe, iir))
3094                                 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
3095
3096                         if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
3097                                 blc_event = true;
3098                 }
3099
3100                 if (blc_event || (iir & I915_ASLE_INTERRUPT))
3101                         intel_opregion_asle_intr(dev);
3102
3103                 /* With MSI, interrupts are only generated when iir
3104                  * transitions from zero to nonzero.  If another bit got
3105                  * set while we were handling the existing iir bits, then
3106                  * we would never get another interrupt.
3107                  *
3108                  * This is fine on non-MSI as well, as if we hit this path
3109                  * we avoid exiting the interrupt handler only to generate
3110                  * another one.
3111                  *
3112                  * Note that for MSI this could cause a stray interrupt report
3113                  * if an interrupt landed in the time between writing IIR and
3114                  * the posting read.  This should be rare enough to never
3115                  * trigger the 99% of 100,000 interrupts test for disabling
3116                  * stray interrupts.
3117                  */
3118                 ret = IRQ_HANDLED;
3119                 iir = new_iir;
3120         } while (iir & ~flip_mask);
3121
3122         i915_update_dri1_breadcrumb(dev);
3123
3124         return ret;
3125 }
3126
3127 static void i915_irq_uninstall(struct drm_device * dev)
3128 {
3129         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
3130         int pipe;
3131
3132         del_timer_sync(&dev_priv->hotplug_reenable_timer);
3133
3134         if (I915_HAS_HOTPLUG(dev)) {
3135                 I915_WRITE(PORT_HOTPLUG_EN, 0);
3136                 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3137         }
3138
3139         I915_WRITE16(HWSTAM, 0xffff);
3140         for_each_pipe(pipe) {
3141                 /* Clear enable bits; then clear status bits */
3142                 I915_WRITE(PIPESTAT(pipe), 0);
3143                 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
3144         }
3145         I915_WRITE(IMR, 0xffffffff);
3146         I915_WRITE(IER, 0x0);
3147
3148         I915_WRITE(IIR, I915_READ(IIR));
3149 }
3150
3151 static void i965_irq_preinstall(struct drm_device * dev)
3152 {
3153         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
3154         int pipe;
3155
3156         atomic_set(&dev_priv->irq_received, 0);
3157
3158         I915_WRITE(PORT_HOTPLUG_EN, 0);
3159         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3160
3161         I915_WRITE(HWSTAM, 0xeffe);
3162         for_each_pipe(pipe)
3163                 I915_WRITE(PIPESTAT(pipe), 0);
3164         I915_WRITE(IMR, 0xffffffff);
3165         I915_WRITE(IER, 0x0);
3166         POSTING_READ(IER);
3167 }
3168
3169 static int i965_irq_postinstall(struct drm_device *dev)
3170 {
3171         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
3172         u32 enable_mask;
3173         u32 error_mask;
3174
3175         /* Unmask the interrupts that we always want on. */
3176         dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT |
3177                                I915_DISPLAY_PORT_INTERRUPT |
3178                                I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3179                                I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3180                                I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3181                                I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
3182                                I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
3183
3184         enable_mask = ~dev_priv->irq_mask;
3185         enable_mask &= ~(I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3186                          I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
3187         enable_mask |= I915_USER_INTERRUPT;
3188
3189         if (IS_G4X(dev))
3190                 enable_mask |= I915_BSD_USER_INTERRUPT;
3191
3192         i915_enable_pipestat(dev_priv, 0, PIPE_GMBUS_EVENT_ENABLE);
3193
3194         /*
3195          * Enable some error detection, note the instruction error mask
3196          * bit is reserved, so we leave it masked.
3197          */
3198         if (IS_G4X(dev)) {
3199                 error_mask = ~(GM45_ERROR_PAGE_TABLE |
3200                                GM45_ERROR_MEM_PRIV |
3201                                GM45_ERROR_CP_PRIV |
3202                                I915_ERROR_MEMORY_REFRESH);
3203         } else {
3204                 error_mask = ~(I915_ERROR_PAGE_TABLE |
3205                                I915_ERROR_MEMORY_REFRESH);
3206         }
3207         I915_WRITE(EMR, error_mask);
3208
3209         I915_WRITE(IMR, dev_priv->irq_mask);
3210         I915_WRITE(IER, enable_mask);
3211         POSTING_READ(IER);
3212
3213         I915_WRITE(PORT_HOTPLUG_EN, 0);
3214         POSTING_READ(PORT_HOTPLUG_EN);
3215
3216         i915_enable_asle_pipestat(dev);
3217
3218         return 0;
3219 }
3220
3221 static void i915_hpd_irq_setup(struct drm_device *dev)
3222 {
3223         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
3224         struct drm_mode_config *mode_config = &dev->mode_config;
3225         struct intel_encoder *intel_encoder;
3226         u32 hotplug_en;
3227
3228         if (I915_HAS_HOTPLUG(dev)) {
3229                 hotplug_en = I915_READ(PORT_HOTPLUG_EN);
3230                 hotplug_en &= ~HOTPLUG_INT_EN_MASK;
3231                 /* Note HDMI and DP share hotplug bits */
3232                 /* enable bits are the same for all generations */
3233                 list_for_each_entry(intel_encoder, &mode_config->encoder_list, base.head)
3234                         if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
3235                                 hotplug_en |= hpd_mask_i915[intel_encoder->hpd_pin];
3236                 /* Programming the CRT detection parameters tends
3237                    to generate a spurious hotplug event about three
3238                    seconds later.  So just do it once.
3239                 */
3240                 if (IS_G4X(dev))
3241                         hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
3242                 hotplug_en &= ~CRT_HOTPLUG_VOLTAGE_COMPARE_MASK;
3243                 hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
3244
3245                 /* Ignore TV since it's buggy */
3246                 I915_WRITE(PORT_HOTPLUG_EN, hotplug_en);
3247         }
3248 }
3249
3250 static irqreturn_t i965_irq_handler(int irq, void *arg)
3251 {
3252         struct drm_device *dev = (struct drm_device *) arg;
3253         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
3254         u32 iir, new_iir;
3255         u32 pipe_stats[I915_MAX_PIPES];
3256         unsigned long irqflags;
3257         int irq_received;
3258         int ret = IRQ_NONE, pipe;
3259         u32 flip_mask =
3260                 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3261                 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
3262
3263         atomic_inc(&dev_priv->irq_received);
3264
3265         iir = I915_READ(IIR);
3266
3267         for (;;) {
3268                 bool blc_event = false;
3269
3270                 irq_received = (iir & ~flip_mask) != 0;
3271
3272                 /* Can't rely on pipestat interrupt bit in iir as it might
3273                  * have been cleared after the pipestat interrupt was received.
3274                  * It doesn't set the bit in iir again, but it still produces
3275                  * interrupts (for non-MSI).
3276                  */
3277                 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3278                 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3279                         i915_handle_error(dev, false);
3280
3281                 for_each_pipe(pipe) {
3282                         int reg = PIPESTAT(pipe);
3283                         pipe_stats[pipe] = I915_READ(reg);
3284
3285                         /*
3286                          * Clear the PIPE*STAT regs before the IIR
3287                          */
3288                         if (pipe_stats[pipe] & 0x8000ffff) {
3289                                 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
3290                                         DRM_DEBUG_DRIVER("pipe %c underrun\n",
3291                                                          pipe_name(pipe));
3292                                 I915_WRITE(reg, pipe_stats[pipe]);
3293                                 irq_received = 1;
3294                         }
3295                 }
3296                 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3297
3298                 if (!irq_received)
3299                         break;
3300
3301                 ret = IRQ_HANDLED;
3302
3303                 /* Consume port.  Then clear IIR or we'll miss events */
3304                 if (iir & I915_DISPLAY_PORT_INTERRUPT) {
3305                         u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
3306                         u32 hotplug_trigger = hotplug_status & (IS_G4X(dev) ?
3307                                                                   HOTPLUG_INT_STATUS_G4X :
3308                                                                   HOTPLUG_INT_STATUS_I965);
3309
3310                         DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n",
3311                                   hotplug_status);
3312                         if (hotplug_trigger) {
3313                                 if (hotplug_irq_storm_detect(dev, hotplug_trigger,
3314                                                             IS_G4X(dev) ? hpd_status_gen4 : hpd_status_i965))
3315                                         i915_hpd_irq_setup(dev);
3316                                 queue_work(dev_priv->wq,
3317                                            &dev_priv->hotplug_work);
3318                         }
3319                         I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
3320                         I915_READ(PORT_HOTPLUG_STAT);
3321                 }
3322
3323                 I915_WRITE(IIR, iir & ~flip_mask);
3324                 new_iir = I915_READ(IIR); /* Flush posted writes */
3325
3326                 if (iir & I915_USER_INTERRUPT)
3327                         notify_ring(dev, &dev_priv->ring[RCS]);
3328                 if (iir & I915_BSD_USER_INTERRUPT)
3329                         notify_ring(dev, &dev_priv->ring[VCS]);
3330
3331                 for_each_pipe(pipe) {
3332                         if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
3333                             i915_handle_vblank(dev, pipe, pipe, iir))
3334                                 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe);
3335
3336                         if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
3337                                 blc_event = true;
3338                 }
3339
3340
3341                 if (blc_event || (iir & I915_ASLE_INTERRUPT))
3342                         intel_opregion_asle_intr(dev);
3343
3344                 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
3345                         gmbus_irq_handler(dev);
3346
3347                 /* With MSI, interrupts are only generated when iir
3348                  * transitions from zero to nonzero.  If another bit got
3349                  * set while we were handling the existing iir bits, then
3350                  * we would never get another interrupt.
3351                  *
3352                  * This is fine on non-MSI as well, as if we hit this path
3353                  * we avoid exiting the interrupt handler only to generate
3354                  * another one.
3355                  *
3356                  * Note that for MSI this could cause a stray interrupt report
3357                  * if an interrupt landed in the time between writing IIR and
3358                  * the posting read.  This should be rare enough to never
3359                  * trigger the 99% of 100,000 interrupts test for disabling
3360                  * stray interrupts.
3361                  */
3362                 iir = new_iir;
3363         }
3364
3365         i915_update_dri1_breadcrumb(dev);
3366
3367         return ret;
3368 }
3369
3370 static void i965_irq_uninstall(struct drm_device * dev)
3371 {
3372         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
3373         int pipe;
3374
3375         if (!dev_priv)
3376                 return;
3377
3378         del_timer_sync(&dev_priv->hotplug_reenable_timer);
3379
3380         I915_WRITE(PORT_HOTPLUG_EN, 0);
3381         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3382
3383         I915_WRITE(HWSTAM, 0xffffffff);
3384         for_each_pipe(pipe)
3385                 I915_WRITE(PIPESTAT(pipe), 0);
3386         I915_WRITE(IMR, 0xffffffff);
3387         I915_WRITE(IER, 0x0);
3388
3389         for_each_pipe(pipe)
3390                 I915_WRITE(PIPESTAT(pipe),
3391                            I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
3392         I915_WRITE(IIR, I915_READ(IIR));
3393 }
3394
3395 static void i915_reenable_hotplug_timer_func(unsigned long data)
3396 {
3397         drm_i915_private_t *dev_priv = (drm_i915_private_t *)data;
3398         struct drm_device *dev = dev_priv->dev;
3399         struct drm_mode_config *mode_config = &dev->mode_config;
3400         unsigned long irqflags;
3401         int i;
3402
3403         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3404         for (i = (HPD_NONE + 1); i < HPD_NUM_PINS; i++) {
3405                 struct drm_connector *connector;
3406
3407                 if (dev_priv->hpd_stats[i].hpd_mark != HPD_DISABLED)
3408                         continue;
3409
3410                 dev_priv->hpd_stats[i].hpd_mark = HPD_ENABLED;
3411
3412                 list_for_each_entry(connector, &mode_config->connector_list, head) {
3413                         struct intel_connector *intel_connector = to_intel_connector(connector);
3414
3415                         if (intel_connector->encoder->hpd_pin == i) {
3416                                 if (connector->polled != intel_connector->polled)
3417                                         DRM_DEBUG_DRIVER("Reenabling HPD on connector %s\n",
3418                                                          drm_get_connector_name(connector));
3419                                 connector->polled = intel_connector->polled;
3420                                 if (!connector->polled)
3421                                         connector->polled = DRM_CONNECTOR_POLL_HPD;
3422                         }
3423                 }
3424         }
3425         if (dev_priv->display.hpd_irq_setup)
3426                 dev_priv->display.hpd_irq_setup(dev);
3427         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3428 }
3429
3430 void intel_irq_init(struct drm_device *dev)
3431 {
3432         struct drm_i915_private *dev_priv = dev->dev_private;
3433
3434         INIT_WORK(&dev_priv->hotplug_work, i915_hotplug_work_func);
3435         INIT_WORK(&dev_priv->gpu_error.work, i915_error_work_func);
3436         INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
3437         INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
3438
3439         setup_timer(&dev_priv->gpu_error.hangcheck_timer,
3440                     i915_hangcheck_elapsed,
3441                     (unsigned long) dev);
3442         setup_timer(&dev_priv->hotplug_reenable_timer, i915_reenable_hotplug_timer_func,
3443                     (unsigned long) dev_priv);
3444
3445         pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY, PM_QOS_DEFAULT_VALUE);
3446
3447         dev->driver->get_vblank_counter = i915_get_vblank_counter;
3448         dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
3449         if (IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
3450                 dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
3451                 dev->driver->get_vblank_counter = gm45_get_vblank_counter;
3452         }
3453
3454         if (drm_core_check_feature(dev, DRIVER_MODESET))
3455                 dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
3456         else
3457                 dev->driver->get_vblank_timestamp = NULL;
3458         dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
3459
3460         if (IS_VALLEYVIEW(dev)) {
3461                 dev->driver->irq_handler = valleyview_irq_handler;
3462                 dev->driver->irq_preinstall = valleyview_irq_preinstall;
3463                 dev->driver->irq_postinstall = valleyview_irq_postinstall;
3464                 dev->driver->irq_uninstall = valleyview_irq_uninstall;
3465                 dev->driver->enable_vblank = valleyview_enable_vblank;
3466                 dev->driver->disable_vblank = valleyview_disable_vblank;
3467                 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
3468         } else if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev)) {
3469                 /* Share pre & uninstall handlers with ILK/SNB */
3470                 dev->driver->irq_handler = ivybridge_irq_handler;
3471                 dev->driver->irq_preinstall = ironlake_irq_preinstall;
3472                 dev->driver->irq_postinstall = ivybridge_irq_postinstall;
3473                 dev->driver->irq_uninstall = ironlake_irq_uninstall;
3474                 dev->driver->enable_vblank = ivybridge_enable_vblank;
3475                 dev->driver->disable_vblank = ivybridge_disable_vblank;
3476                 dev_priv->display.hpd_irq_setup = ibx_hpd_irq_setup;
3477         } else if (HAS_PCH_SPLIT(dev)) {
3478                 dev->driver->irq_handler = ironlake_irq_handler;
3479                 dev->driver->irq_preinstall = ironlake_irq_preinstall;
3480                 dev->driver->irq_postinstall = ironlake_irq_postinstall;
3481                 dev->driver->irq_uninstall = ironlake_irq_uninstall;
3482                 dev->driver->enable_vblank = ironlake_enable_vblank;
3483                 dev->driver->disable_vblank = ironlake_disable_vblank;
3484                 dev_priv->display.hpd_irq_setup = ibx_hpd_irq_setup;
3485         } else {
3486                 if (INTEL_INFO(dev)->gen == 2) {
3487                         dev->driver->irq_preinstall = i8xx_irq_preinstall;
3488                         dev->driver->irq_postinstall = i8xx_irq_postinstall;
3489                         dev->driver->irq_handler = i8xx_irq_handler;
3490                         dev->driver->irq_uninstall = i8xx_irq_uninstall;
3491                 } else if (INTEL_INFO(dev)->gen == 3) {
3492                         dev->driver->irq_preinstall = i915_irq_preinstall;
3493                         dev->driver->irq_postinstall = i915_irq_postinstall;
3494                         dev->driver->irq_uninstall = i915_irq_uninstall;
3495                         dev->driver->irq_handler = i915_irq_handler;
3496                         dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
3497                 } else {
3498                         dev->driver->irq_preinstall = i965_irq_preinstall;
3499                         dev->driver->irq_postinstall = i965_irq_postinstall;
3500                         dev->driver->irq_uninstall = i965_irq_uninstall;
3501                         dev->driver->irq_handler = i965_irq_handler;
3502                         dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
3503                 }
3504                 dev->driver->enable_vblank = i915_enable_vblank;
3505                 dev->driver->disable_vblank = i915_disable_vblank;
3506         }
3507 }
3508
3509 void intel_hpd_init(struct drm_device *dev)
3510 {
3511         struct drm_i915_private *dev_priv = dev->dev_private;
3512         struct drm_mode_config *mode_config = &dev->mode_config;
3513         struct drm_connector *connector;
3514         int i;
3515
3516         for (i = 1; i < HPD_NUM_PINS; i++) {
3517                 dev_priv->hpd_stats[i].hpd_cnt = 0;
3518                 dev_priv->hpd_stats[i].hpd_mark = HPD_ENABLED;
3519         }
3520         list_for_each_entry(connector, &mode_config->connector_list, head) {
3521                 struct intel_connector *intel_connector = to_intel_connector(connector);
3522                 connector->polled = intel_connector->polled;
3523                 if (!connector->polled && I915_HAS_HOTPLUG(dev) && intel_connector->encoder->hpd_pin > HPD_NONE)
3524                         connector->polled = DRM_CONNECTOR_POLL_HPD;
3525         }
3526         if (dev_priv->display.hpd_irq_setup)
3527                 dev_priv->display.hpd_irq_setup(dev);
3528 }