1 /* i915_irq.c -- IRQ support for the I915 -*- linux-c -*- 2 */ 3 /* 4 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas. 5 * All Rights Reserved. 6 * 7 * Permission is hereby granted, free of charge, to any person obtaining a 8 * copy of this software and associated documentation files (the 9 * "Software"), to deal in the Software without restriction, including 10 * without limitation the rights to use, copy, modify, merge, publish, 11 * distribute, sub license, and/or sell copies of the Software, and to 12 * permit persons to whom the Software is furnished to do so, subject to 13 * the following conditions: 14 * 15 * The above copyright notice and this permission notice (including the 16 * next paragraph) shall be included in all copies or substantial portions 17 * of the Software. 18 * 19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 20 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 21 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. 22 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR 23 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 24 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 25 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 26 * 27 */ 28 29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 30 31 #include <linux/sysrq.h> 32 #include <linux/slab.h> 33 #include <linux/circ_buf.h> 34 #include <drm/drmP.h> 35 #include <drm/i915_drm.h> 36 #include "i915_drv.h" 37 #include "i915_trace.h" 38 #include "intel_drv.h" 39 40 /** 41 * DOC: interrupt handling 42 * 43 * These functions provide the basic support for enabling and disabling the 44 * interrupt handling support. There's a lot more functionality in i915_irq.c 45 * and related files, but that will be described in separate chapters. 46 */ 47 48 static const u32 hpd_ibx[HPD_NUM_PINS] = { 49 [HPD_CRT] = SDE_CRT_HOTPLUG, 50 [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG, 51 [HPD_PORT_B] = SDE_PORTB_HOTPLUG, 52 [HPD_PORT_C] = SDE_PORTC_HOTPLUG, 53 [HPD_PORT_D] = SDE_PORTD_HOTPLUG 54 }; 55 56 static const u32 hpd_cpt[HPD_NUM_PINS] = { 57 [HPD_CRT] = SDE_CRT_HOTPLUG_CPT, 58 [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT, 59 [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT, 60 [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT, 61 [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT 62 }; 63 64 static const u32 hpd_spt[HPD_NUM_PINS] = { 65 [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT, 66 [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT, 67 [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT, 68 [HPD_PORT_E] = SDE_PORTE_HOTPLUG_SPT 69 }; 70 71 static const u32 hpd_mask_i915[HPD_NUM_PINS] = { 72 [HPD_CRT] = CRT_HOTPLUG_INT_EN, 73 [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN, 74 [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN, 75 [HPD_PORT_B] = PORTB_HOTPLUG_INT_EN, 76 [HPD_PORT_C] = PORTC_HOTPLUG_INT_EN, 77 [HPD_PORT_D] = PORTD_HOTPLUG_INT_EN 78 }; 79 80 static const u32 hpd_status_g4x[HPD_NUM_PINS] = { 81 [HPD_CRT] = CRT_HOTPLUG_INT_STATUS, 82 [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X, 83 [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X, 84 [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS, 85 [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS, 86 [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS 87 }; 88 89 static const u32 hpd_status_i915[HPD_NUM_PINS] = { 90 [HPD_CRT] = CRT_HOTPLUG_INT_STATUS, 91 [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915, 92 [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915, 93 [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS, 94 [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS, 95 [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS 96 }; 97 98 /* BXT hpd list */ 99 static const u32 hpd_bxt[HPD_NUM_PINS] = { 100 [HPD_PORT_B] = BXT_DE_PORT_HP_DDIB, 101 [HPD_PORT_C] = BXT_DE_PORT_HP_DDIC 102 }; 103 104 /* IIR can theoretically queue up two events. Be paranoid. */ 105 #define GEN8_IRQ_RESET_NDX(type, which) do { \ 106 I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \ 107 POSTING_READ(GEN8_##type##_IMR(which)); \ 108 I915_WRITE(GEN8_##type##_IER(which), 0); \ 109 I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \ 110 POSTING_READ(GEN8_##type##_IIR(which)); \ 111 I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \ 112 POSTING_READ(GEN8_##type##_IIR(which)); \ 113 } while (0) 114 115 #define GEN5_IRQ_RESET(type) do { \ 116 I915_WRITE(type##IMR, 0xffffffff); \ 117 POSTING_READ(type##IMR); \ 118 I915_WRITE(type##IER, 0); \ 119 I915_WRITE(type##IIR, 0xffffffff); \ 120 POSTING_READ(type##IIR); \ 121 I915_WRITE(type##IIR, 0xffffffff); \ 122 POSTING_READ(type##IIR); \ 123 } while (0) 124 125 /* 126 * We should clear IMR at preinstall/uninstall, and just check at postinstall. 127 */ 128 #define GEN5_ASSERT_IIR_IS_ZERO(reg) do { \ 129 u32 val = I915_READ(reg); \ 130 if (val) { \ 131 WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n", \ 132 (reg), val); \ 133 I915_WRITE((reg), 0xffffffff); \ 134 POSTING_READ(reg); \ 135 I915_WRITE((reg), 0xffffffff); \ 136 POSTING_READ(reg); \ 137 } \ 138 } while (0) 139 140 #define GEN8_IRQ_INIT_NDX(type, which, imr_val, ier_val) do { \ 141 GEN5_ASSERT_IIR_IS_ZERO(GEN8_##type##_IIR(which)); \ 142 I915_WRITE(GEN8_##type##_IER(which), (ier_val)); \ 143 I915_WRITE(GEN8_##type##_IMR(which), (imr_val)); \ 144 POSTING_READ(GEN8_##type##_IMR(which)); \ 145 } while (0) 146 147 #define GEN5_IRQ_INIT(type, imr_val, ier_val) do { \ 148 GEN5_ASSERT_IIR_IS_ZERO(type##IIR); \ 149 I915_WRITE(type##IER, (ier_val)); \ 150 I915_WRITE(type##IMR, (imr_val)); \ 151 POSTING_READ(type##IMR); \ 152 } while (0) 153 154 static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir); 155 156 /* For display hotplug interrupt */ 157 void 158 ironlake_enable_display_irq(struct drm_i915_private *dev_priv, u32 mask) 159 { 160 assert_spin_locked(&dev_priv->irq_lock); 161 162 if (WARN_ON(!intel_irqs_enabled(dev_priv))) 163 return; 164 165 if ((dev_priv->irq_mask & mask) != 0) { 166 dev_priv->irq_mask &= ~mask; 167 I915_WRITE(DEIMR, dev_priv->irq_mask); 168 POSTING_READ(DEIMR); 169 } 170 } 171 172 void 173 ironlake_disable_display_irq(struct drm_i915_private *dev_priv, u32 mask) 174 { 175 assert_spin_locked(&dev_priv->irq_lock); 176 177 if (WARN_ON(!intel_irqs_enabled(dev_priv))) 178 return; 179 180 if ((dev_priv->irq_mask & mask) != mask) { 181 dev_priv->irq_mask |= mask; 182 I915_WRITE(DEIMR, dev_priv->irq_mask); 183 POSTING_READ(DEIMR); 184 } 185 } 186 187 /** 188 * ilk_update_gt_irq - update GTIMR 189 * @dev_priv: driver private 190 * @interrupt_mask: mask of interrupt bits to update 191 * @enabled_irq_mask: mask of interrupt bits to enable 192 */ 193 static void ilk_update_gt_irq(struct drm_i915_private *dev_priv, 194 uint32_t interrupt_mask, 195 uint32_t enabled_irq_mask) 196 { 197 assert_spin_locked(&dev_priv->irq_lock); 198 199 WARN_ON(enabled_irq_mask & ~interrupt_mask); 200 201 if (WARN_ON(!intel_irqs_enabled(dev_priv))) 202 return; 203 204 dev_priv->gt_irq_mask &= ~interrupt_mask; 205 dev_priv->gt_irq_mask |= (~enabled_irq_mask & interrupt_mask); 206 I915_WRITE(GTIMR, dev_priv->gt_irq_mask); 207 POSTING_READ(GTIMR); 208 } 209 210 void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask) 211 { 212 ilk_update_gt_irq(dev_priv, mask, mask); 213 } 214 215 void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask) 216 { 217 ilk_update_gt_irq(dev_priv, mask, 0); 218 } 219 220 static u32 gen6_pm_iir(struct drm_i915_private *dev_priv) 221 { 222 return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR; 223 } 224 225 static u32 gen6_pm_imr(struct drm_i915_private *dev_priv) 226 { 227 return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IMR(2) : GEN6_PMIMR; 228 } 229 230 static u32 gen6_pm_ier(struct drm_i915_private *dev_priv) 231 { 232 return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IER(2) : GEN6_PMIER; 233 } 234 235 /** 236 * snb_update_pm_irq - update GEN6_PMIMR 237 * @dev_priv: driver private 238 * @interrupt_mask: mask of interrupt bits to update 239 * @enabled_irq_mask: mask of interrupt bits to enable 240 */ 241 static void snb_update_pm_irq(struct drm_i915_private *dev_priv, 242 uint32_t interrupt_mask, 243 uint32_t enabled_irq_mask) 244 { 245 uint32_t new_val; 246 247 WARN_ON(enabled_irq_mask & ~interrupt_mask); 248 249 assert_spin_locked(&dev_priv->irq_lock); 250 251 new_val = dev_priv->pm_irq_mask; 252 new_val &= ~interrupt_mask; 253 new_val |= (~enabled_irq_mask & interrupt_mask); 254 255 if (new_val != dev_priv->pm_irq_mask) { 256 dev_priv->pm_irq_mask = new_val; 257 I915_WRITE(gen6_pm_imr(dev_priv), dev_priv->pm_irq_mask); 258 POSTING_READ(gen6_pm_imr(dev_priv)); 259 } 260 } 261 262 void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask) 263 { 264 if (WARN_ON(!intel_irqs_enabled(dev_priv))) 265 return; 266 267 snb_update_pm_irq(dev_priv, mask, mask); 268 } 269 270 static void __gen6_disable_pm_irq(struct drm_i915_private *dev_priv, 271 uint32_t mask) 272 { 273 snb_update_pm_irq(dev_priv, mask, 0); 274 } 275 276 void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask) 277 { 278 if (WARN_ON(!intel_irqs_enabled(dev_priv))) 279 return; 280 281 __gen6_disable_pm_irq(dev_priv, mask); 282 } 283 284 void gen6_reset_rps_interrupts(struct drm_device *dev) 285 { 286 struct drm_i915_private *dev_priv = dev->dev_private; 287 uint32_t reg = gen6_pm_iir(dev_priv); 288 289 spin_lock_irq(&dev_priv->irq_lock); 290 I915_WRITE(reg, dev_priv->pm_rps_events); 291 I915_WRITE(reg, dev_priv->pm_rps_events); 292 POSTING_READ(reg); 293 dev_priv->rps.pm_iir = 0; 294 spin_unlock_irq(&dev_priv->irq_lock); 295 } 296 297 void gen6_enable_rps_interrupts(struct drm_device *dev) 298 { 299 struct drm_i915_private *dev_priv = dev->dev_private; 300 301 spin_lock_irq(&dev_priv->irq_lock); 302 303 WARN_ON(dev_priv->rps.pm_iir); 304 WARN_ON(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events); 305 dev_priv->rps.interrupts_enabled = true; 306 I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) | 307 dev_priv->pm_rps_events); 308 gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events); 309 310 spin_unlock_irq(&dev_priv->irq_lock); 311 } 312 313 u32 gen6_sanitize_rps_pm_mask(struct drm_i915_private *dev_priv, u32 mask) 314 { 315 /* 316 * SNB,IVB can while VLV,CHV may hard hang on looping batchbuffer 317 * if GEN6_PM_UP_EI_EXPIRED is masked. 318 * 319 * TODO: verify if this can be reproduced on VLV,CHV. 320 */ 321 if (INTEL_INFO(dev_priv)->gen <= 7 && !IS_HASWELL(dev_priv)) 322 mask &= ~GEN6_PM_RP_UP_EI_EXPIRED; 323 324 if (INTEL_INFO(dev_priv)->gen >= 8) 325 mask &= ~GEN8_PMINTR_REDIRECT_TO_NON_DISP; 326 327 return mask; 328 } 329 330 void gen6_disable_rps_interrupts(struct drm_device *dev) 331 { 332 struct drm_i915_private *dev_priv = dev->dev_private; 333 334 spin_lock_irq(&dev_priv->irq_lock); 335 dev_priv->rps.interrupts_enabled = false; 336 spin_unlock_irq(&dev_priv->irq_lock); 337 338 cancel_work_sync(&dev_priv->rps.work); 339 340 spin_lock_irq(&dev_priv->irq_lock); 341 342 I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0)); 343 344 __gen6_disable_pm_irq(dev_priv, dev_priv->pm_rps_events); 345 I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) & 346 ~dev_priv->pm_rps_events); 347 348 spin_unlock_irq(&dev_priv->irq_lock); 349 350 synchronize_irq(dev->irq); 351 } 352 353 /** 354 * ibx_display_interrupt_update - update SDEIMR 355 * @dev_priv: driver private 356 * @interrupt_mask: mask of interrupt bits to update 357 * @enabled_irq_mask: mask of interrupt bits to enable 358 */ 359 void ibx_display_interrupt_update(struct drm_i915_private *dev_priv, 360 uint32_t interrupt_mask, 361 uint32_t enabled_irq_mask) 362 { 363 uint32_t sdeimr = I915_READ(SDEIMR); 364 sdeimr &= ~interrupt_mask; 365 sdeimr |= (~enabled_irq_mask & interrupt_mask); 366 367 WARN_ON(enabled_irq_mask & ~interrupt_mask); 368 369 assert_spin_locked(&dev_priv->irq_lock); 370 371 if (WARN_ON(!intel_irqs_enabled(dev_priv))) 372 return; 373 374 I915_WRITE(SDEIMR, sdeimr); 375 POSTING_READ(SDEIMR); 376 } 377 378 static void 379 __i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe, 380 u32 enable_mask, u32 status_mask) 381 { 382 u32 reg = PIPESTAT(pipe); 383 u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK; 384 385 assert_spin_locked(&dev_priv->irq_lock); 386 WARN_ON(!intel_irqs_enabled(dev_priv)); 387 388 if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK || 389 status_mask & ~PIPESTAT_INT_STATUS_MASK, 390 "pipe %c: enable_mask=0x%x, status_mask=0x%x\n", 391 pipe_name(pipe), enable_mask, status_mask)) 392 return; 393 394 if ((pipestat & enable_mask) == enable_mask) 395 return; 396 397 dev_priv->pipestat_irq_mask[pipe] |= status_mask; 398 399 /* Enable the interrupt, clear any pending status */ 400 pipestat |= enable_mask | status_mask; 401 I915_WRITE(reg, pipestat); 402 POSTING_READ(reg); 403 } 404 405 static void 406 __i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe, 407 u32 enable_mask, u32 status_mask) 408 { 409 u32 reg = PIPESTAT(pipe); 410 u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK; 411 412 assert_spin_locked(&dev_priv->irq_lock); 413 WARN_ON(!intel_irqs_enabled(dev_priv)); 414 415 if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK || 416 status_mask & ~PIPESTAT_INT_STATUS_MASK, 417 "pipe %c: enable_mask=0x%x, status_mask=0x%x\n", 418 pipe_name(pipe), enable_mask, status_mask)) 419 return; 420 421 if ((pipestat & enable_mask) == 0) 422 return; 423 424 dev_priv->pipestat_irq_mask[pipe] &= ~status_mask; 425 426 pipestat &= ~enable_mask; 427 I915_WRITE(reg, pipestat); 428 POSTING_READ(reg); 429 } 430 431 static u32 vlv_get_pipestat_enable_mask(struct drm_device *dev, u32 status_mask) 432 { 433 u32 enable_mask = status_mask << 16; 434 435 /* 436 * On pipe A we don't support the PSR interrupt yet, 437 * on pipe B and C the same bit MBZ. 438 */ 439 if (WARN_ON_ONCE(status_mask & PIPE_A_PSR_STATUS_VLV)) 440 return 0; 441 /* 442 * On pipe B and C we don't support the PSR interrupt yet, on pipe 443 * A the same bit is for perf counters which we don't use either. 444 */ 445 if (WARN_ON_ONCE(status_mask & PIPE_B_PSR_STATUS_VLV)) 446 return 0; 447 448 enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS | 449 SPRITE0_FLIP_DONE_INT_EN_VLV | 450 SPRITE1_FLIP_DONE_INT_EN_VLV); 451 if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV) 452 enable_mask |= SPRITE0_FLIP_DONE_INT_EN_VLV; 453 if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV) 454 enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV; 455 456 return enable_mask; 457 } 458 459 void 460 i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe, 461 u32 status_mask) 462 { 463 u32 enable_mask; 464 465 if (IS_VALLEYVIEW(dev_priv->dev)) 466 enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev, 467 status_mask); 468 else 469 enable_mask = status_mask << 16; 470 __i915_enable_pipestat(dev_priv, pipe, enable_mask, status_mask); 471 } 472 473 void 474 i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe, 475 u32 status_mask) 476 { 477 u32 enable_mask; 478 479 if (IS_VALLEYVIEW(dev_priv->dev)) 480 enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev, 481 status_mask); 482 else 483 enable_mask = status_mask << 16; 484 __i915_disable_pipestat(dev_priv, pipe, enable_mask, status_mask); 485 } 486 487 /** 488 * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion 489 */ 490 static void i915_enable_asle_pipestat(struct drm_device *dev) 491 { 492 struct drm_i915_private *dev_priv = dev->dev_private; 493 494 if (!dev_priv->opregion.asle || !IS_MOBILE(dev)) 495 return; 496 497 spin_lock_irq(&dev_priv->irq_lock); 498 499 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS); 500 if (INTEL_INFO(dev)->gen >= 4) 501 i915_enable_pipestat(dev_priv, PIPE_A, 502 PIPE_LEGACY_BLC_EVENT_STATUS); 503 504 spin_unlock_irq(&dev_priv->irq_lock); 505 } 506 507 /* 508 * This timing diagram depicts the video signal in and 509 * around the vertical blanking period. 510 * 511 * Assumptions about the fictitious mode used in this example: 512 * vblank_start >= 3 513 * vsync_start = vblank_start + 1 514 * vsync_end = vblank_start + 2 515 * vtotal = vblank_start + 3 516 * 517 * start of vblank: 518 * latch double buffered registers 519 * increment frame counter (ctg+) 520 * generate start of vblank interrupt (gen4+) 521 * | 522 * | frame start: 523 * | generate frame start interrupt (aka. vblank interrupt) (gmch) 524 * | may be shifted forward 1-3 extra lines via PIPECONF 525 * | | 526 * | | start of vsync: 527 * | | generate vsync interrupt 528 * | | | 529 * ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx 530 * . \hs/ . \hs/ \hs/ \hs/ . \hs/ 531 * ----va---> <-----------------vb--------------------> <--------va------------- 532 * | | <----vs-----> | 533 * -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2) 534 * -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+) 535 * -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi) 536 * | | | 537 * last visible pixel first visible pixel 538 * | increment frame counter (gen3/4) 539 * pixel counter = vblank_start * htotal pixel counter = 0 (gen3/4) 540 * 541 * x = horizontal active 542 * _ = horizontal blanking 543 * hs = horizontal sync 544 * va = vertical active 545 * vb = vertical blanking 546 * vs = vertical sync 547 * vbs = vblank_start (number) 548 * 549 * Summary: 550 * - most events happen at the start of horizontal sync 551 * - frame start happens at the start of horizontal blank, 1-4 lines 552 * (depending on PIPECONF settings) after the start of vblank 553 * - gen3/4 pixel and frame counter are synchronized with the start 554 * of horizontal active on the first line of vertical active 555 */ 556 557 static u32 i8xx_get_vblank_counter(struct drm_device *dev, int pipe) 558 { 559 /* Gen2 doesn't have a hardware frame counter */ 560 return 0; 561 } 562 563 /* Called from drm generic code, passed a 'crtc', which 564 * we use as a pipe index 565 */ 566 static u32 i915_get_vblank_counter(struct drm_device *dev, int pipe) 567 { 568 struct drm_i915_private *dev_priv = dev->dev_private; 569 unsigned long high_frame; 570 unsigned long low_frame; 571 u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal; 572 struct intel_crtc *intel_crtc = 573 to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]); 574 const struct drm_display_mode *mode = &intel_crtc->base.hwmode; 575 576 htotal = mode->crtc_htotal; 577 hsync_start = mode->crtc_hsync_start; 578 vbl_start = mode->crtc_vblank_start; 579 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 580 vbl_start = DIV_ROUND_UP(vbl_start, 2); 581 582 /* Convert to pixel count */ 583 vbl_start *= htotal; 584 585 /* Start of vblank event occurs at start of hsync */ 586 vbl_start -= htotal - hsync_start; 587 588 high_frame = PIPEFRAME(pipe); 589 low_frame = PIPEFRAMEPIXEL(pipe); 590 591 /* 592 * High & low register fields aren't synchronized, so make sure 593 * we get a low value that's stable across two reads of the high 594 * register. 595 */ 596 do { 597 high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK; 598 low = I915_READ(low_frame); 599 high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK; 600 } while (high1 != high2); 601 602 high1 >>= PIPE_FRAME_HIGH_SHIFT; 603 pixel = low & PIPE_PIXEL_MASK; 604 low >>= PIPE_FRAME_LOW_SHIFT; 605 606 /* 607 * The frame counter increments at beginning of active. 608 * Cook up a vblank counter by also checking the pixel 609 * counter against vblank start. 610 */ 611 return (((high1 << 8) | low) + (pixel >= vbl_start)) & 0xffffff; 612 } 613 614 static u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe) 615 { 616 struct drm_i915_private *dev_priv = dev->dev_private; 617 int reg = PIPE_FRMCOUNT_GM45(pipe); 618 619 return I915_READ(reg); 620 } 621 622 /* raw reads, only for fast reads of display block, no need for forcewake etc. */ 623 #define __raw_i915_read32(dev_priv__, reg__) readl((dev_priv__)->regs + (reg__)) 624 625 static int __intel_get_crtc_scanline(struct intel_crtc *crtc) 626 { 627 struct drm_device *dev = crtc->base.dev; 628 struct drm_i915_private *dev_priv = dev->dev_private; 629 const struct drm_display_mode *mode = &crtc->base.hwmode; 630 enum pipe pipe = crtc->pipe; 631 int position, vtotal; 632 633 vtotal = mode->crtc_vtotal; 634 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 635 vtotal /= 2; 636 637 if (IS_GEN2(dev)) 638 position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN2; 639 else 640 position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN3; 641 642 /* 643 * On HSW, the DSL reg (0x70000) appears to return 0 if we 644 * read it just before the start of vblank. So try it again 645 * so we don't accidentally end up spanning a vblank frame 646 * increment, causing the pipe_update_end() code to squak at us. 647 * 648 * The nature of this problem means we can't simply check the ISR 649 * bit and return the vblank start value; nor can we use the scanline 650 * debug register in the transcoder as it appears to have the same 651 * problem. We may need to extend this to include other platforms, 652 * but so far testing only shows the problem on HSW. 653 */ 654 if (IS_HASWELL(dev) && !position) { 655 int i, temp; 656 657 for (i = 0; i < 100; i++) { 658 udelay(1); 659 temp = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & 660 DSL_LINEMASK_GEN3; 661 if (temp != position) { 662 position = temp; 663 break; 664 } 665 } 666 } 667 668 /* 669 * See update_scanline_offset() for the details on the 670 * scanline_offset adjustment. 671 */ 672 return (position + crtc->scanline_offset) % vtotal; 673 } 674 675 static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe, 676 unsigned int flags, int *vpos, int *hpos, 677 ktime_t *stime, ktime_t *etime) 678 { 679 struct drm_i915_private *dev_priv = dev->dev_private; 680 struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; 681 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 682 const struct drm_display_mode *mode = &intel_crtc->base.hwmode; 683 int position; 684 int vbl_start, vbl_end, hsync_start, htotal, vtotal; 685 bool in_vbl = true; 686 int ret = 0; 687 unsigned long irqflags; 688 689 if (WARN_ON(!mode->crtc_clock)) { 690 DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled " 691 "pipe %c\n", pipe_name(pipe)); 692 return 0; 693 } 694 695 htotal = mode->crtc_htotal; 696 hsync_start = mode->crtc_hsync_start; 697 vtotal = mode->crtc_vtotal; 698 vbl_start = mode->crtc_vblank_start; 699 vbl_end = mode->crtc_vblank_end; 700 701 if (mode->flags & DRM_MODE_FLAG_INTERLACE) { 702 vbl_start = DIV_ROUND_UP(vbl_start, 2); 703 vbl_end /= 2; 704 vtotal /= 2; 705 } 706 707 ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE; 708 709 /* 710 * Lock uncore.lock, as we will do multiple timing critical raw 711 * register reads, potentially with preemption disabled, so the 712 * following code must not block on uncore.lock. 713 */ 714 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); 715 716 /* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */ 717 718 /* Get optional system timestamp before query. */ 719 if (stime) 720 *stime = ktime_get(); 721 722 if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) { 723 /* No obvious pixelcount register. Only query vertical 724 * scanout position from Display scan line register. 725 */ 726 position = __intel_get_crtc_scanline(intel_crtc); 727 } else { 728 /* Have access to pixelcount since start of frame. 729 * We can split this into vertical and horizontal 730 * scanout position. 731 */ 732 position = (__raw_i915_read32(dev_priv, PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT; 733 734 /* convert to pixel counts */ 735 vbl_start *= htotal; 736 vbl_end *= htotal; 737 vtotal *= htotal; 738 739 /* 740 * In interlaced modes, the pixel counter counts all pixels, 741 * so one field will have htotal more pixels. In order to avoid 742 * the reported position from jumping backwards when the pixel 743 * counter is beyond the length of the shorter field, just 744 * clamp the position the length of the shorter field. This 745 * matches how the scanline counter based position works since 746 * the scanline counter doesn't count the two half lines. 747 */ 748 if (position >= vtotal) 749 position = vtotal - 1; 750 751 /* 752 * Start of vblank interrupt is triggered at start of hsync, 753 * just prior to the first active line of vblank. However we 754 * consider lines to start at the leading edge of horizontal 755 * active. So, should we get here before we've crossed into 756 * the horizontal active of the first line in vblank, we would 757 * not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that, 758 * always add htotal-hsync_start to the current pixel position. 759 */ 760 position = (position + htotal - hsync_start) % vtotal; 761 } 762 763 /* Get optional system timestamp after query. */ 764 if (etime) 765 *etime = ktime_get(); 766 767 /* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */ 768 769 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); 770 771 in_vbl = position >= vbl_start && position < vbl_end; 772 773 /* 774 * While in vblank, position will be negative 775 * counting up towards 0 at vbl_end. And outside 776 * vblank, position will be positive counting 777 * up since vbl_end. 778 */ 779 if (position >= vbl_start) 780 position -= vbl_end; 781 else 782 position += vtotal - vbl_end; 783 784 if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) { 785 *vpos = position; 786 *hpos = 0; 787 } else { 788 *vpos = position / htotal; 789 *hpos = position - (*vpos * htotal); 790 } 791 792 /* In vblank? */ 793 if (in_vbl) 794 ret |= DRM_SCANOUTPOS_IN_VBLANK; 795 796 return ret; 797 } 798 799 int intel_get_crtc_scanline(struct intel_crtc *crtc) 800 { 801 struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; 802 unsigned long irqflags; 803 int position; 804 805 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); 806 position = __intel_get_crtc_scanline(crtc); 807 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); 808 809 return position; 810 } 811 812 static int i915_get_vblank_timestamp(struct drm_device *dev, int pipe, 813 int *max_error, 814 struct timeval *vblank_time, 815 unsigned flags) 816 { 817 struct drm_crtc *crtc; 818 819 if (pipe < 0 || pipe >= INTEL_INFO(dev)->num_pipes) { 820 DRM_ERROR("Invalid crtc %d\n", pipe); 821 return -EINVAL; 822 } 823 824 /* Get drm_crtc to timestamp: */ 825 crtc = intel_get_crtc_for_pipe(dev, pipe); 826 if (crtc == NULL) { 827 DRM_ERROR("Invalid crtc %d\n", pipe); 828 return -EINVAL; 829 } 830 831 if (!crtc->hwmode.crtc_clock) { 832 DRM_DEBUG_KMS("crtc %d is disabled\n", pipe); 833 return -EBUSY; 834 } 835 836 /* Helper routine in DRM core does all the work: */ 837 return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error, 838 vblank_time, flags, 839 crtc, 840 &crtc->hwmode); 841 } 842 843 static void ironlake_rps_change_irq_handler(struct drm_device *dev) 844 { 845 struct drm_i915_private *dev_priv = dev->dev_private; 846 u32 busy_up, busy_down, max_avg, min_avg; 847 u8 new_delay; 848 849 spin_lock(&mchdev_lock); 850 851 I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS)); 852 853 new_delay = dev_priv->ips.cur_delay; 854 855 I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG); 856 busy_up = I915_READ(RCPREVBSYTUPAVG); 857 busy_down = I915_READ(RCPREVBSYTDNAVG); 858 max_avg = I915_READ(RCBMAXAVG); 859 min_avg = I915_READ(RCBMINAVG); 860 861 /* Handle RCS change request from hw */ 862 if (busy_up > max_avg) { 863 if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay) 864 new_delay = dev_priv->ips.cur_delay - 1; 865 if (new_delay < dev_priv->ips.max_delay) 866 new_delay = dev_priv->ips.max_delay; 867 } else if (busy_down < min_avg) { 868 if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay) 869 new_delay = dev_priv->ips.cur_delay + 1; 870 if (new_delay > dev_priv->ips.min_delay) 871 new_delay = dev_priv->ips.min_delay; 872 } 873 874 if (ironlake_set_drps(dev, new_delay)) 875 dev_priv->ips.cur_delay = new_delay; 876 877 spin_unlock(&mchdev_lock); 878 879 return; 880 } 881 882 static void notify_ring(struct intel_engine_cs *ring) 883 { 884 if (!intel_ring_initialized(ring)) 885 return; 886 887 trace_i915_gem_request_notify(ring); 888 889 wake_up_all(&ring->irq_queue); 890 } 891 892 static void vlv_c0_read(struct drm_i915_private *dev_priv, 893 struct intel_rps_ei *ei) 894 { 895 ei->cz_clock = vlv_punit_read(dev_priv, PUNIT_REG_CZ_TIMESTAMP); 896 ei->render_c0 = I915_READ(VLV_RENDER_C0_COUNT); 897 ei->media_c0 = I915_READ(VLV_MEDIA_C0_COUNT); 898 } 899 900 static bool vlv_c0_above(struct drm_i915_private *dev_priv, 901 const struct intel_rps_ei *old, 902 const struct intel_rps_ei *now, 903 int threshold) 904 { 905 u64 time, c0; 906 907 if (old->cz_clock == 0) 908 return false; 909 910 time = now->cz_clock - old->cz_clock; 911 time *= threshold * dev_priv->mem_freq; 912 913 /* Workload can be split between render + media, e.g. SwapBuffers 914 * being blitted in X after being rendered in mesa. To account for 915 * this we need to combine both engines into our activity counter. 916 */ 917 c0 = now->render_c0 - old->render_c0; 918 c0 += now->media_c0 - old->media_c0; 919 c0 *= 100 * VLV_CZ_CLOCK_TO_MILLI_SEC * 4 / 1000; 920 921 return c0 >= time; 922 } 923 924 void gen6_rps_reset_ei(struct drm_i915_private *dev_priv) 925 { 926 vlv_c0_read(dev_priv, &dev_priv->rps.down_ei); 927 dev_priv->rps.up_ei = dev_priv->rps.down_ei; 928 } 929 930 static u32 vlv_wa_c0_ei(struct drm_i915_private *dev_priv, u32 pm_iir) 931 { 932 struct intel_rps_ei now; 933 u32 events = 0; 934 935 if ((pm_iir & (GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED)) == 0) 936 return 0; 937 938 vlv_c0_read(dev_priv, &now); 939 if (now.cz_clock == 0) 940 return 0; 941 942 if (pm_iir & GEN6_PM_RP_DOWN_EI_EXPIRED) { 943 if (!vlv_c0_above(dev_priv, 944 &dev_priv->rps.down_ei, &now, 945 dev_priv->rps.down_threshold)) 946 events |= GEN6_PM_RP_DOWN_THRESHOLD; 947 dev_priv->rps.down_ei = now; 948 } 949 950 if (pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) { 951 if (vlv_c0_above(dev_priv, 952 &dev_priv->rps.up_ei, &now, 953 dev_priv->rps.up_threshold)) 954 events |= GEN6_PM_RP_UP_THRESHOLD; 955 dev_priv->rps.up_ei = now; 956 } 957 958 return events; 959 } 960 961 static bool any_waiters(struct drm_i915_private *dev_priv) 962 { 963 struct intel_engine_cs *ring; 964 int i; 965 966 for_each_ring(ring, dev_priv, i) 967 if (ring->irq_refcount) 968 return true; 969 970 return false; 971 } 972 973 static void gen6_pm_rps_work(struct work_struct *work) 974 { 975 struct drm_i915_private *dev_priv = 976 container_of(work, struct drm_i915_private, rps.work); 977 bool client_boost; 978 int new_delay, adj, min, max; 979 u32 pm_iir; 980 981 spin_lock_irq(&dev_priv->irq_lock); 982 /* Speed up work cancelation during disabling rps interrupts. */ 983 if (!dev_priv->rps.interrupts_enabled) { 984 spin_unlock_irq(&dev_priv->irq_lock); 985 return; 986 } 987 pm_iir = dev_priv->rps.pm_iir; 988 dev_priv->rps.pm_iir = 0; 989 /* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */ 990 gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events); 991 client_boost = dev_priv->rps.client_boost; 992 dev_priv->rps.client_boost = false; 993 spin_unlock_irq(&dev_priv->irq_lock); 994 995 /* Make sure we didn't queue anything we're not going to process. */ 996 WARN_ON(pm_iir & ~dev_priv->pm_rps_events); 997 998 if ((pm_iir & dev_priv->pm_rps_events) == 0 && !client_boost) 999 return; 1000 1001 mutex_lock(&dev_priv->rps.hw_lock); 1002 1003 pm_iir |= vlv_wa_c0_ei(dev_priv, pm_iir); 1004 1005 adj = dev_priv->rps.last_adj; 1006 new_delay = dev_priv->rps.cur_freq; 1007 min = dev_priv->rps.min_freq_softlimit; 1008 max = dev_priv->rps.max_freq_softlimit; 1009 1010 if (client_boost) { 1011 new_delay = dev_priv->rps.max_freq_softlimit; 1012 adj = 0; 1013 } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) { 1014 if (adj > 0) 1015 adj *= 2; 1016 else /* CHV needs even encode values */ 1017 adj = IS_CHERRYVIEW(dev_priv) ? 2 : 1; 1018 /* 1019 * For better performance, jump directly 1020 * to RPe if we're below it. 1021 */ 1022 if (new_delay < dev_priv->rps.efficient_freq - adj) { 1023 new_delay = dev_priv->rps.efficient_freq; 1024 adj = 0; 1025 } 1026 } else if (any_waiters(dev_priv)) { 1027 adj = 0; 1028 } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) { 1029 if (dev_priv->rps.cur_freq > dev_priv->rps.efficient_freq) 1030 new_delay = dev_priv->rps.efficient_freq; 1031 else 1032 new_delay = dev_priv->rps.min_freq_softlimit; 1033 adj = 0; 1034 } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) { 1035 if (adj < 0) 1036 adj *= 2; 1037 else /* CHV needs even encode values */ 1038 adj = IS_CHERRYVIEW(dev_priv) ? -2 : -1; 1039 } else { /* unknown event */ 1040 adj = 0; 1041 } 1042 1043 dev_priv->rps.last_adj = adj; 1044 1045 /* sysfs frequency interfaces may have snuck in while servicing the 1046 * interrupt 1047 */ 1048 new_delay += adj; 1049 new_delay = clamp_t(int, new_delay, min, max); 1050 1051 intel_set_rps(dev_priv->dev, new_delay); 1052 1053 mutex_unlock(&dev_priv->rps.hw_lock); 1054 } 1055 1056 1057 /** 1058 * ivybridge_parity_work - Workqueue called when a parity error interrupt 1059 * occurred. 1060 * @work: workqueue struct 1061 * 1062 * Doesn't actually do anything except notify userspace. As a consequence of 1063 * this event, userspace should try to remap the bad rows since statistically 1064 * it is likely the same row is more likely to go bad again. 1065 */ 1066 static void ivybridge_parity_work(struct work_struct *work) 1067 { 1068 struct drm_i915_private *dev_priv = 1069 container_of(work, struct drm_i915_private, l3_parity.error_work); 1070 u32 error_status, row, bank, subbank; 1071 char *parity_event[6]; 1072 uint32_t misccpctl; 1073 uint8_t slice = 0; 1074 1075 /* We must turn off DOP level clock gating to access the L3 registers. 1076 * In order to prevent a get/put style interface, acquire struct mutex 1077 * any time we access those registers. 1078 */ 1079 mutex_lock(&dev_priv->dev->struct_mutex); 1080 1081 /* If we've screwed up tracking, just let the interrupt fire again */ 1082 if (WARN_ON(!dev_priv->l3_parity.which_slice)) 1083 goto out; 1084 1085 misccpctl = I915_READ(GEN7_MISCCPCTL); 1086 I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE); 1087 POSTING_READ(GEN7_MISCCPCTL); 1088 1089 while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) { 1090 u32 reg; 1091 1092 slice--; 1093 if (WARN_ON_ONCE(slice >= NUM_L3_SLICES(dev_priv->dev))) 1094 break; 1095 1096 dev_priv->l3_parity.which_slice &= ~(1<<slice); 1097 1098 reg = GEN7_L3CDERRST1 + (slice * 0x200); 1099 1100 error_status = I915_READ(reg); 1101 row = GEN7_PARITY_ERROR_ROW(error_status); 1102 bank = GEN7_PARITY_ERROR_BANK(error_status); 1103 subbank = GEN7_PARITY_ERROR_SUBBANK(error_status); 1104 1105 I915_WRITE(reg, GEN7_PARITY_ERROR_VALID | GEN7_L3CDERRST1_ENABLE); 1106 POSTING_READ(reg); 1107 1108 parity_event[0] = I915_L3_PARITY_UEVENT "=1"; 1109 parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row); 1110 parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank); 1111 parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank); 1112 parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice); 1113 parity_event[5] = NULL; 1114 1115 kobject_uevent_env(&dev_priv->dev->primary->kdev->kobj, 1116 KOBJ_CHANGE, parity_event); 1117 1118 DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n", 1119 slice, row, bank, subbank); 1120 1121 kfree(parity_event[4]); 1122 kfree(parity_event[3]); 1123 kfree(parity_event[2]); 1124 kfree(parity_event[1]); 1125 } 1126 1127 I915_WRITE(GEN7_MISCCPCTL, misccpctl); 1128 1129 out: 1130 WARN_ON(dev_priv->l3_parity.which_slice); 1131 spin_lock_irq(&dev_priv->irq_lock); 1132 gen5_enable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv->dev)); 1133 spin_unlock_irq(&dev_priv->irq_lock); 1134 1135 mutex_unlock(&dev_priv->dev->struct_mutex); 1136 } 1137 1138 static void ivybridge_parity_error_irq_handler(struct drm_device *dev, u32 iir) 1139 { 1140 struct drm_i915_private *dev_priv = dev->dev_private; 1141 1142 if (!HAS_L3_DPF(dev)) 1143 return; 1144 1145 spin_lock(&dev_priv->irq_lock); 1146 gen5_disable_gt_irq(dev_priv, GT_PARITY_ERROR(dev)); 1147 spin_unlock(&dev_priv->irq_lock); 1148 1149 iir &= GT_PARITY_ERROR(dev); 1150 if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1) 1151 dev_priv->l3_parity.which_slice |= 1 << 1; 1152 1153 if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT) 1154 dev_priv->l3_parity.which_slice |= 1 << 0; 1155 1156 queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work); 1157 } 1158 1159 static void ilk_gt_irq_handler(struct drm_device *dev, 1160 struct drm_i915_private *dev_priv, 1161 u32 gt_iir) 1162 { 1163 if (gt_iir & 1164 (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT)) 1165 notify_ring(&dev_priv->ring[RCS]); 1166 if (gt_iir & ILK_BSD_USER_INTERRUPT) 1167 notify_ring(&dev_priv->ring[VCS]); 1168 } 1169 1170 static void snb_gt_irq_handler(struct drm_device *dev, 1171 struct drm_i915_private *dev_priv, 1172 u32 gt_iir) 1173 { 1174 1175 if (gt_iir & 1176 (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT)) 1177 notify_ring(&dev_priv->ring[RCS]); 1178 if (gt_iir & GT_BSD_USER_INTERRUPT) 1179 notify_ring(&dev_priv->ring[VCS]); 1180 if (gt_iir & GT_BLT_USER_INTERRUPT) 1181 notify_ring(&dev_priv->ring[BCS]); 1182 1183 if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT | 1184 GT_BSD_CS_ERROR_INTERRUPT | 1185 GT_RENDER_CS_MASTER_ERROR_INTERRUPT)) 1186 DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir); 1187 1188 if (gt_iir & GT_PARITY_ERROR(dev)) 1189 ivybridge_parity_error_irq_handler(dev, gt_iir); 1190 } 1191 1192 static irqreturn_t gen8_gt_irq_handler(struct drm_i915_private *dev_priv, 1193 u32 master_ctl) 1194 { 1195 irqreturn_t ret = IRQ_NONE; 1196 1197 if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) { 1198 u32 tmp = I915_READ_FW(GEN8_GT_IIR(0)); 1199 if (tmp) { 1200 I915_WRITE_FW(GEN8_GT_IIR(0), tmp); 1201 ret = IRQ_HANDLED; 1202 1203 if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT)) 1204 intel_lrc_irq_handler(&dev_priv->ring[RCS]); 1205 if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT)) 1206 notify_ring(&dev_priv->ring[RCS]); 1207 1208 if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT)) 1209 intel_lrc_irq_handler(&dev_priv->ring[BCS]); 1210 if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT)) 1211 notify_ring(&dev_priv->ring[BCS]); 1212 } else 1213 DRM_ERROR("The master control interrupt lied (GT0)!\n"); 1214 } 1215 1216 if (master_ctl & (GEN8_GT_VCS1_IRQ | GEN8_GT_VCS2_IRQ)) { 1217 u32 tmp = I915_READ_FW(GEN8_GT_IIR(1)); 1218 if (tmp) { 1219 I915_WRITE_FW(GEN8_GT_IIR(1), tmp); 1220 ret = IRQ_HANDLED; 1221 1222 if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT)) 1223 intel_lrc_irq_handler(&dev_priv->ring[VCS]); 1224 if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT)) 1225 notify_ring(&dev_priv->ring[VCS]); 1226 1227 if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT)) 1228 intel_lrc_irq_handler(&dev_priv->ring[VCS2]); 1229 if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT)) 1230 notify_ring(&dev_priv->ring[VCS2]); 1231 } else 1232 DRM_ERROR("The master control interrupt lied (GT1)!\n"); 1233 } 1234 1235 if (master_ctl & GEN8_GT_VECS_IRQ) { 1236 u32 tmp = I915_READ_FW(GEN8_GT_IIR(3)); 1237 if (tmp) { 1238 I915_WRITE_FW(GEN8_GT_IIR(3), tmp); 1239 ret = IRQ_HANDLED; 1240 1241 if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT)) 1242 intel_lrc_irq_handler(&dev_priv->ring[VECS]); 1243 if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT)) 1244 notify_ring(&dev_priv->ring[VECS]); 1245 } else 1246 DRM_ERROR("The master control interrupt lied (GT3)!\n"); 1247 } 1248 1249 if (master_ctl & GEN8_GT_PM_IRQ) { 1250 u32 tmp = I915_READ_FW(GEN8_GT_IIR(2)); 1251 if (tmp & dev_priv->pm_rps_events) { 1252 I915_WRITE_FW(GEN8_GT_IIR(2), 1253 tmp & dev_priv->pm_rps_events); 1254 ret = IRQ_HANDLED; 1255 gen6_rps_irq_handler(dev_priv, tmp); 1256 } else 1257 DRM_ERROR("The master control interrupt lied (PM)!\n"); 1258 } 1259 1260 return ret; 1261 } 1262 1263 static bool bxt_port_hotplug_long_detect(enum port port, u32 val) 1264 { 1265 switch (port) { 1266 case PORT_A: 1267 return val & BXT_PORTA_HOTPLUG_LONG_DETECT; 1268 case PORT_B: 1269 return val & PORTB_HOTPLUG_LONG_DETECT; 1270 case PORT_C: 1271 return val & PORTC_HOTPLUG_LONG_DETECT; 1272 case PORT_D: 1273 return val & PORTD_HOTPLUG_LONG_DETECT; 1274 default: 1275 return false; 1276 } 1277 } 1278 1279 static bool pch_port_hotplug_long_detect(enum port port, u32 val) 1280 { 1281 switch (port) { 1282 case PORT_B: 1283 return val & PORTB_HOTPLUG_LONG_DETECT; 1284 case PORT_C: 1285 return val & PORTC_HOTPLUG_LONG_DETECT; 1286 case PORT_D: 1287 return val & PORTD_HOTPLUG_LONG_DETECT; 1288 case PORT_E: 1289 return val & PORTE_HOTPLUG_LONG_DETECT; 1290 default: 1291 return false; 1292 } 1293 } 1294 1295 static bool i9xx_port_hotplug_long_detect(enum port port, u32 val) 1296 { 1297 switch (port) { 1298 case PORT_B: 1299 return val & PORTB_HOTPLUG_INT_LONG_PULSE; 1300 case PORT_C: 1301 return val & PORTC_HOTPLUG_INT_LONG_PULSE; 1302 case PORT_D: 1303 return val & PORTD_HOTPLUG_INT_LONG_PULSE; 1304 default: 1305 return false; 1306 } 1307 } 1308 1309 /* Get a bit mask of pins that have triggered, and which ones may be long. */ 1310 static void intel_get_hpd_pins(u32 *pin_mask, u32 *long_mask, 1311 u32 hotplug_trigger, u32 dig_hotplug_reg, 1312 const u32 hpd[HPD_NUM_PINS], 1313 bool long_pulse_detect(enum port port, u32 val)) 1314 { 1315 enum port port; 1316 int i; 1317 1318 *pin_mask = 0; 1319 *long_mask = 0; 1320 1321 for_each_hpd_pin(i) { 1322 if ((hpd[i] & hotplug_trigger) == 0) 1323 continue; 1324 1325 *pin_mask |= BIT(i); 1326 1327 if (!intel_hpd_pin_to_port(i, &port)) 1328 continue; 1329 1330 if (long_pulse_detect(port, dig_hotplug_reg)) 1331 *long_mask |= BIT(i); 1332 } 1333 1334 DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x, dig 0x%08x, pins 0x%08x\n", 1335 hotplug_trigger, dig_hotplug_reg, *pin_mask); 1336 1337 } 1338 1339 static void gmbus_irq_handler(struct drm_device *dev) 1340 { 1341 struct drm_i915_private *dev_priv = dev->dev_private; 1342 1343 wake_up_all(&dev_priv->gmbus_wait_queue); 1344 } 1345 1346 static void dp_aux_irq_handler(struct drm_device *dev) 1347 { 1348 struct drm_i915_private *dev_priv = dev->dev_private; 1349 1350 wake_up_all(&dev_priv->gmbus_wait_queue); 1351 } 1352 1353 #if defined(CONFIG_DEBUG_FS) 1354 static void display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe, 1355 uint32_t crc0, uint32_t crc1, 1356 uint32_t crc2, uint32_t crc3, 1357 uint32_t crc4) 1358 { 1359 struct drm_i915_private *dev_priv = dev->dev_private; 1360 struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe]; 1361 struct intel_pipe_crc_entry *entry; 1362 int head, tail; 1363 1364 spin_lock(&pipe_crc->lock); 1365 1366 if (!pipe_crc->entries) { 1367 spin_unlock(&pipe_crc->lock); 1368 DRM_DEBUG_KMS("spurious interrupt\n"); 1369 return; 1370 } 1371 1372 head = pipe_crc->head; 1373 tail = pipe_crc->tail; 1374 1375 if (CIRC_SPACE(head, tail, INTEL_PIPE_CRC_ENTRIES_NR) < 1) { 1376 spin_unlock(&pipe_crc->lock); 1377 DRM_ERROR("CRC buffer overflowing\n"); 1378 return; 1379 } 1380 1381 entry = &pipe_crc->entries[head]; 1382 1383 entry->frame = dev->driver->get_vblank_counter(dev, pipe); 1384 entry->crc[0] = crc0; 1385 entry->crc[1] = crc1; 1386 entry->crc[2] = crc2; 1387 entry->crc[3] = crc3; 1388 entry->crc[4] = crc4; 1389 1390 head = (head + 1) & (INTEL_PIPE_CRC_ENTRIES_NR - 1); 1391 pipe_crc->head = head; 1392 1393 spin_unlock(&pipe_crc->lock); 1394 1395 wake_up_interruptible(&pipe_crc->wq); 1396 } 1397 #else 1398 static inline void 1399 display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe, 1400 uint32_t crc0, uint32_t crc1, 1401 uint32_t crc2, uint32_t crc3, 1402 uint32_t crc4) {} 1403 #endif 1404 1405 1406 static void hsw_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe) 1407 { 1408 struct drm_i915_private *dev_priv = dev->dev_private; 1409 1410 display_pipe_crc_irq_handler(dev, pipe, 1411 I915_READ(PIPE_CRC_RES_1_IVB(pipe)), 1412 0, 0, 0, 0); 1413 } 1414 1415 static void ivb_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe) 1416 { 1417 struct drm_i915_private *dev_priv = dev->dev_private; 1418 1419 display_pipe_crc_irq_handler(dev, pipe, 1420 I915_READ(PIPE_CRC_RES_1_IVB(pipe)), 1421 I915_READ(PIPE_CRC_RES_2_IVB(pipe)), 1422 I915_READ(PIPE_CRC_RES_3_IVB(pipe)), 1423 I915_READ(PIPE_CRC_RES_4_IVB(pipe)), 1424 I915_READ(PIPE_CRC_RES_5_IVB(pipe))); 1425 } 1426 1427 static void i9xx_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe) 1428 { 1429 struct drm_i915_private *dev_priv = dev->dev_private; 1430 uint32_t res1, res2; 1431 1432 if (INTEL_INFO(dev)->gen >= 3) 1433 res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe)); 1434 else 1435 res1 = 0; 1436 1437 if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev)) 1438 res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe)); 1439 else 1440 res2 = 0; 1441 1442 display_pipe_crc_irq_handler(dev, pipe, 1443 I915_READ(PIPE_CRC_RES_RED(pipe)), 1444 I915_READ(PIPE_CRC_RES_GREEN(pipe)), 1445 I915_READ(PIPE_CRC_RES_BLUE(pipe)), 1446 res1, res2); 1447 } 1448 1449 /* The RPS events need forcewake, so we add them to a work queue and mask their 1450 * IMR bits until the work is done. Other interrupts can be processed without 1451 * the work queue. */ 1452 static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir) 1453 { 1454 if (pm_iir & dev_priv->pm_rps_events) { 1455 spin_lock(&dev_priv->irq_lock); 1456 gen6_disable_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events); 1457 if (dev_priv->rps.interrupts_enabled) { 1458 dev_priv->rps.pm_iir |= pm_iir & dev_priv->pm_rps_events; 1459 queue_work(dev_priv->wq, &dev_priv->rps.work); 1460 } 1461 spin_unlock(&dev_priv->irq_lock); 1462 } 1463 1464 if (INTEL_INFO(dev_priv)->gen >= 8) 1465 return; 1466 1467 if (HAS_VEBOX(dev_priv->dev)) { 1468 if (pm_iir & PM_VEBOX_USER_INTERRUPT) 1469 notify_ring(&dev_priv->ring[VECS]); 1470 1471 if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) 1472 DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir); 1473 } 1474 } 1475 1476 static bool intel_pipe_handle_vblank(struct drm_device *dev, enum pipe pipe) 1477 { 1478 if (!drm_handle_vblank(dev, pipe)) 1479 return false; 1480 1481 return true; 1482 } 1483 1484 static void valleyview_pipestat_irq_handler(struct drm_device *dev, u32 iir) 1485 { 1486 struct drm_i915_private *dev_priv = dev->dev_private; 1487 u32 pipe_stats[I915_MAX_PIPES] = { }; 1488 int pipe; 1489 1490 spin_lock(&dev_priv->irq_lock); 1491 for_each_pipe(dev_priv, pipe) { 1492 int reg; 1493 u32 mask, iir_bit = 0; 1494 1495 /* 1496 * PIPESTAT bits get signalled even when the interrupt is 1497 * disabled with the mask bits, and some of the status bits do 1498 * not generate interrupts at all (like the underrun bit). Hence 1499 * we need to be careful that we only handle what we want to 1500 * handle. 1501 */ 1502 1503 /* fifo underruns are filterered in the underrun handler. */ 1504 mask = PIPE_FIFO_UNDERRUN_STATUS; 1505 1506 switch (pipe) { 1507 case PIPE_A: 1508 iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT; 1509 break; 1510 case PIPE_B: 1511 iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT; 1512 break; 1513 case PIPE_C: 1514 iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT; 1515 break; 1516 } 1517 if (iir & iir_bit) 1518 mask |= dev_priv->pipestat_irq_mask[pipe]; 1519 1520 if (!mask) 1521 continue; 1522 1523 reg = PIPESTAT(pipe); 1524 mask |= PIPESTAT_INT_ENABLE_MASK; 1525 pipe_stats[pipe] = I915_READ(reg) & mask; 1526 1527 /* 1528 * Clear the PIPE*STAT regs before the IIR 1529 */ 1530 if (pipe_stats[pipe] & (PIPE_FIFO_UNDERRUN_STATUS | 1531 PIPESTAT_INT_STATUS_MASK)) 1532 I915_WRITE(reg, pipe_stats[pipe]); 1533 } 1534 spin_unlock(&dev_priv->irq_lock); 1535 1536 for_each_pipe(dev_priv, pipe) { 1537 if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS && 1538 intel_pipe_handle_vblank(dev, pipe)) 1539 intel_check_page_flip(dev, pipe); 1540 1541 if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV) { 1542 intel_prepare_page_flip(dev, pipe); 1543 intel_finish_page_flip(dev, pipe); 1544 } 1545 1546 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS) 1547 i9xx_pipe_crc_irq_handler(dev, pipe); 1548 1549 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS) 1550 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe); 1551 } 1552 1553 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS) 1554 gmbus_irq_handler(dev); 1555 } 1556 1557 static void i9xx_hpd_irq_handler(struct drm_device *dev) 1558 { 1559 struct drm_i915_private *dev_priv = dev->dev_private; 1560 u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT); 1561 u32 pin_mask, long_mask; 1562 1563 if (!hotplug_status) 1564 return; 1565 1566 I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status); 1567 /* 1568 * Make sure hotplug status is cleared before we clear IIR, or else we 1569 * may miss hotplug events. 1570 */ 1571 POSTING_READ(PORT_HOTPLUG_STAT); 1572 1573 if (IS_G4X(dev) || IS_VALLEYVIEW(dev)) { 1574 u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X; 1575 1576 intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger, 1577 hotplug_trigger, hpd_status_g4x, 1578 i9xx_port_hotplug_long_detect); 1579 intel_hpd_irq_handler(dev, pin_mask, long_mask); 1580 1581 if (hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X) 1582 dp_aux_irq_handler(dev); 1583 } else { 1584 u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915; 1585 1586 intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger, 1587 hotplug_trigger, hpd_status_i915, 1588 i9xx_port_hotplug_long_detect); 1589 intel_hpd_irq_handler(dev, pin_mask, long_mask); 1590 } 1591 } 1592 1593 static irqreturn_t valleyview_irq_handler(int irq, void *arg) 1594 { 1595 struct drm_device *dev = arg; 1596 struct drm_i915_private *dev_priv = dev->dev_private; 1597 u32 iir, gt_iir, pm_iir; 1598 irqreturn_t ret = IRQ_NONE; 1599 1600 if (!intel_irqs_enabled(dev_priv)) 1601 return IRQ_NONE; 1602 1603 while (true) { 1604 /* Find, clear, then process each source of interrupt */ 1605 1606 gt_iir = I915_READ(GTIIR); 1607 if (gt_iir) 1608 I915_WRITE(GTIIR, gt_iir); 1609 1610 pm_iir = I915_READ(GEN6_PMIIR); 1611 if (pm_iir) 1612 I915_WRITE(GEN6_PMIIR, pm_iir); 1613 1614 iir = I915_READ(VLV_IIR); 1615 if (iir) { 1616 /* Consume port before clearing IIR or we'll miss events */ 1617 if (iir & I915_DISPLAY_PORT_INTERRUPT) 1618 i9xx_hpd_irq_handler(dev); 1619 I915_WRITE(VLV_IIR, iir); 1620 } 1621 1622 if (gt_iir == 0 && pm_iir == 0 && iir == 0) 1623 goto out; 1624 1625 ret = IRQ_HANDLED; 1626 1627 if (gt_iir) 1628 snb_gt_irq_handler(dev, dev_priv, gt_iir); 1629 if (pm_iir) 1630 gen6_rps_irq_handler(dev_priv, pm_iir); 1631 /* Call regardless, as some status bits might not be 1632 * signalled in iir */ 1633 valleyview_pipestat_irq_handler(dev, iir); 1634 } 1635 1636 out: 1637 return ret; 1638 } 1639 1640 static irqreturn_t cherryview_irq_handler(int irq, void *arg) 1641 { 1642 struct drm_device *dev = arg; 1643 struct drm_i915_private *dev_priv = dev->dev_private; 1644 u32 master_ctl, iir; 1645 irqreturn_t ret = IRQ_NONE; 1646 1647 if (!intel_irqs_enabled(dev_priv)) 1648 return IRQ_NONE; 1649 1650 for (;;) { 1651 master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL; 1652 iir = I915_READ(VLV_IIR); 1653 1654 if (master_ctl == 0 && iir == 0) 1655 break; 1656 1657 ret = IRQ_HANDLED; 1658 1659 I915_WRITE(GEN8_MASTER_IRQ, 0); 1660 1661 /* Find, clear, then process each source of interrupt */ 1662 1663 if (iir) { 1664 /* Consume port before clearing IIR or we'll miss events */ 1665 if (iir & I915_DISPLAY_PORT_INTERRUPT) 1666 i9xx_hpd_irq_handler(dev); 1667 I915_WRITE(VLV_IIR, iir); 1668 } 1669 1670 gen8_gt_irq_handler(dev_priv, master_ctl); 1671 1672 /* Call regardless, as some status bits might not be 1673 * signalled in iir */ 1674 valleyview_pipestat_irq_handler(dev, iir); 1675 1676 I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL); 1677 POSTING_READ(GEN8_MASTER_IRQ); 1678 } 1679 1680 return ret; 1681 } 1682 1683 static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir) 1684 { 1685 struct drm_i915_private *dev_priv = dev->dev_private; 1686 int pipe; 1687 u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK; 1688 1689 if (hotplug_trigger) { 1690 u32 dig_hotplug_reg, pin_mask, long_mask; 1691 1692 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG); 1693 I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg); 1694 1695 intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger, 1696 dig_hotplug_reg, hpd_ibx, 1697 pch_port_hotplug_long_detect); 1698 intel_hpd_irq_handler(dev, pin_mask, long_mask); 1699 } 1700 1701 if (pch_iir & SDE_AUDIO_POWER_MASK) { 1702 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >> 1703 SDE_AUDIO_POWER_SHIFT); 1704 DRM_DEBUG_DRIVER("PCH audio power change on port %d\n", 1705 port_name(port)); 1706 } 1707 1708 if (pch_iir & SDE_AUX_MASK) 1709 dp_aux_irq_handler(dev); 1710 1711 if (pch_iir & SDE_GMBUS) 1712 gmbus_irq_handler(dev); 1713 1714 if (pch_iir & SDE_AUDIO_HDCP_MASK) 1715 DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n"); 1716 1717 if (pch_iir & SDE_AUDIO_TRANS_MASK) 1718 DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n"); 1719 1720 if (pch_iir & SDE_POISON) 1721 DRM_ERROR("PCH poison interrupt\n"); 1722 1723 if (pch_iir & SDE_FDI_MASK) 1724 for_each_pipe(dev_priv, pipe) 1725 DRM_DEBUG_DRIVER(" pipe %c FDI IIR: 0x%08x\n", 1726 pipe_name(pipe), 1727 I915_READ(FDI_RX_IIR(pipe))); 1728 1729 if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE)) 1730 DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n"); 1731 1732 if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR)) 1733 DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n"); 1734 1735 if (pch_iir & SDE_TRANSA_FIFO_UNDER) 1736 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A); 1737 1738 if (pch_iir & SDE_TRANSB_FIFO_UNDER) 1739 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B); 1740 } 1741 1742 static void ivb_err_int_handler(struct drm_device *dev) 1743 { 1744 struct drm_i915_private *dev_priv = dev->dev_private; 1745 u32 err_int = I915_READ(GEN7_ERR_INT); 1746 enum pipe pipe; 1747 1748 if (err_int & ERR_INT_POISON) 1749 DRM_ERROR("Poison interrupt\n"); 1750 1751 for_each_pipe(dev_priv, pipe) { 1752 if (err_int & ERR_INT_FIFO_UNDERRUN(pipe)) 1753 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe); 1754 1755 if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) { 1756 if (IS_IVYBRIDGE(dev)) 1757 ivb_pipe_crc_irq_handler(dev, pipe); 1758 else 1759 hsw_pipe_crc_irq_handler(dev, pipe); 1760 } 1761 } 1762 1763 I915_WRITE(GEN7_ERR_INT, err_int); 1764 } 1765 1766 static void cpt_serr_int_handler(struct drm_device *dev) 1767 { 1768 struct drm_i915_private *dev_priv = dev->dev_private; 1769 u32 serr_int = I915_READ(SERR_INT); 1770 1771 if (serr_int & SERR_INT_POISON) 1772 DRM_ERROR("PCH poison interrupt\n"); 1773 1774 if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN) 1775 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A); 1776 1777 if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN) 1778 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B); 1779 1780 if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN) 1781 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_C); 1782 1783 I915_WRITE(SERR_INT, serr_int); 1784 } 1785 1786 static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir) 1787 { 1788 struct drm_i915_private *dev_priv = dev->dev_private; 1789 int pipe; 1790 u32 hotplug_trigger; 1791 1792 if (HAS_PCH_SPT(dev)) 1793 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_SPT; 1794 else 1795 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT; 1796 1797 if (hotplug_trigger) { 1798 u32 dig_hotplug_reg, pin_mask, long_mask; 1799 1800 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG); 1801 I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg); 1802 1803 if (HAS_PCH_SPT(dev)) { 1804 intel_get_hpd_pins(&pin_mask, &long_mask, 1805 hotplug_trigger, 1806 dig_hotplug_reg, hpd_spt, 1807 pch_port_hotplug_long_detect); 1808 1809 /* detect PORTE HP event */ 1810 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG2); 1811 if (pch_port_hotplug_long_detect(PORT_E, 1812 dig_hotplug_reg)) 1813 long_mask |= 1 << HPD_PORT_E; 1814 } else 1815 intel_get_hpd_pins(&pin_mask, &long_mask, 1816 hotplug_trigger, 1817 dig_hotplug_reg, hpd_cpt, 1818 pch_port_hotplug_long_detect); 1819 1820 intel_hpd_irq_handler(dev, pin_mask, long_mask); 1821 } 1822 1823 if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) { 1824 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >> 1825 SDE_AUDIO_POWER_SHIFT_CPT); 1826 DRM_DEBUG_DRIVER("PCH audio power change on port %c\n", 1827 port_name(port)); 1828 } 1829 1830 if (pch_iir & SDE_AUX_MASK_CPT) 1831 dp_aux_irq_handler(dev); 1832 1833 if (pch_iir & SDE_GMBUS_CPT) 1834 gmbus_irq_handler(dev); 1835 1836 if (pch_iir & SDE_AUDIO_CP_REQ_CPT) 1837 DRM_DEBUG_DRIVER("Audio CP request interrupt\n"); 1838 1839 if (pch_iir & SDE_AUDIO_CP_CHG_CPT) 1840 DRM_DEBUG_DRIVER("Audio CP change interrupt\n"); 1841 1842 if (pch_iir & SDE_FDI_MASK_CPT) 1843 for_each_pipe(dev_priv, pipe) 1844 DRM_DEBUG_DRIVER(" pipe %c FDI IIR: 0x%08x\n", 1845 pipe_name(pipe), 1846 I915_READ(FDI_RX_IIR(pipe))); 1847 1848 if (pch_iir & SDE_ERROR_CPT) 1849 cpt_serr_int_handler(dev); 1850 } 1851 1852 static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir) 1853 { 1854 struct drm_i915_private *dev_priv = dev->dev_private; 1855 enum pipe pipe; 1856 1857 if (de_iir & DE_AUX_CHANNEL_A) 1858 dp_aux_irq_handler(dev); 1859 1860 if (de_iir & DE_GSE) 1861 intel_opregion_asle_intr(dev); 1862 1863 if (de_iir & DE_POISON) 1864 DRM_ERROR("Poison interrupt\n"); 1865 1866 for_each_pipe(dev_priv, pipe) { 1867 if (de_iir & DE_PIPE_VBLANK(pipe) && 1868 intel_pipe_handle_vblank(dev, pipe)) 1869 intel_check_page_flip(dev, pipe); 1870 1871 if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe)) 1872 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe); 1873 1874 if (de_iir & DE_PIPE_CRC_DONE(pipe)) 1875 i9xx_pipe_crc_irq_handler(dev, pipe); 1876 1877 /* plane/pipes map 1:1 on ilk+ */ 1878 if (de_iir & DE_PLANE_FLIP_DONE(pipe)) { 1879 intel_prepare_page_flip(dev, pipe); 1880 intel_finish_page_flip_plane(dev, pipe); 1881 } 1882 } 1883 1884 /* check event from PCH */ 1885 if (de_iir & DE_PCH_EVENT) { 1886 u32 pch_iir = I915_READ(SDEIIR); 1887 1888 if (HAS_PCH_CPT(dev)) 1889 cpt_irq_handler(dev, pch_iir); 1890 else 1891 ibx_irq_handler(dev, pch_iir); 1892 1893 /* should clear PCH hotplug event before clear CPU irq */ 1894 I915_WRITE(SDEIIR, pch_iir); 1895 } 1896 1897 if (IS_GEN5(dev) && de_iir & DE_PCU_EVENT) 1898 ironlake_rps_change_irq_handler(dev); 1899 } 1900 1901 static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir) 1902 { 1903 struct drm_i915_private *dev_priv = dev->dev_private; 1904 enum pipe pipe; 1905 1906 if (de_iir & DE_ERR_INT_IVB) 1907 ivb_err_int_handler(dev); 1908 1909 if (de_iir & DE_AUX_CHANNEL_A_IVB) 1910 dp_aux_irq_handler(dev); 1911 1912 if (de_iir & DE_GSE_IVB) 1913 intel_opregion_asle_intr(dev); 1914 1915 for_each_pipe(dev_priv, pipe) { 1916 if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)) && 1917 intel_pipe_handle_vblank(dev, pipe)) 1918 intel_check_page_flip(dev, pipe); 1919 1920 /* plane/pipes map 1:1 on ilk+ */ 1921 if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe)) { 1922 intel_prepare_page_flip(dev, pipe); 1923 intel_finish_page_flip_plane(dev, pipe); 1924 } 1925 } 1926 1927 /* check event from PCH */ 1928 if (!HAS_PCH_NOP(dev) && (de_iir & DE_PCH_EVENT_IVB)) { 1929 u32 pch_iir = I915_READ(SDEIIR); 1930 1931 cpt_irq_handler(dev, pch_iir); 1932 1933 /* clear PCH hotplug event before clear CPU irq */ 1934 I915_WRITE(SDEIIR, pch_iir); 1935 } 1936 } 1937 1938 /* 1939 * To handle irqs with the minimum potential races with fresh interrupts, we: 1940 * 1 - Disable Master Interrupt Control. 1941 * 2 - Find the source(s) of the interrupt. 1942 * 3 - Clear the Interrupt Identity bits (IIR). 1943 * 4 - Process the interrupt(s) that had bits set in the IIRs. 1944 * 5 - Re-enable Master Interrupt Control. 1945 */ 1946 static irqreturn_t ironlake_irq_handler(int irq, void *arg) 1947 { 1948 struct drm_device *dev = arg; 1949 struct drm_i915_private *dev_priv = dev->dev_private; 1950 u32 de_iir, gt_iir, de_ier, sde_ier = 0; 1951 irqreturn_t ret = IRQ_NONE; 1952 1953 if (!intel_irqs_enabled(dev_priv)) 1954 return IRQ_NONE; 1955 1956 /* We get interrupts on unclaimed registers, so check for this before we 1957 * do any I915_{READ,WRITE}. */ 1958 intel_uncore_check_errors(dev); 1959 1960 /* disable master interrupt before clearing iir */ 1961 de_ier = I915_READ(DEIER); 1962 I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL); 1963 POSTING_READ(DEIER); 1964 1965 /* Disable south interrupts. We'll only write to SDEIIR once, so further 1966 * interrupts will will be stored on its back queue, and then we'll be 1967 * able to process them after we restore SDEIER (as soon as we restore 1968 * it, we'll get an interrupt if SDEIIR still has something to process 1969 * due to its back queue). */ 1970 if (!HAS_PCH_NOP(dev)) { 1971 sde_ier = I915_READ(SDEIER); 1972 I915_WRITE(SDEIER, 0); 1973 POSTING_READ(SDEIER); 1974 } 1975 1976 /* Find, clear, then process each source of interrupt */ 1977 1978 gt_iir = I915_READ(GTIIR); 1979 if (gt_iir) { 1980 I915_WRITE(GTIIR, gt_iir); 1981 ret = IRQ_HANDLED; 1982 if (INTEL_INFO(dev)->gen >= 6) 1983 snb_gt_irq_handler(dev, dev_priv, gt_iir); 1984 else 1985 ilk_gt_irq_handler(dev, dev_priv, gt_iir); 1986 } 1987 1988 de_iir = I915_READ(DEIIR); 1989 if (de_iir) { 1990 I915_WRITE(DEIIR, de_iir); 1991 ret = IRQ_HANDLED; 1992 if (INTEL_INFO(dev)->gen >= 7) 1993 ivb_display_irq_handler(dev, de_iir); 1994 else 1995 ilk_display_irq_handler(dev, de_iir); 1996 } 1997 1998 if (INTEL_INFO(dev)->gen >= 6) { 1999 u32 pm_iir = I915_READ(GEN6_PMIIR); 2000 if (pm_iir) { 2001 I915_WRITE(GEN6_PMIIR, pm_iir); 2002 ret = IRQ_HANDLED; 2003 gen6_rps_irq_handler(dev_priv, pm_iir); 2004 } 2005 } 2006 2007 I915_WRITE(DEIER, de_ier); 2008 POSTING_READ(DEIER); 2009 if (!HAS_PCH_NOP(dev)) { 2010 I915_WRITE(SDEIER, sde_ier); 2011 POSTING_READ(SDEIER); 2012 } 2013 2014 return ret; 2015 } 2016 2017 static void bxt_hpd_handler(struct drm_device *dev, uint32_t iir_status) 2018 { 2019 struct drm_i915_private *dev_priv = dev->dev_private; 2020 u32 hp_control, hp_trigger; 2021 u32 pin_mask, long_mask; 2022 2023 /* Get the status */ 2024 hp_trigger = iir_status & BXT_DE_PORT_HOTPLUG_MASK; 2025 hp_control = I915_READ(BXT_HOTPLUG_CTL); 2026 2027 /* Hotplug not enabled ? */ 2028 if (!(hp_control & BXT_HOTPLUG_CTL_MASK)) { 2029 DRM_ERROR("Interrupt when HPD disabled\n"); 2030 return; 2031 } 2032 2033 /* Clear sticky bits in hpd status */ 2034 I915_WRITE(BXT_HOTPLUG_CTL, hp_control); 2035 2036 intel_get_hpd_pins(&pin_mask, &long_mask, hp_trigger, hp_control, 2037 hpd_bxt, bxt_port_hotplug_long_detect); 2038 intel_hpd_irq_handler(dev, pin_mask, long_mask); 2039 } 2040 2041 static irqreturn_t gen8_irq_handler(int irq, void *arg) 2042 { 2043 struct drm_device *dev = arg; 2044 struct drm_i915_private *dev_priv = dev->dev_private; 2045 u32 master_ctl; 2046 irqreturn_t ret = IRQ_NONE; 2047 uint32_t tmp = 0; 2048 enum pipe pipe; 2049 u32 aux_mask = GEN8_AUX_CHANNEL_A; 2050 2051 if (!intel_irqs_enabled(dev_priv)) 2052 return IRQ_NONE; 2053 2054 if (IS_GEN9(dev)) 2055 aux_mask |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C | 2056 GEN9_AUX_CHANNEL_D; 2057 2058 master_ctl = I915_READ_FW(GEN8_MASTER_IRQ); 2059 master_ctl &= ~GEN8_MASTER_IRQ_CONTROL; 2060 if (!master_ctl) 2061 return IRQ_NONE; 2062 2063 I915_WRITE_FW(GEN8_MASTER_IRQ, 0); 2064 2065 /* Find, clear, then process each source of interrupt */ 2066 2067 ret = gen8_gt_irq_handler(dev_priv, master_ctl); 2068 2069 if (master_ctl & GEN8_DE_MISC_IRQ) { 2070 tmp = I915_READ(GEN8_DE_MISC_IIR); 2071 if (tmp) { 2072 I915_WRITE(GEN8_DE_MISC_IIR, tmp); 2073 ret = IRQ_HANDLED; 2074 if (tmp & GEN8_DE_MISC_GSE) 2075 intel_opregion_asle_intr(dev); 2076 else 2077 DRM_ERROR("Unexpected DE Misc interrupt\n"); 2078 } 2079 else 2080 DRM_ERROR("The master control interrupt lied (DE MISC)!\n"); 2081 } 2082 2083 if (master_ctl & GEN8_DE_PORT_IRQ) { 2084 tmp = I915_READ(GEN8_DE_PORT_IIR); 2085 if (tmp) { 2086 bool found = false; 2087 2088 I915_WRITE(GEN8_DE_PORT_IIR, tmp); 2089 ret = IRQ_HANDLED; 2090 2091 if (tmp & aux_mask) { 2092 dp_aux_irq_handler(dev); 2093 found = true; 2094 } 2095 2096 if (IS_BROXTON(dev) && tmp & BXT_DE_PORT_HOTPLUG_MASK) { 2097 bxt_hpd_handler(dev, tmp); 2098 found = true; 2099 } 2100 2101 if (IS_BROXTON(dev) && (tmp & BXT_DE_PORT_GMBUS)) { 2102 gmbus_irq_handler(dev); 2103 found = true; 2104 } 2105 2106 if (!found) 2107 DRM_ERROR("Unexpected DE Port interrupt\n"); 2108 } 2109 else 2110 DRM_ERROR("The master control interrupt lied (DE PORT)!\n"); 2111 } 2112 2113 for_each_pipe(dev_priv, pipe) { 2114 uint32_t pipe_iir, flip_done = 0, fault_errors = 0; 2115 2116 if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe))) 2117 continue; 2118 2119 pipe_iir = I915_READ(GEN8_DE_PIPE_IIR(pipe)); 2120 if (pipe_iir) { 2121 ret = IRQ_HANDLED; 2122 I915_WRITE(GEN8_DE_PIPE_IIR(pipe), pipe_iir); 2123 2124 if (pipe_iir & GEN8_PIPE_VBLANK && 2125 intel_pipe_handle_vblank(dev, pipe)) 2126 intel_check_page_flip(dev, pipe); 2127 2128 if (IS_GEN9(dev)) 2129 flip_done = pipe_iir & GEN9_PIPE_PLANE1_FLIP_DONE; 2130 else 2131 flip_done = pipe_iir & GEN8_PIPE_PRIMARY_FLIP_DONE; 2132 2133 if (flip_done) { 2134 intel_prepare_page_flip(dev, pipe); 2135 intel_finish_page_flip_plane(dev, pipe); 2136 } 2137 2138 if (pipe_iir & GEN8_PIPE_CDCLK_CRC_DONE) 2139 hsw_pipe_crc_irq_handler(dev, pipe); 2140 2141 if (pipe_iir & GEN8_PIPE_FIFO_UNDERRUN) 2142 intel_cpu_fifo_underrun_irq_handler(dev_priv, 2143 pipe); 2144 2145 2146 if (IS_GEN9(dev)) 2147 fault_errors = pipe_iir & GEN9_DE_PIPE_IRQ_FAULT_ERRORS; 2148 else 2149 fault_errors = pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS; 2150 2151 if (fault_errors) 2152 DRM_ERROR("Fault errors on pipe %c\n: 0x%08x", 2153 pipe_name(pipe), 2154 pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS); 2155 } else 2156 DRM_ERROR("The master control interrupt lied (DE PIPE)!\n"); 2157 } 2158 2159 if (HAS_PCH_SPLIT(dev) && !HAS_PCH_NOP(dev) && 2160 master_ctl & GEN8_DE_PCH_IRQ) { 2161 /* 2162 * FIXME(BDW): Assume for now that the new interrupt handling 2163 * scheme also closed the SDE interrupt handling race we've seen 2164 * on older pch-split platforms. But this needs testing. 2165 */ 2166 u32 pch_iir = I915_READ(SDEIIR); 2167 if (pch_iir) { 2168 I915_WRITE(SDEIIR, pch_iir); 2169 ret = IRQ_HANDLED; 2170 cpt_irq_handler(dev, pch_iir); 2171 } else 2172 DRM_ERROR("The master control interrupt lied (SDE)!\n"); 2173 2174 } 2175 2176 I915_WRITE_FW(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL); 2177 POSTING_READ_FW(GEN8_MASTER_IRQ); 2178 2179 return ret; 2180 } 2181 2182 static void i915_error_wake_up(struct drm_i915_private *dev_priv, 2183 bool reset_completed) 2184 { 2185 struct intel_engine_cs *ring; 2186 int i; 2187 2188 /* 2189 * Notify all waiters for GPU completion events that reset state has 2190 * been changed, and that they need to restart their wait after 2191 * checking for potential errors (and bail out to drop locks if there is 2192 * a gpu reset pending so that i915_error_work_func can acquire them). 2193 */ 2194 2195 /* Wake up __wait_seqno, potentially holding dev->struct_mutex. */ 2196 for_each_ring(ring, dev_priv, i) 2197 wake_up_all(&ring->irq_queue); 2198 2199 /* Wake up intel_crtc_wait_for_pending_flips, holding crtc->mutex. */ 2200 wake_up_all(&dev_priv->pending_flip_queue); 2201 2202 /* 2203 * Signal tasks blocked in i915_gem_wait_for_error that the pending 2204 * reset state is cleared. 2205 */ 2206 if (reset_completed) 2207 wake_up_all(&dev_priv->gpu_error.reset_queue); 2208 } 2209 2210 /** 2211 * i915_reset_and_wakeup - do process context error handling work 2212 * 2213 * Fire an error uevent so userspace can see that a hang or error 2214 * was detected. 2215 */ 2216 static void i915_reset_and_wakeup(struct drm_device *dev) 2217 { 2218 struct drm_i915_private *dev_priv = to_i915(dev); 2219 struct i915_gpu_error *error = &dev_priv->gpu_error; 2220 char *error_event[] = { I915_ERROR_UEVENT "=1", NULL }; 2221 char *reset_event[] = { I915_RESET_UEVENT "=1", NULL }; 2222 char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL }; 2223 int ret; 2224 2225 kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE, error_event); 2226 2227 /* 2228 * Note that there's only one work item which does gpu resets, so we 2229 * need not worry about concurrent gpu resets potentially incrementing 2230 * error->reset_counter twice. We only need to take care of another 2231 * racing irq/hangcheck declaring the gpu dead for a second time. A 2232 * quick check for that is good enough: schedule_work ensures the 2233 * correct ordering between hang detection and this work item, and since 2234 * the reset in-progress bit is only ever set by code outside of this 2235 * work we don't need to worry about any other races. 2236 */ 2237 if (i915_reset_in_progress(error) && !i915_terminally_wedged(error)) { 2238 DRM_DEBUG_DRIVER("resetting chip\n"); 2239 kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE, 2240 reset_event); 2241 2242 /* 2243 * In most cases it's guaranteed that we get here with an RPM 2244 * reference held, for example because there is a pending GPU 2245 * request that won't finish until the reset is done. This 2246 * isn't the case at least when we get here by doing a 2247 * simulated reset via debugs, so get an RPM reference. 2248 */ 2249 intel_runtime_pm_get(dev_priv); 2250 2251 intel_prepare_reset(dev); 2252 2253 /* 2254 * All state reset _must_ be completed before we update the 2255 * reset counter, for otherwise waiters might miss the reset 2256 * pending state and not properly drop locks, resulting in 2257 * deadlocks with the reset work. 2258 */ 2259 ret = i915_reset(dev); 2260 2261 intel_finish_reset(dev); 2262 2263 intel_runtime_pm_put(dev_priv); 2264 2265 if (ret == 0) { 2266 /* 2267 * After all the gem state is reset, increment the reset 2268 * counter and wake up everyone waiting for the reset to 2269 * complete. 2270 * 2271 * Since unlock operations are a one-sided barrier only, 2272 * we need to insert a barrier here to order any seqno 2273 * updates before 2274 * the counter increment. 2275 */ 2276 smp_mb__before_atomic(); 2277 atomic_inc(&dev_priv->gpu_error.reset_counter); 2278 2279 kobject_uevent_env(&dev->primary->kdev->kobj, 2280 KOBJ_CHANGE, reset_done_event); 2281 } else { 2282 atomic_or(I915_WEDGED, &error->reset_counter); 2283 } 2284 2285 /* 2286 * Note: The wake_up also serves as a memory barrier so that 2287 * waiters see the update value of the reset counter atomic_t. 2288 */ 2289 i915_error_wake_up(dev_priv, true); 2290 } 2291 } 2292 2293 static void i915_report_and_clear_eir(struct drm_device *dev) 2294 { 2295 struct drm_i915_private *dev_priv = dev->dev_private; 2296 uint32_t instdone[I915_NUM_INSTDONE_REG]; 2297 u32 eir = I915_READ(EIR); 2298 int pipe, i; 2299 2300 if (!eir) 2301 return; 2302 2303 pr_err("render error detected, EIR: 0x%08x\n", eir); 2304 2305 i915_get_extra_instdone(dev, instdone); 2306 2307 if (IS_G4X(dev)) { 2308 if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) { 2309 u32 ipeir = I915_READ(IPEIR_I965); 2310 2311 pr_err(" IPEIR: 0x%08x\n", I915_READ(IPEIR_I965)); 2312 pr_err(" IPEHR: 0x%08x\n", I915_READ(IPEHR_I965)); 2313 for (i = 0; i < ARRAY_SIZE(instdone); i++) 2314 pr_err(" INSTDONE_%d: 0x%08x\n", i, instdone[i]); 2315 pr_err(" INSTPS: 0x%08x\n", I915_READ(INSTPS)); 2316 pr_err(" ACTHD: 0x%08x\n", I915_READ(ACTHD_I965)); 2317 I915_WRITE(IPEIR_I965, ipeir); 2318 POSTING_READ(IPEIR_I965); 2319 } 2320 if (eir & GM45_ERROR_PAGE_TABLE) { 2321 u32 pgtbl_err = I915_READ(PGTBL_ER); 2322 pr_err("page table error\n"); 2323 pr_err(" PGTBL_ER: 0x%08x\n", pgtbl_err); 2324 I915_WRITE(PGTBL_ER, pgtbl_err); 2325 POSTING_READ(PGTBL_ER); 2326 } 2327 } 2328 2329 if (!IS_GEN2(dev)) { 2330 if (eir & I915_ERROR_PAGE_TABLE) { 2331 u32 pgtbl_err = I915_READ(PGTBL_ER); 2332 pr_err("page table error\n"); 2333 pr_err(" PGTBL_ER: 0x%08x\n", pgtbl_err); 2334 I915_WRITE(PGTBL_ER, pgtbl_err); 2335 POSTING_READ(PGTBL_ER); 2336 } 2337 } 2338 2339 if (eir & I915_ERROR_MEMORY_REFRESH) { 2340 pr_err("memory refresh error:\n"); 2341 for_each_pipe(dev_priv, pipe) 2342 pr_err("pipe %c stat: 0x%08x\n", 2343 pipe_name(pipe), I915_READ(PIPESTAT(pipe))); 2344 /* pipestat has already been acked */ 2345 } 2346 if (eir & I915_ERROR_INSTRUCTION) { 2347 pr_err("instruction error\n"); 2348 pr_err(" INSTPM: 0x%08x\n", I915_READ(INSTPM)); 2349 for (i = 0; i < ARRAY_SIZE(instdone); i++) 2350 pr_err(" INSTDONE_%d: 0x%08x\n", i, instdone[i]); 2351 if (INTEL_INFO(dev)->gen < 4) { 2352 u32 ipeir = I915_READ(IPEIR); 2353 2354 pr_err(" IPEIR: 0x%08x\n", I915_READ(IPEIR)); 2355 pr_err(" IPEHR: 0x%08x\n", I915_READ(IPEHR)); 2356 pr_err(" ACTHD: 0x%08x\n", I915_READ(ACTHD)); 2357 I915_WRITE(IPEIR, ipeir); 2358 POSTING_READ(IPEIR); 2359 } else { 2360 u32 ipeir = I915_READ(IPEIR_I965); 2361 2362 pr_err(" IPEIR: 0x%08x\n", I915_READ(IPEIR_I965)); 2363 pr_err(" IPEHR: 0x%08x\n", I915_READ(IPEHR_I965)); 2364 pr_err(" INSTPS: 0x%08x\n", I915_READ(INSTPS)); 2365 pr_err(" ACTHD: 0x%08x\n", I915_READ(ACTHD_I965)); 2366 I915_WRITE(IPEIR_I965, ipeir); 2367 POSTING_READ(IPEIR_I965); 2368 } 2369 } 2370 2371 I915_WRITE(EIR, eir); 2372 POSTING_READ(EIR); 2373 eir = I915_READ(EIR); 2374 if (eir) { 2375 /* 2376 * some errors might have become stuck, 2377 * mask them. 2378 */ 2379 DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir); 2380 I915_WRITE(EMR, I915_READ(EMR) | eir); 2381 I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT); 2382 } 2383 } 2384 2385 /** 2386 * i915_handle_error - handle a gpu error 2387 * @dev: drm device 2388 * 2389 * Do some basic checking of regsiter state at error time and 2390 * dump it to the syslog. Also call i915_capture_error_state() to make 2391 * sure we get a record and make it available in debugfs. Fire a uevent 2392 * so userspace knows something bad happened (should trigger collection 2393 * of a ring dump etc.). 2394 */ 2395 void i915_handle_error(struct drm_device *dev, bool wedged, 2396 const char *fmt, ...) 2397 { 2398 struct drm_i915_private *dev_priv = dev->dev_private; 2399 va_list args; 2400 char error_msg[80]; 2401 2402 va_start(args, fmt); 2403 vscnprintf(error_msg, sizeof(error_msg), fmt, args); 2404 va_end(args); 2405 2406 i915_capture_error_state(dev, wedged, error_msg); 2407 i915_report_and_clear_eir(dev); 2408 2409 if (wedged) { 2410 atomic_or(I915_RESET_IN_PROGRESS_FLAG, 2411 &dev_priv->gpu_error.reset_counter); 2412 2413 /* 2414 * Wakeup waiting processes so that the reset function 2415 * i915_reset_and_wakeup doesn't deadlock trying to grab 2416 * various locks. By bumping the reset counter first, the woken 2417 * processes will see a reset in progress and back off, 2418 * releasing their locks and then wait for the reset completion. 2419 * We must do this for _all_ gpu waiters that might hold locks 2420 * that the reset work needs to acquire. 2421 * 2422 * Note: The wake_up serves as the required memory barrier to 2423 * ensure that the waiters see the updated value of the reset 2424 * counter atomic_t. 2425 */ 2426 i915_error_wake_up(dev_priv, false); 2427 } 2428 2429 i915_reset_and_wakeup(dev); 2430 } 2431 2432 /* Called from drm generic code, passed 'crtc' which 2433 * we use as a pipe index 2434 */ 2435 static int i915_enable_vblank(struct drm_device *dev, int pipe) 2436 { 2437 struct drm_i915_private *dev_priv = dev->dev_private; 2438 unsigned long irqflags; 2439 2440 spin_lock_irqsave(&dev_priv->irq_lock, irqflags); 2441 if (INTEL_INFO(dev)->gen >= 4) 2442 i915_enable_pipestat(dev_priv, pipe, 2443 PIPE_START_VBLANK_INTERRUPT_STATUS); 2444 else 2445 i915_enable_pipestat(dev_priv, pipe, 2446 PIPE_VBLANK_INTERRUPT_STATUS); 2447 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); 2448 2449 return 0; 2450 } 2451 2452 static int ironlake_enable_vblank(struct drm_device *dev, int pipe) 2453 { 2454 struct drm_i915_private *dev_priv = dev->dev_private; 2455 unsigned long irqflags; 2456 uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) : 2457 DE_PIPE_VBLANK(pipe); 2458 2459 spin_lock_irqsave(&dev_priv->irq_lock, irqflags); 2460 ironlake_enable_display_irq(dev_priv, bit); 2461 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); 2462 2463 return 0; 2464 } 2465 2466 static int valleyview_enable_vblank(struct drm_device *dev, int pipe) 2467 { 2468 struct drm_i915_private *dev_priv = dev->dev_private; 2469 unsigned long irqflags; 2470 2471 spin_lock_irqsave(&dev_priv->irq_lock, irqflags); 2472 i915_enable_pipestat(dev_priv, pipe, 2473 PIPE_START_VBLANK_INTERRUPT_STATUS); 2474 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); 2475 2476 return 0; 2477 } 2478 2479 static int gen8_enable_vblank(struct drm_device *dev, int pipe) 2480 { 2481 struct drm_i915_private *dev_priv = dev->dev_private; 2482 unsigned long irqflags; 2483 2484 spin_lock_irqsave(&dev_priv->irq_lock, irqflags); 2485 dev_priv->de_irq_mask[pipe] &= ~GEN8_PIPE_VBLANK; 2486 I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]); 2487 POSTING_READ(GEN8_DE_PIPE_IMR(pipe)); 2488 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); 2489 return 0; 2490 } 2491 2492 /* Called from drm generic code, passed 'crtc' which 2493 * we use as a pipe index 2494 */ 2495 static void i915_disable_vblank(struct drm_device *dev, int pipe) 2496 { 2497 struct drm_i915_private *dev_priv = dev->dev_private; 2498 unsigned long irqflags; 2499 2500 spin_lock_irqsave(&dev_priv->irq_lock, irqflags); 2501 i915_disable_pipestat(dev_priv, pipe, 2502 PIPE_VBLANK_INTERRUPT_STATUS | 2503 PIPE_START_VBLANK_INTERRUPT_STATUS); 2504 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); 2505 } 2506 2507 static void ironlake_disable_vblank(struct drm_device *dev, int pipe) 2508 { 2509 struct drm_i915_private *dev_priv = dev->dev_private; 2510 unsigned long irqflags; 2511 uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) : 2512 DE_PIPE_VBLANK(pipe); 2513 2514 spin_lock_irqsave(&dev_priv->irq_lock, irqflags); 2515 ironlake_disable_display_irq(dev_priv, bit); 2516 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); 2517 } 2518 2519 static void valleyview_disable_vblank(struct drm_device *dev, int pipe) 2520 { 2521 struct drm_i915_private *dev_priv = dev->dev_private; 2522 unsigned long irqflags; 2523 2524 spin_lock_irqsave(&dev_priv->irq_lock, irqflags); 2525 i915_disable_pipestat(dev_priv, pipe, 2526 PIPE_START_VBLANK_INTERRUPT_STATUS); 2527 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); 2528 } 2529 2530 static void gen8_disable_vblank(struct drm_device *dev, int pipe) 2531 { 2532 struct drm_i915_private *dev_priv = dev->dev_private; 2533 unsigned long irqflags; 2534 2535 spin_lock_irqsave(&dev_priv->irq_lock, irqflags); 2536 dev_priv->de_irq_mask[pipe] |= GEN8_PIPE_VBLANK; 2537 I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]); 2538 POSTING_READ(GEN8_DE_PIPE_IMR(pipe)); 2539 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); 2540 } 2541 2542 static bool 2543 ring_idle(struct intel_engine_cs *ring, u32 seqno) 2544 { 2545 return (list_empty(&ring->request_list) || 2546 i915_seqno_passed(seqno, ring->last_submitted_seqno)); 2547 } 2548 2549 static bool 2550 ipehr_is_semaphore_wait(struct drm_device *dev, u32 ipehr) 2551 { 2552 if (INTEL_INFO(dev)->gen >= 8) { 2553 return (ipehr >> 23) == 0x1c; 2554 } else { 2555 ipehr &= ~MI_SEMAPHORE_SYNC_MASK; 2556 return ipehr == (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE | 2557 MI_SEMAPHORE_REGISTER); 2558 } 2559 } 2560 2561 static struct intel_engine_cs * 2562 semaphore_wait_to_signaller_ring(struct intel_engine_cs *ring, u32 ipehr, u64 offset) 2563 { 2564 struct drm_i915_private *dev_priv = ring->dev->dev_private; 2565 struct intel_engine_cs *signaller; 2566 int i; 2567 2568 if (INTEL_INFO(dev_priv->dev)->gen >= 8) { 2569 for_each_ring(signaller, dev_priv, i) { 2570 if (ring == signaller) 2571 continue; 2572 2573 if (offset == signaller->semaphore.signal_ggtt[ring->id]) 2574 return signaller; 2575 } 2576 } else { 2577 u32 sync_bits = ipehr & MI_SEMAPHORE_SYNC_MASK; 2578 2579 for_each_ring(signaller, dev_priv, i) { 2580 if(ring == signaller) 2581 continue; 2582 2583 if (sync_bits == signaller->semaphore.mbox.wait[ring->id]) 2584 return signaller; 2585 } 2586 } 2587 2588 DRM_ERROR("No signaller ring found for ring %i, ipehr 0x%08x, offset 0x%016llx\n", 2589 ring->id, ipehr, offset); 2590 2591 return NULL; 2592 } 2593 2594 static struct intel_engine_cs * 2595 semaphore_waits_for(struct intel_engine_cs *ring, u32 *seqno) 2596 { 2597 struct drm_i915_private *dev_priv = ring->dev->dev_private; 2598 u32 cmd, ipehr, head; 2599 u64 offset = 0; 2600 int i, backwards; 2601 2602 ipehr = I915_READ(RING_IPEHR(ring->mmio_base)); 2603 if (!ipehr_is_semaphore_wait(ring->dev, ipehr)) 2604 return NULL; 2605 2606 /* 2607 * HEAD is likely pointing to the dword after the actual command, 2608 * so scan backwards until we find the MBOX. But limit it to just 3 2609 * or 4 dwords depending on the semaphore wait command size. 2610 * Note that we don't care about ACTHD here since that might 2611 * point at at batch, and semaphores are always emitted into the 2612 * ringbuffer itself. 2613 */ 2614 head = I915_READ_HEAD(ring) & HEAD_ADDR; 2615 backwards = (INTEL_INFO(ring->dev)->gen >= 8) ? 5 : 4; 2616 2617 for (i = backwards; i; --i) { 2618 /* 2619 * Be paranoid and presume the hw has gone off into the wild - 2620 * our ring is smaller than what the hardware (and hence 2621 * HEAD_ADDR) allows. Also handles wrap-around. 2622 */ 2623 head &= ring->buffer->size - 1; 2624 2625 /* This here seems to blow up */ 2626 cmd = ioread32(ring->buffer->virtual_start + head); 2627 if (cmd == ipehr) 2628 break; 2629 2630 head -= 4; 2631 } 2632 2633 if (!i) 2634 return NULL; 2635 2636 *seqno = ioread32(ring->buffer->virtual_start + head + 4) + 1; 2637 if (INTEL_INFO(ring->dev)->gen >= 8) { 2638 offset = ioread32(ring->buffer->virtual_start + head + 12); 2639 offset <<= 32; 2640 offset = ioread32(ring->buffer->virtual_start + head + 8); 2641 } 2642 return semaphore_wait_to_signaller_ring(ring, ipehr, offset); 2643 } 2644 2645 static int semaphore_passed(struct intel_engine_cs *ring) 2646 { 2647 struct drm_i915_private *dev_priv = ring->dev->dev_private; 2648 struct intel_engine_cs *signaller; 2649 u32 seqno; 2650 2651 ring->hangcheck.deadlock++; 2652 2653 signaller = semaphore_waits_for(ring, &seqno); 2654 if (signaller == NULL) 2655 return -1; 2656 2657 /* Prevent pathological recursion due to driver bugs */ 2658 if (signaller->hangcheck.deadlock >= I915_NUM_RINGS) 2659 return -1; 2660 2661 if (i915_seqno_passed(signaller->get_seqno(signaller, false), seqno)) 2662 return 1; 2663 2664 /* cursory check for an unkickable deadlock */ 2665 if (I915_READ_CTL(signaller) & RING_WAIT_SEMAPHORE && 2666 semaphore_passed(signaller) < 0) 2667 return -1; 2668 2669 return 0; 2670 } 2671 2672 static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv) 2673 { 2674 struct intel_engine_cs *ring; 2675 int i; 2676 2677 for_each_ring(ring, dev_priv, i) 2678 ring->hangcheck.deadlock = 0; 2679 } 2680 2681 static enum intel_ring_hangcheck_action 2682 ring_stuck(struct intel_engine_cs *ring, u64 acthd) 2683 { 2684 struct drm_device *dev = ring->dev; 2685 struct drm_i915_private *dev_priv = dev->dev_private; 2686 u32 tmp; 2687 2688 if (acthd != ring->hangcheck.acthd) { 2689 if (acthd > ring->hangcheck.max_acthd) { 2690 ring->hangcheck.max_acthd = acthd; 2691 return HANGCHECK_ACTIVE; 2692 } 2693 2694 return HANGCHECK_ACTIVE_LOOP; 2695 } 2696 2697 if (IS_GEN2(dev)) 2698 return HANGCHECK_HUNG; 2699 2700 /* Is the chip hanging on a WAIT_FOR_EVENT? 2701 * If so we can simply poke the RB_WAIT bit 2702 * and break the hang. This should work on 2703 * all but the second generation chipsets. 2704 */ 2705 tmp = I915_READ_CTL(ring); 2706 if (tmp & RING_WAIT) { 2707 i915_handle_error(dev, false, 2708 "Kicking stuck wait on %s", 2709 ring->name); 2710 I915_WRITE_CTL(ring, tmp); 2711 return HANGCHECK_KICK; 2712 } 2713 2714 if (INTEL_INFO(dev)->gen >= 6 && tmp & RING_WAIT_SEMAPHORE) { 2715 switch (semaphore_passed(ring)) { 2716 default: 2717 return HANGCHECK_HUNG; 2718 case 1: 2719 i915_handle_error(dev, false, 2720 "Kicking stuck semaphore on %s", 2721 ring->name); 2722 I915_WRITE_CTL(ring, tmp); 2723 return HANGCHECK_KICK; 2724 case 0: 2725 return HANGCHECK_WAIT; 2726 } 2727 } 2728 2729 return HANGCHECK_HUNG; 2730 } 2731 2732 /* 2733 * This is called when the chip hasn't reported back with completed 2734 * batchbuffers in a long time. We keep track per ring seqno progress and 2735 * if there are no progress, hangcheck score for that ring is increased. 2736 * Further, acthd is inspected to see if the ring is stuck. On stuck case 2737 * we kick the ring. If we see no progress on three subsequent calls 2738 * we assume chip is wedged and try to fix it by resetting the chip. 2739 */ 2740 static void i915_hangcheck_elapsed(struct work_struct *work) 2741 { 2742 struct drm_i915_private *dev_priv = 2743 container_of(work, typeof(*dev_priv), 2744 gpu_error.hangcheck_work.work); 2745 struct drm_device *dev = dev_priv->dev; 2746 struct intel_engine_cs *ring; 2747 int i; 2748 int busy_count = 0, rings_hung = 0; 2749 bool stuck[I915_NUM_RINGS] = { 0 }; 2750 #define BUSY 1 2751 #define KICK 5 2752 #define HUNG 20 2753 2754 if (!i915.enable_hangcheck) 2755 return; 2756 2757 for_each_ring(ring, dev_priv, i) { 2758 u64 acthd; 2759 u32 seqno; 2760 bool busy = true; 2761 2762 semaphore_clear_deadlocks(dev_priv); 2763 2764 seqno = ring->get_seqno(ring, false); 2765 acthd = intel_ring_get_active_head(ring); 2766 2767 if (ring->hangcheck.seqno == seqno) { 2768 if (ring_idle(ring, seqno)) { 2769 ring->hangcheck.action = HANGCHECK_IDLE; 2770 2771 if (waitqueue_active(&ring->irq_queue)) { 2772 /* Issue a wake-up to catch stuck h/w. */ 2773 if (!test_and_set_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings)) { 2774 if (!(dev_priv->gpu_error.test_irq_rings & intel_ring_flag(ring))) 2775 DRM_ERROR("Hangcheck timer elapsed... %s idle\n", 2776 ring->name); 2777 else 2778 DRM_INFO("Fake missed irq on %s\n", 2779 ring->name); 2780 wake_up_all(&ring->irq_queue); 2781 } 2782 /* Safeguard against driver failure */ 2783 ring->hangcheck.score += BUSY; 2784 } else 2785 busy = false; 2786 } else { 2787 /* We always increment the hangcheck score 2788 * if the ring is busy and still processing 2789 * the same request, so that no single request 2790 * can run indefinitely (such as a chain of 2791 * batches). The only time we do not increment 2792 * the hangcheck score on this ring, if this 2793 * ring is in a legitimate wait for another 2794 * ring. In that case the waiting ring is a 2795 * victim and we want to be sure we catch the 2796 * right culprit. Then every time we do kick 2797 * the ring, add a small increment to the 2798 * score so that we can catch a batch that is 2799 * being repeatedly kicked and so responsible 2800 * for stalling the machine. 2801 */ 2802 ring->hangcheck.action = ring_stuck(ring, 2803 acthd); 2804 2805 switch (ring->hangcheck.action) { 2806 case HANGCHECK_IDLE: 2807 case HANGCHECK_WAIT: 2808 case HANGCHECK_ACTIVE: 2809 break; 2810 case HANGCHECK_ACTIVE_LOOP: 2811 ring->hangcheck.score += BUSY; 2812 break; 2813 case HANGCHECK_KICK: 2814 ring->hangcheck.score += KICK; 2815 break; 2816 case HANGCHECK_HUNG: 2817 ring->hangcheck.score += HUNG; 2818 stuck[i] = true; 2819 break; 2820 } 2821 } 2822 } else { 2823 ring->hangcheck.action = HANGCHECK_ACTIVE; 2824 2825 /* Gradually reduce the count so that we catch DoS 2826 * attempts across multiple batches. 2827 */ 2828 if (ring->hangcheck.score > 0) 2829 ring->hangcheck.score--; 2830 2831 ring->hangcheck.acthd = ring->hangcheck.max_acthd = 0; 2832 } 2833 2834 ring->hangcheck.seqno = seqno; 2835 ring->hangcheck.acthd = acthd; 2836 busy_count += busy; 2837 } 2838 2839 for_each_ring(ring, dev_priv, i) { 2840 if (ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) { 2841 DRM_INFO("%s on %s\n", 2842 stuck[i] ? "stuck" : "no progress", 2843 ring->name); 2844 rings_hung++; 2845 } 2846 } 2847 2848 if (rings_hung) 2849 return i915_handle_error(dev, true, "Ring hung"); 2850 2851 if (busy_count) 2852 /* Reset timer case chip hangs without another request 2853 * being added */ 2854 i915_queue_hangcheck(dev); 2855 } 2856 2857 void i915_queue_hangcheck(struct drm_device *dev) 2858 { 2859 struct i915_gpu_error *e = &to_i915(dev)->gpu_error; 2860 2861 if (!i915.enable_hangcheck) 2862 return; 2863 2864 /* Don't continually defer the hangcheck so that it is always run at 2865 * least once after work has been scheduled on any ring. Otherwise, 2866 * we will ignore a hung ring if a second ring is kept busy. 2867 */ 2868 2869 queue_delayed_work(e->hangcheck_wq, &e->hangcheck_work, 2870 round_jiffies_up_relative(DRM_I915_HANGCHECK_JIFFIES)); 2871 } 2872 2873 static void ibx_irq_reset(struct drm_device *dev) 2874 { 2875 struct drm_i915_private *dev_priv = dev->dev_private; 2876 2877 if (HAS_PCH_NOP(dev)) 2878 return; 2879 2880 GEN5_IRQ_RESET(SDE); 2881 2882 if (HAS_PCH_CPT(dev) || HAS_PCH_LPT(dev)) 2883 I915_WRITE(SERR_INT, 0xffffffff); 2884 } 2885 2886 /* 2887 * SDEIER is also touched by the interrupt handler to work around missed PCH 2888 * interrupts. Hence we can't update it after the interrupt handler is enabled - 2889 * instead we unconditionally enable all PCH interrupt sources here, but then 2890 * only unmask them as needed with SDEIMR. 2891 * 2892 * This function needs to be called before interrupts are enabled. 2893 */ 2894 static void ibx_irq_pre_postinstall(struct drm_device *dev) 2895 { 2896 struct drm_i915_private *dev_priv = dev->dev_private; 2897 2898 if (HAS_PCH_NOP(dev)) 2899 return; 2900 2901 WARN_ON(I915_READ(SDEIER) != 0); 2902 I915_WRITE(SDEIER, 0xffffffff); 2903 POSTING_READ(SDEIER); 2904 } 2905 2906 static void gen5_gt_irq_reset(struct drm_device *dev) 2907 { 2908 struct drm_i915_private *dev_priv = dev->dev_private; 2909 2910 GEN5_IRQ_RESET(GT); 2911 if (INTEL_INFO(dev)->gen >= 6) 2912 GEN5_IRQ_RESET(GEN6_PM); 2913 } 2914 2915 /* drm_dma.h hooks 2916 */ 2917 static void ironlake_irq_reset(struct drm_device *dev) 2918 { 2919 struct drm_i915_private *dev_priv = dev->dev_private; 2920 2921 I915_WRITE(HWSTAM, 0xffffffff); 2922 2923 GEN5_IRQ_RESET(DE); 2924 if (IS_GEN7(dev)) 2925 I915_WRITE(GEN7_ERR_INT, 0xffffffff); 2926 2927 gen5_gt_irq_reset(dev); 2928 2929 ibx_irq_reset(dev); 2930 } 2931 2932 static void vlv_display_irq_reset(struct drm_i915_private *dev_priv) 2933 { 2934 enum pipe pipe; 2935 2936 I915_WRITE(PORT_HOTPLUG_EN, 0); 2937 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); 2938 2939 for_each_pipe(dev_priv, pipe) 2940 I915_WRITE(PIPESTAT(pipe), 0xffff); 2941 2942 GEN5_IRQ_RESET(VLV_); 2943 } 2944 2945 static void valleyview_irq_preinstall(struct drm_device *dev) 2946 { 2947 struct drm_i915_private *dev_priv = dev->dev_private; 2948 2949 /* VLV magic */ 2950 I915_WRITE(VLV_IMR, 0); 2951 I915_WRITE(RING_IMR(RENDER_RING_BASE), 0); 2952 I915_WRITE(RING_IMR(GEN6_BSD_RING_BASE), 0); 2953 I915_WRITE(RING_IMR(BLT_RING_BASE), 0); 2954 2955 gen5_gt_irq_reset(dev); 2956 2957 I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK); 2958 2959 vlv_display_irq_reset(dev_priv); 2960 } 2961 2962 static void gen8_gt_irq_reset(struct drm_i915_private *dev_priv) 2963 { 2964 GEN8_IRQ_RESET_NDX(GT, 0); 2965 GEN8_IRQ_RESET_NDX(GT, 1); 2966 GEN8_IRQ_RESET_NDX(GT, 2); 2967 GEN8_IRQ_RESET_NDX(GT, 3); 2968 } 2969 2970 static void gen8_irq_reset(struct drm_device *dev) 2971 { 2972 struct drm_i915_private *dev_priv = dev->dev_private; 2973 int pipe; 2974 2975 I915_WRITE(GEN8_MASTER_IRQ, 0); 2976 POSTING_READ(GEN8_MASTER_IRQ); 2977 2978 gen8_gt_irq_reset(dev_priv); 2979 2980 for_each_pipe(dev_priv, pipe) 2981 if (intel_display_power_is_enabled(dev_priv, 2982 POWER_DOMAIN_PIPE(pipe))) 2983 GEN8_IRQ_RESET_NDX(DE_PIPE, pipe); 2984 2985 GEN5_IRQ_RESET(GEN8_DE_PORT_); 2986 GEN5_IRQ_RESET(GEN8_DE_MISC_); 2987 GEN5_IRQ_RESET(GEN8_PCU_); 2988 2989 if (HAS_PCH_SPLIT(dev)) 2990 ibx_irq_reset(dev); 2991 } 2992 2993 void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv, 2994 unsigned int pipe_mask) 2995 { 2996 uint32_t extra_ier = GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN; 2997 2998 spin_lock_irq(&dev_priv->irq_lock); 2999 if (pipe_mask & 1 << PIPE_A) 3000 GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_A, 3001 dev_priv->de_irq_mask[PIPE_A], 3002 ~dev_priv->de_irq_mask[PIPE_A] | extra_ier); 3003 if (pipe_mask & 1 << PIPE_B) 3004 GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_B, 3005 dev_priv->de_irq_mask[PIPE_B], 3006 ~dev_priv->de_irq_mask[PIPE_B] | extra_ier); 3007 if (pipe_mask & 1 << PIPE_C) 3008 GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_C, 3009 dev_priv->de_irq_mask[PIPE_C], 3010 ~dev_priv->de_irq_mask[PIPE_C] | extra_ier); 3011 spin_unlock_irq(&dev_priv->irq_lock); 3012 } 3013 3014 static void cherryview_irq_preinstall(struct drm_device *dev) 3015 { 3016 struct drm_i915_private *dev_priv = dev->dev_private; 3017 3018 I915_WRITE(GEN8_MASTER_IRQ, 0); 3019 POSTING_READ(GEN8_MASTER_IRQ); 3020 3021 gen8_gt_irq_reset(dev_priv); 3022 3023 GEN5_IRQ_RESET(GEN8_PCU_); 3024 3025 I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK_CHV); 3026 3027 vlv_display_irq_reset(dev_priv); 3028 } 3029 3030 static void ibx_hpd_irq_setup(struct drm_device *dev) 3031 { 3032 struct drm_i915_private *dev_priv = dev->dev_private; 3033 struct intel_encoder *intel_encoder; 3034 u32 hotplug_irqs, hotplug, enabled_irqs = 0; 3035 3036 if (HAS_PCH_IBX(dev)) { 3037 hotplug_irqs = SDE_HOTPLUG_MASK; 3038 for_each_intel_encoder(dev, intel_encoder) 3039 if (dev_priv->hotplug.stats[intel_encoder->hpd_pin].state == HPD_ENABLED) 3040 enabled_irqs |= hpd_ibx[intel_encoder->hpd_pin]; 3041 } else if (HAS_PCH_SPT(dev)) { 3042 hotplug_irqs = SDE_HOTPLUG_MASK_SPT; 3043 for_each_intel_encoder(dev, intel_encoder) 3044 if (dev_priv->hotplug.stats[intel_encoder->hpd_pin].state == HPD_ENABLED) 3045 enabled_irqs |= hpd_spt[intel_encoder->hpd_pin]; 3046 } else { 3047 hotplug_irqs = SDE_HOTPLUG_MASK_CPT; 3048 for_each_intel_encoder(dev, intel_encoder) 3049 if (dev_priv->hotplug.stats[intel_encoder->hpd_pin].state == HPD_ENABLED) 3050 enabled_irqs |= hpd_cpt[intel_encoder->hpd_pin]; 3051 } 3052 3053 ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs); 3054 3055 /* 3056 * Enable digital hotplug on the PCH, and configure the DP short pulse 3057 * duration to 2ms (which is the minimum in the Display Port spec) 3058 * 3059 * This register is the same on all known PCH chips. 3060 */ 3061 hotplug = I915_READ(PCH_PORT_HOTPLUG); 3062 hotplug &= ~(PORTD_PULSE_DURATION_MASK|PORTC_PULSE_DURATION_MASK|PORTB_PULSE_DURATION_MASK); 3063 hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms; 3064 hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms; 3065 hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms; 3066 I915_WRITE(PCH_PORT_HOTPLUG, hotplug); 3067 3068 /* enable SPT PORTE hot plug */ 3069 if (HAS_PCH_SPT(dev)) { 3070 hotplug = I915_READ(PCH_PORT_HOTPLUG2); 3071 hotplug |= PORTE_HOTPLUG_ENABLE; 3072 I915_WRITE(PCH_PORT_HOTPLUG2, hotplug); 3073 } 3074 } 3075 3076 static void bxt_hpd_irq_setup(struct drm_device *dev) 3077 { 3078 struct drm_i915_private *dev_priv = dev->dev_private; 3079 struct intel_encoder *intel_encoder; 3080 u32 hotplug_port = 0; 3081 u32 hotplug_ctrl; 3082 3083 /* Now, enable HPD */ 3084 for_each_intel_encoder(dev, intel_encoder) { 3085 if (dev_priv->hotplug.stats[intel_encoder->hpd_pin].state 3086 == HPD_ENABLED) 3087 hotplug_port |= hpd_bxt[intel_encoder->hpd_pin]; 3088 } 3089 3090 /* Mask all HPD control bits */ 3091 hotplug_ctrl = I915_READ(BXT_HOTPLUG_CTL) & ~BXT_HOTPLUG_CTL_MASK; 3092 3093 /* Enable requested port in hotplug control */ 3094 /* TODO: implement (short) HPD support on port A */ 3095 WARN_ON_ONCE(hotplug_port & BXT_DE_PORT_HP_DDIA); 3096 if (hotplug_port & BXT_DE_PORT_HP_DDIB) 3097 hotplug_ctrl |= BXT_DDIB_HPD_ENABLE; 3098 if (hotplug_port & BXT_DE_PORT_HP_DDIC) 3099 hotplug_ctrl |= BXT_DDIC_HPD_ENABLE; 3100 I915_WRITE(BXT_HOTPLUG_CTL, hotplug_ctrl); 3101 3102 /* Unmask DDI hotplug in IMR */ 3103 hotplug_ctrl = I915_READ(GEN8_DE_PORT_IMR) & ~hotplug_port; 3104 I915_WRITE(GEN8_DE_PORT_IMR, hotplug_ctrl); 3105 3106 /* Enable DDI hotplug in IER */ 3107 hotplug_ctrl = I915_READ(GEN8_DE_PORT_IER) | hotplug_port; 3108 I915_WRITE(GEN8_DE_PORT_IER, hotplug_ctrl); 3109 POSTING_READ(GEN8_DE_PORT_IER); 3110 } 3111 3112 static void ibx_irq_postinstall(struct drm_device *dev) 3113 { 3114 struct drm_i915_private *dev_priv = dev->dev_private; 3115 u32 mask; 3116 3117 if (HAS_PCH_NOP(dev)) 3118 return; 3119 3120 if (HAS_PCH_IBX(dev)) 3121 mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON; 3122 else 3123 mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT; 3124 3125 GEN5_ASSERT_IIR_IS_ZERO(SDEIIR); 3126 I915_WRITE(SDEIMR, ~mask); 3127 } 3128 3129 static void gen5_gt_irq_postinstall(struct drm_device *dev) 3130 { 3131 struct drm_i915_private *dev_priv = dev->dev_private; 3132 u32 pm_irqs, gt_irqs; 3133 3134 pm_irqs = gt_irqs = 0; 3135 3136 dev_priv->gt_irq_mask = ~0; 3137 if (HAS_L3_DPF(dev)) { 3138 /* L3 parity interrupt is always unmasked. */ 3139 dev_priv->gt_irq_mask = ~GT_PARITY_ERROR(dev); 3140 gt_irqs |= GT_PARITY_ERROR(dev); 3141 } 3142 3143 gt_irqs |= GT_RENDER_USER_INTERRUPT; 3144 if (IS_GEN5(dev)) { 3145 gt_irqs |= GT_RENDER_PIPECTL_NOTIFY_INTERRUPT | 3146 ILK_BSD_USER_INTERRUPT; 3147 } else { 3148 gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT; 3149 } 3150 3151 GEN5_IRQ_INIT(GT, dev_priv->gt_irq_mask, gt_irqs); 3152 3153 if (INTEL_INFO(dev)->gen >= 6) { 3154 /* 3155 * RPS interrupts will get enabled/disabled on demand when RPS 3156 * itself is enabled/disabled. 3157 */ 3158 if (HAS_VEBOX(dev)) 3159 pm_irqs |= PM_VEBOX_USER_INTERRUPT; 3160 3161 dev_priv->pm_irq_mask = 0xffffffff; 3162 GEN5_IRQ_INIT(GEN6_PM, dev_priv->pm_irq_mask, pm_irqs); 3163 } 3164 } 3165 3166 static int ironlake_irq_postinstall(struct drm_device *dev) 3167 { 3168 struct drm_i915_private *dev_priv = dev->dev_private; 3169 u32 display_mask, extra_mask; 3170 3171 if (INTEL_INFO(dev)->gen >= 7) { 3172 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB | 3173 DE_PCH_EVENT_IVB | DE_PLANEC_FLIP_DONE_IVB | 3174 DE_PLANEB_FLIP_DONE_IVB | 3175 DE_PLANEA_FLIP_DONE_IVB | DE_AUX_CHANNEL_A_IVB); 3176 extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB | 3177 DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB); 3178 } else { 3179 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT | 3180 DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE | 3181 DE_AUX_CHANNEL_A | 3182 DE_PIPEB_CRC_DONE | DE_PIPEA_CRC_DONE | 3183 DE_POISON); 3184 extra_mask = DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT | 3185 DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN; 3186 } 3187 3188 dev_priv->irq_mask = ~display_mask; 3189 3190 I915_WRITE(HWSTAM, 0xeffe); 3191 3192 ibx_irq_pre_postinstall(dev); 3193 3194 GEN5_IRQ_INIT(DE, dev_priv->irq_mask, display_mask | extra_mask); 3195 3196 gen5_gt_irq_postinstall(dev); 3197 3198 ibx_irq_postinstall(dev); 3199 3200 if (IS_IRONLAKE_M(dev)) { 3201 /* Enable PCU event interrupts 3202 * 3203 * spinlocking not required here for correctness since interrupt 3204 * setup is guaranteed to run in single-threaded context. But we 3205 * need it to make the assert_spin_locked happy. */ 3206 spin_lock_irq(&dev_priv->irq_lock); 3207 ironlake_enable_display_irq(dev_priv, DE_PCU_EVENT); 3208 spin_unlock_irq(&dev_priv->irq_lock); 3209 } 3210 3211 return 0; 3212 } 3213 3214 static void valleyview_display_irqs_install(struct drm_i915_private *dev_priv) 3215 { 3216 u32 pipestat_mask; 3217 u32 iir_mask; 3218 enum pipe pipe; 3219 3220 pipestat_mask = PIPESTAT_INT_STATUS_MASK | 3221 PIPE_FIFO_UNDERRUN_STATUS; 3222 3223 for_each_pipe(dev_priv, pipe) 3224 I915_WRITE(PIPESTAT(pipe), pipestat_mask); 3225 POSTING_READ(PIPESTAT(PIPE_A)); 3226 3227 pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV | 3228 PIPE_CRC_DONE_INTERRUPT_STATUS; 3229 3230 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS); 3231 for_each_pipe(dev_priv, pipe) 3232 i915_enable_pipestat(dev_priv, pipe, pipestat_mask); 3233 3234 iir_mask = I915_DISPLAY_PORT_INTERRUPT | 3235 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | 3236 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT; 3237 if (IS_CHERRYVIEW(dev_priv)) 3238 iir_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT; 3239 dev_priv->irq_mask &= ~iir_mask; 3240 3241 I915_WRITE(VLV_IIR, iir_mask); 3242 I915_WRITE(VLV_IIR, iir_mask); 3243 I915_WRITE(VLV_IER, ~dev_priv->irq_mask); 3244 I915_WRITE(VLV_IMR, dev_priv->irq_mask); 3245 POSTING_READ(VLV_IMR); 3246 } 3247 3248 static void valleyview_display_irqs_uninstall(struct drm_i915_private *dev_priv) 3249 { 3250 u32 pipestat_mask; 3251 u32 iir_mask; 3252 enum pipe pipe; 3253 3254 iir_mask = I915_DISPLAY_PORT_INTERRUPT | 3255 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | 3256 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT; 3257 if (IS_CHERRYVIEW(dev_priv)) 3258 iir_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT; 3259 3260 dev_priv->irq_mask |= iir_mask; 3261 I915_WRITE(VLV_IMR, dev_priv->irq_mask); 3262 I915_WRITE(VLV_IER, ~dev_priv->irq_mask); 3263 I915_WRITE(VLV_IIR, iir_mask); 3264 I915_WRITE(VLV_IIR, iir_mask); 3265 POSTING_READ(VLV_IIR); 3266 3267 pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV | 3268 PIPE_CRC_DONE_INTERRUPT_STATUS; 3269 3270 i915_disable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS); 3271 for_each_pipe(dev_priv, pipe) 3272 i915_disable_pipestat(dev_priv, pipe, pipestat_mask); 3273 3274 pipestat_mask = PIPESTAT_INT_STATUS_MASK | 3275 PIPE_FIFO_UNDERRUN_STATUS; 3276 3277 for_each_pipe(dev_priv, pipe) 3278 I915_WRITE(PIPESTAT(pipe), pipestat_mask); 3279 POSTING_READ(PIPESTAT(PIPE_A)); 3280 } 3281 3282 void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv) 3283 { 3284 assert_spin_locked(&dev_priv->irq_lock); 3285 3286 if (dev_priv->display_irqs_enabled) 3287 return; 3288 3289 dev_priv->display_irqs_enabled = true; 3290 3291 if (intel_irqs_enabled(dev_priv)) 3292 valleyview_display_irqs_install(dev_priv); 3293 } 3294 3295 void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv) 3296 { 3297 assert_spin_locked(&dev_priv->irq_lock); 3298 3299 if (!dev_priv->display_irqs_enabled) 3300 return; 3301 3302 dev_priv->display_irqs_enabled = false; 3303 3304 if (intel_irqs_enabled(dev_priv)) 3305 valleyview_display_irqs_uninstall(dev_priv); 3306 } 3307 3308 static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv) 3309 { 3310 dev_priv->irq_mask = ~0; 3311 3312 I915_WRITE(PORT_HOTPLUG_EN, 0); 3313 POSTING_READ(PORT_HOTPLUG_EN); 3314 3315 I915_WRITE(VLV_IIR, 0xffffffff); 3316 I915_WRITE(VLV_IIR, 0xffffffff); 3317 I915_WRITE(VLV_IER, ~dev_priv->irq_mask); 3318 I915_WRITE(VLV_IMR, dev_priv->irq_mask); 3319 POSTING_READ(VLV_IMR); 3320 3321 /* Interrupt setup is already guaranteed to be single-threaded, this is 3322 * just to make the assert_spin_locked check happy. */ 3323 spin_lock_irq(&dev_priv->irq_lock); 3324 if (dev_priv->display_irqs_enabled) 3325 valleyview_display_irqs_install(dev_priv); 3326 spin_unlock_irq(&dev_priv->irq_lock); 3327 } 3328 3329 static int valleyview_irq_postinstall(struct drm_device *dev) 3330 { 3331 struct drm_i915_private *dev_priv = dev->dev_private; 3332 3333 vlv_display_irq_postinstall(dev_priv); 3334 3335 gen5_gt_irq_postinstall(dev); 3336 3337 /* ack & enable invalid PTE error interrupts */ 3338 #if 0 /* FIXME: add support to irq handler for checking these bits */ 3339 I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK); 3340 I915_WRITE(DPINVGTT, DPINVGTT_EN_MASK); 3341 #endif 3342 3343 I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE); 3344 3345 return 0; 3346 } 3347 3348 static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv) 3349 { 3350 /* These are interrupts we'll toggle with the ring mask register */ 3351 uint32_t gt_interrupts[] = { 3352 GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT | 3353 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT | 3354 GT_RENDER_L3_PARITY_ERROR_INTERRUPT | 3355 GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT | 3356 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT, 3357 GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT | 3358 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT | 3359 GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT | 3360 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT, 3361 0, 3362 GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT | 3363 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT 3364 }; 3365 3366 dev_priv->pm_irq_mask = 0xffffffff; 3367 GEN8_IRQ_INIT_NDX(GT, 0, ~gt_interrupts[0], gt_interrupts[0]); 3368 GEN8_IRQ_INIT_NDX(GT, 1, ~gt_interrupts[1], gt_interrupts[1]); 3369 /* 3370 * RPS interrupts will get enabled/disabled on demand when RPS itself 3371 * is enabled/disabled. 3372 */ 3373 GEN8_IRQ_INIT_NDX(GT, 2, dev_priv->pm_irq_mask, 0); 3374 GEN8_IRQ_INIT_NDX(GT, 3, ~gt_interrupts[3], gt_interrupts[3]); 3375 } 3376 3377 static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv) 3378 { 3379 uint32_t de_pipe_masked = GEN8_PIPE_CDCLK_CRC_DONE; 3380 uint32_t de_pipe_enables; 3381 int pipe; 3382 u32 de_port_en = GEN8_AUX_CHANNEL_A; 3383 3384 if (IS_GEN9(dev_priv)) { 3385 de_pipe_masked |= GEN9_PIPE_PLANE1_FLIP_DONE | 3386 GEN9_DE_PIPE_IRQ_FAULT_ERRORS; 3387 de_port_en |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C | 3388 GEN9_AUX_CHANNEL_D; 3389 3390 if (IS_BROXTON(dev_priv)) 3391 de_port_en |= BXT_DE_PORT_GMBUS; 3392 } else 3393 de_pipe_masked |= GEN8_PIPE_PRIMARY_FLIP_DONE | 3394 GEN8_DE_PIPE_IRQ_FAULT_ERRORS; 3395 3396 de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK | 3397 GEN8_PIPE_FIFO_UNDERRUN; 3398 3399 dev_priv->de_irq_mask[PIPE_A] = ~de_pipe_masked; 3400 dev_priv->de_irq_mask[PIPE_B] = ~de_pipe_masked; 3401 dev_priv->de_irq_mask[PIPE_C] = ~de_pipe_masked; 3402 3403 for_each_pipe(dev_priv, pipe) 3404 if (intel_display_power_is_enabled(dev_priv, 3405 POWER_DOMAIN_PIPE(pipe))) 3406 GEN8_IRQ_INIT_NDX(DE_PIPE, pipe, 3407 dev_priv->de_irq_mask[pipe], 3408 de_pipe_enables); 3409 3410 GEN5_IRQ_INIT(GEN8_DE_PORT_, ~de_port_en, de_port_en); 3411 } 3412 3413 static int gen8_irq_postinstall(struct drm_device *dev) 3414 { 3415 struct drm_i915_private *dev_priv = dev->dev_private; 3416 3417 if (HAS_PCH_SPLIT(dev)) 3418 ibx_irq_pre_postinstall(dev); 3419 3420 gen8_gt_irq_postinstall(dev_priv); 3421 gen8_de_irq_postinstall(dev_priv); 3422 3423 if (HAS_PCH_SPLIT(dev)) 3424 ibx_irq_postinstall(dev); 3425 3426 I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL); 3427 POSTING_READ(GEN8_MASTER_IRQ); 3428 3429 return 0; 3430 } 3431 3432 static int cherryview_irq_postinstall(struct drm_device *dev) 3433 { 3434 struct drm_i915_private *dev_priv = dev->dev_private; 3435 3436 vlv_display_irq_postinstall(dev_priv); 3437 3438 gen8_gt_irq_postinstall(dev_priv); 3439 3440 I915_WRITE(GEN8_MASTER_IRQ, MASTER_INTERRUPT_ENABLE); 3441 POSTING_READ(GEN8_MASTER_IRQ); 3442 3443 return 0; 3444 } 3445 3446 static void gen8_irq_uninstall(struct drm_device *dev) 3447 { 3448 struct drm_i915_private *dev_priv = dev->dev_private; 3449 3450 if (!dev_priv) 3451 return; 3452 3453 gen8_irq_reset(dev); 3454 } 3455 3456 static void vlv_display_irq_uninstall(struct drm_i915_private *dev_priv) 3457 { 3458 /* Interrupt setup is already guaranteed to be single-threaded, this is 3459 * just to make the assert_spin_locked check happy. */ 3460 spin_lock_irq(&dev_priv->irq_lock); 3461 if (dev_priv->display_irqs_enabled) 3462 valleyview_display_irqs_uninstall(dev_priv); 3463 spin_unlock_irq(&dev_priv->irq_lock); 3464 3465 vlv_display_irq_reset(dev_priv); 3466 3467 dev_priv->irq_mask = ~0; 3468 } 3469 3470 static void valleyview_irq_uninstall(struct drm_device *dev) 3471 { 3472 struct drm_i915_private *dev_priv = dev->dev_private; 3473 3474 if (!dev_priv) 3475 return; 3476 3477 I915_WRITE(VLV_MASTER_IER, 0); 3478 3479 gen5_gt_irq_reset(dev); 3480 3481 I915_WRITE(HWSTAM, 0xffffffff); 3482 3483 vlv_display_irq_uninstall(dev_priv); 3484 } 3485 3486 static void cherryview_irq_uninstall(struct drm_device *dev) 3487 { 3488 struct drm_i915_private *dev_priv = dev->dev_private; 3489 3490 if (!dev_priv) 3491 return; 3492 3493 I915_WRITE(GEN8_MASTER_IRQ, 0); 3494 POSTING_READ(GEN8_MASTER_IRQ); 3495 3496 gen8_gt_irq_reset(dev_priv); 3497 3498 GEN5_IRQ_RESET(GEN8_PCU_); 3499 3500 vlv_display_irq_uninstall(dev_priv); 3501 } 3502 3503 static void ironlake_irq_uninstall(struct drm_device *dev) 3504 { 3505 struct drm_i915_private *dev_priv = dev->dev_private; 3506 3507 if (!dev_priv) 3508 return; 3509 3510 ironlake_irq_reset(dev); 3511 } 3512 3513 static void i8xx_irq_preinstall(struct drm_device * dev) 3514 { 3515 struct drm_i915_private *dev_priv = dev->dev_private; 3516 int pipe; 3517 3518 for_each_pipe(dev_priv, pipe) 3519 I915_WRITE(PIPESTAT(pipe), 0); 3520 I915_WRITE16(IMR, 0xffff); 3521 I915_WRITE16(IER, 0x0); 3522 POSTING_READ16(IER); 3523 } 3524 3525 static int i8xx_irq_postinstall(struct drm_device *dev) 3526 { 3527 struct drm_i915_private *dev_priv = dev->dev_private; 3528 3529 I915_WRITE16(EMR, 3530 ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH)); 3531 3532 /* Unmask the interrupts that we always want on. */ 3533 dev_priv->irq_mask = 3534 ~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | 3535 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT | 3536 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT | 3537 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT); 3538 I915_WRITE16(IMR, dev_priv->irq_mask); 3539 3540 I915_WRITE16(IER, 3541 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | 3542 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT | 3543 I915_USER_INTERRUPT); 3544 POSTING_READ16(IER); 3545 3546 /* Interrupt setup is already guaranteed to be single-threaded, this is 3547 * just to make the assert_spin_locked check happy. */ 3548 spin_lock_irq(&dev_priv->irq_lock); 3549 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS); 3550 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS); 3551 spin_unlock_irq(&dev_priv->irq_lock); 3552 3553 return 0; 3554 } 3555 3556 /* 3557 * Returns true when a page flip has completed. 3558 */ 3559 static bool i8xx_handle_vblank(struct drm_device *dev, 3560 int plane, int pipe, u32 iir) 3561 { 3562 struct drm_i915_private *dev_priv = dev->dev_private; 3563 u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane); 3564 3565 if (!intel_pipe_handle_vblank(dev, pipe)) 3566 return false; 3567 3568 if ((iir & flip_pending) == 0) 3569 goto check_page_flip; 3570 3571 /* We detect FlipDone by looking for the change in PendingFlip from '1' 3572 * to '0' on the following vblank, i.e. IIR has the Pendingflip 3573 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence 3574 * the flip is completed (no longer pending). Since this doesn't raise 3575 * an interrupt per se, we watch for the change at vblank. 3576 */ 3577 if (I915_READ16(ISR) & flip_pending) 3578 goto check_page_flip; 3579 3580 intel_prepare_page_flip(dev, plane); 3581 intel_finish_page_flip(dev, pipe); 3582 return true; 3583 3584 check_page_flip: 3585 intel_check_page_flip(dev, pipe); 3586 return false; 3587 } 3588 3589 static irqreturn_t i8xx_irq_handler(int irq, void *arg) 3590 { 3591 struct drm_device *dev = arg; 3592 struct drm_i915_private *dev_priv = dev->dev_private; 3593 u16 iir, new_iir; 3594 u32 pipe_stats[2]; 3595 int pipe; 3596 u16 flip_mask = 3597 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT | 3598 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT; 3599 3600 if (!intel_irqs_enabled(dev_priv)) 3601 return IRQ_NONE; 3602 3603 iir = I915_READ16(IIR); 3604 if (iir == 0) 3605 return IRQ_NONE; 3606 3607 while (iir & ~flip_mask) { 3608 /* Can't rely on pipestat interrupt bit in iir as it might 3609 * have been cleared after the pipestat interrupt was received. 3610 * It doesn't set the bit in iir again, but it still produces 3611 * interrupts (for non-MSI). 3612 */ 3613 spin_lock(&dev_priv->irq_lock); 3614 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT) 3615 DRM_DEBUG("Command parser error, iir 0x%08x\n", iir); 3616 3617 for_each_pipe(dev_priv, pipe) { 3618 int reg = PIPESTAT(pipe); 3619 pipe_stats[pipe] = I915_READ(reg); 3620 3621 /* 3622 * Clear the PIPE*STAT regs before the IIR 3623 */ 3624 if (pipe_stats[pipe] & 0x8000ffff) 3625 I915_WRITE(reg, pipe_stats[pipe]); 3626 } 3627 spin_unlock(&dev_priv->irq_lock); 3628 3629 I915_WRITE16(IIR, iir & ~flip_mask); 3630 new_iir = I915_READ16(IIR); /* Flush posted writes */ 3631 3632 if (iir & I915_USER_INTERRUPT) 3633 notify_ring(&dev_priv->ring[RCS]); 3634 3635 for_each_pipe(dev_priv, pipe) { 3636 int plane = pipe; 3637 if (HAS_FBC(dev)) 3638 plane = !plane; 3639 3640 if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS && 3641 i8xx_handle_vblank(dev, plane, pipe, iir)) 3642 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane); 3643 3644 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS) 3645 i9xx_pipe_crc_irq_handler(dev, pipe); 3646 3647 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS) 3648 intel_cpu_fifo_underrun_irq_handler(dev_priv, 3649 pipe); 3650 } 3651 3652 iir = new_iir; 3653 } 3654 3655 return IRQ_HANDLED; 3656 } 3657 3658 static void i8xx_irq_uninstall(struct drm_device * dev) 3659 { 3660 struct drm_i915_private *dev_priv = dev->dev_private; 3661 int pipe; 3662 3663 for_each_pipe(dev_priv, pipe) { 3664 /* Clear enable bits; then clear status bits */ 3665 I915_WRITE(PIPESTAT(pipe), 0); 3666 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe))); 3667 } 3668 I915_WRITE16(IMR, 0xffff); 3669 I915_WRITE16(IER, 0x0); 3670 I915_WRITE16(IIR, I915_READ16(IIR)); 3671 } 3672 3673 static void i915_irq_preinstall(struct drm_device * dev) 3674 { 3675 struct drm_i915_private *dev_priv = dev->dev_private; 3676 int pipe; 3677 3678 if (I915_HAS_HOTPLUG(dev)) { 3679 I915_WRITE(PORT_HOTPLUG_EN, 0); 3680 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); 3681 } 3682 3683 I915_WRITE16(HWSTAM, 0xeffe); 3684 for_each_pipe(dev_priv, pipe) 3685 I915_WRITE(PIPESTAT(pipe), 0); 3686 I915_WRITE(IMR, 0xffffffff); 3687 I915_WRITE(IER, 0x0); 3688 POSTING_READ(IER); 3689 } 3690 3691 static int i915_irq_postinstall(struct drm_device *dev) 3692 { 3693 struct drm_i915_private *dev_priv = dev->dev_private; 3694 u32 enable_mask; 3695 3696 I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH)); 3697 3698 /* Unmask the interrupts that we always want on. */ 3699 dev_priv->irq_mask = 3700 ~(I915_ASLE_INTERRUPT | 3701 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | 3702 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT | 3703 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT | 3704 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT); 3705 3706 enable_mask = 3707 I915_ASLE_INTERRUPT | 3708 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | 3709 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT | 3710 I915_USER_INTERRUPT; 3711 3712 if (I915_HAS_HOTPLUG(dev)) { 3713 I915_WRITE(PORT_HOTPLUG_EN, 0); 3714 POSTING_READ(PORT_HOTPLUG_EN); 3715 3716 /* Enable in IER... */ 3717 enable_mask |= I915_DISPLAY_PORT_INTERRUPT; 3718 /* and unmask in IMR */ 3719 dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT; 3720 } 3721 3722 I915_WRITE(IMR, dev_priv->irq_mask); 3723 I915_WRITE(IER, enable_mask); 3724 POSTING_READ(IER); 3725 3726 i915_enable_asle_pipestat(dev); 3727 3728 /* Interrupt setup is already guaranteed to be single-threaded, this is 3729 * just to make the assert_spin_locked check happy. */ 3730 spin_lock_irq(&dev_priv->irq_lock); 3731 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS); 3732 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS); 3733 spin_unlock_irq(&dev_priv->irq_lock); 3734 3735 return 0; 3736 } 3737 3738 /* 3739 * Returns true when a page flip has completed. 3740 */ 3741 static bool i915_handle_vblank(struct drm_device *dev, 3742 int plane, int pipe, u32 iir) 3743 { 3744 struct drm_i915_private *dev_priv = dev->dev_private; 3745 u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane); 3746 3747 if (!intel_pipe_handle_vblank(dev, pipe)) 3748 return false; 3749 3750 if ((iir & flip_pending) == 0) 3751 goto check_page_flip; 3752 3753 /* We detect FlipDone by looking for the change in PendingFlip from '1' 3754 * to '0' on the following vblank, i.e. IIR has the Pendingflip 3755 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence 3756 * the flip is completed (no longer pending). Since this doesn't raise 3757 * an interrupt per se, we watch for the change at vblank. 3758 */ 3759 if (I915_READ(ISR) & flip_pending) 3760 goto check_page_flip; 3761 3762 intel_prepare_page_flip(dev, plane); 3763 intel_finish_page_flip(dev, pipe); 3764 return true; 3765 3766 check_page_flip: 3767 intel_check_page_flip(dev, pipe); 3768 return false; 3769 } 3770 3771 static irqreturn_t i915_irq_handler(int irq, void *arg) 3772 { 3773 struct drm_device *dev = arg; 3774 struct drm_i915_private *dev_priv = dev->dev_private; 3775 u32 iir, new_iir, pipe_stats[I915_MAX_PIPES]; 3776 u32 flip_mask = 3777 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT | 3778 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT; 3779 int pipe, ret = IRQ_NONE; 3780 3781 if (!intel_irqs_enabled(dev_priv)) 3782 return IRQ_NONE; 3783 3784 iir = I915_READ(IIR); 3785 do { 3786 bool irq_received = (iir & ~flip_mask) != 0; 3787 bool blc_event = false; 3788 3789 /* Can't rely on pipestat interrupt bit in iir as it might 3790 * have been cleared after the pipestat interrupt was received. 3791 * It doesn't set the bit in iir again, but it still produces 3792 * interrupts (for non-MSI). 3793 */ 3794 spin_lock(&dev_priv->irq_lock); 3795 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT) 3796 DRM_DEBUG("Command parser error, iir 0x%08x\n", iir); 3797 3798 for_each_pipe(dev_priv, pipe) { 3799 int reg = PIPESTAT(pipe); 3800 pipe_stats[pipe] = I915_READ(reg); 3801 3802 /* Clear the PIPE*STAT regs before the IIR */ 3803 if (pipe_stats[pipe] & 0x8000ffff) { 3804 I915_WRITE(reg, pipe_stats[pipe]); 3805 irq_received = true; 3806 } 3807 } 3808 spin_unlock(&dev_priv->irq_lock); 3809 3810 if (!irq_received) 3811 break; 3812 3813 /* Consume port. Then clear IIR or we'll miss events */ 3814 if (I915_HAS_HOTPLUG(dev) && 3815 iir & I915_DISPLAY_PORT_INTERRUPT) 3816 i9xx_hpd_irq_handler(dev); 3817 3818 I915_WRITE(IIR, iir & ~flip_mask); 3819 new_iir = I915_READ(IIR); /* Flush posted writes */ 3820 3821 if (iir & I915_USER_INTERRUPT) 3822 notify_ring(&dev_priv->ring[RCS]); 3823 3824 for_each_pipe(dev_priv, pipe) { 3825 int plane = pipe; 3826 if (HAS_FBC(dev)) 3827 plane = !plane; 3828 3829 if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS && 3830 i915_handle_vblank(dev, plane, pipe, iir)) 3831 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane); 3832 3833 if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS) 3834 blc_event = true; 3835 3836 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS) 3837 i9xx_pipe_crc_irq_handler(dev, pipe); 3838 3839 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS) 3840 intel_cpu_fifo_underrun_irq_handler(dev_priv, 3841 pipe); 3842 } 3843 3844 if (blc_event || (iir & I915_ASLE_INTERRUPT)) 3845 intel_opregion_asle_intr(dev); 3846 3847 /* With MSI, interrupts are only generated when iir 3848 * transitions from zero to nonzero. If another bit got 3849 * set while we were handling the existing iir bits, then 3850 * we would never get another interrupt. 3851 * 3852 * This is fine on non-MSI as well, as if we hit this path 3853 * we avoid exiting the interrupt handler only to generate 3854 * another one. 3855 * 3856 * Note that for MSI this could cause a stray interrupt report 3857 * if an interrupt landed in the time between writing IIR and 3858 * the posting read. This should be rare enough to never 3859 * trigger the 99% of 100,000 interrupts test for disabling 3860 * stray interrupts. 3861 */ 3862 ret = IRQ_HANDLED; 3863 iir = new_iir; 3864 } while (iir & ~flip_mask); 3865 3866 return ret; 3867 } 3868 3869 static void i915_irq_uninstall(struct drm_device * dev) 3870 { 3871 struct drm_i915_private *dev_priv = dev->dev_private; 3872 int pipe; 3873 3874 if (I915_HAS_HOTPLUG(dev)) { 3875 I915_WRITE(PORT_HOTPLUG_EN, 0); 3876 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); 3877 } 3878 3879 I915_WRITE16(HWSTAM, 0xffff); 3880 for_each_pipe(dev_priv, pipe) { 3881 /* Clear enable bits; then clear status bits */ 3882 I915_WRITE(PIPESTAT(pipe), 0); 3883 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe))); 3884 } 3885 I915_WRITE(IMR, 0xffffffff); 3886 I915_WRITE(IER, 0x0); 3887 3888 I915_WRITE(IIR, I915_READ(IIR)); 3889 } 3890 3891 static void i965_irq_preinstall(struct drm_device * dev) 3892 { 3893 struct drm_i915_private *dev_priv = dev->dev_private; 3894 int pipe; 3895 3896 I915_WRITE(PORT_HOTPLUG_EN, 0); 3897 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); 3898 3899 I915_WRITE(HWSTAM, 0xeffe); 3900 for_each_pipe(dev_priv, pipe) 3901 I915_WRITE(PIPESTAT(pipe), 0); 3902 I915_WRITE(IMR, 0xffffffff); 3903 I915_WRITE(IER, 0x0); 3904 POSTING_READ(IER); 3905 } 3906 3907 static int i965_irq_postinstall(struct drm_device *dev) 3908 { 3909 struct drm_i915_private *dev_priv = dev->dev_private; 3910 u32 enable_mask; 3911 u32 error_mask; 3912 3913 /* Unmask the interrupts that we always want on. */ 3914 dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT | 3915 I915_DISPLAY_PORT_INTERRUPT | 3916 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | 3917 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT | 3918 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT | 3919 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT | 3920 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT); 3921 3922 enable_mask = ~dev_priv->irq_mask; 3923 enable_mask &= ~(I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT | 3924 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT); 3925 enable_mask |= I915_USER_INTERRUPT; 3926 3927 if (IS_G4X(dev)) 3928 enable_mask |= I915_BSD_USER_INTERRUPT; 3929 3930 /* Interrupt setup is already guaranteed to be single-threaded, this is 3931 * just to make the assert_spin_locked check happy. */ 3932 spin_lock_irq(&dev_priv->irq_lock); 3933 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS); 3934 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS); 3935 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS); 3936 spin_unlock_irq(&dev_priv->irq_lock); 3937 3938 /* 3939 * Enable some error detection, note the instruction error mask 3940 * bit is reserved, so we leave it masked. 3941 */ 3942 if (IS_G4X(dev)) { 3943 error_mask = ~(GM45_ERROR_PAGE_TABLE | 3944 GM45_ERROR_MEM_PRIV | 3945 GM45_ERROR_CP_PRIV | 3946 I915_ERROR_MEMORY_REFRESH); 3947 } else { 3948 error_mask = ~(I915_ERROR_PAGE_TABLE | 3949 I915_ERROR_MEMORY_REFRESH); 3950 } 3951 I915_WRITE(EMR, error_mask); 3952 3953 I915_WRITE(IMR, dev_priv->irq_mask); 3954 I915_WRITE(IER, enable_mask); 3955 POSTING_READ(IER); 3956 3957 I915_WRITE(PORT_HOTPLUG_EN, 0); 3958 POSTING_READ(PORT_HOTPLUG_EN); 3959 3960 i915_enable_asle_pipestat(dev); 3961 3962 return 0; 3963 } 3964 3965 static void i915_hpd_irq_setup(struct drm_device *dev) 3966 { 3967 struct drm_i915_private *dev_priv = dev->dev_private; 3968 struct intel_encoder *intel_encoder; 3969 u32 hotplug_en; 3970 3971 assert_spin_locked(&dev_priv->irq_lock); 3972 3973 hotplug_en = I915_READ(PORT_HOTPLUG_EN); 3974 hotplug_en &= ~HOTPLUG_INT_EN_MASK; 3975 /* Note HDMI and DP share hotplug bits */ 3976 /* enable bits are the same for all generations */ 3977 for_each_intel_encoder(dev, intel_encoder) 3978 if (dev_priv->hotplug.stats[intel_encoder->hpd_pin].state == HPD_ENABLED) 3979 hotplug_en |= hpd_mask_i915[intel_encoder->hpd_pin]; 3980 /* Programming the CRT detection parameters tends 3981 to generate a spurious hotplug event about three 3982 seconds later. So just do it once. 3983 */ 3984 if (IS_G4X(dev)) 3985 hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64; 3986 hotplug_en &= ~CRT_HOTPLUG_VOLTAGE_COMPARE_MASK; 3987 hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50; 3988 3989 /* Ignore TV since it's buggy */ 3990 I915_WRITE(PORT_HOTPLUG_EN, hotplug_en); 3991 } 3992 3993 static irqreturn_t i965_irq_handler(int irq, void *arg) 3994 { 3995 struct drm_device *dev = arg; 3996 struct drm_i915_private *dev_priv = dev->dev_private; 3997 u32 iir, new_iir; 3998 u32 pipe_stats[I915_MAX_PIPES]; 3999 int ret = IRQ_NONE, pipe; 4000 u32 flip_mask = 4001 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT | 4002 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT; 4003 4004 if (!intel_irqs_enabled(dev_priv)) 4005 return IRQ_NONE; 4006 4007 iir = I915_READ(IIR); 4008 4009 for (;;) { 4010 bool irq_received = (iir & ~flip_mask) != 0; 4011 bool blc_event = false; 4012 4013 /* Can't rely on pipestat interrupt bit in iir as it might 4014 * have been cleared after the pipestat interrupt was received. 4015 * It doesn't set the bit in iir again, but it still produces 4016 * interrupts (for non-MSI). 4017 */ 4018 spin_lock(&dev_priv->irq_lock); 4019 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT) 4020 DRM_DEBUG("Command parser error, iir 0x%08x\n", iir); 4021 4022 for_each_pipe(dev_priv, pipe) { 4023 int reg = PIPESTAT(pipe); 4024 pipe_stats[pipe] = I915_READ(reg); 4025 4026 /* 4027 * Clear the PIPE*STAT regs before the IIR 4028 */ 4029 if (pipe_stats[pipe] & 0x8000ffff) { 4030 I915_WRITE(reg, pipe_stats[pipe]); 4031 irq_received = true; 4032 } 4033 } 4034 spin_unlock(&dev_priv->irq_lock); 4035 4036 if (!irq_received) 4037 break; 4038 4039 ret = IRQ_HANDLED; 4040 4041 /* Consume port. Then clear IIR or we'll miss events */ 4042 if (iir & I915_DISPLAY_PORT_INTERRUPT) 4043 i9xx_hpd_irq_handler(dev); 4044 4045 I915_WRITE(IIR, iir & ~flip_mask); 4046 new_iir = I915_READ(IIR); /* Flush posted writes */ 4047 4048 if (iir & I915_USER_INTERRUPT) 4049 notify_ring(&dev_priv->ring[RCS]); 4050 if (iir & I915_BSD_USER_INTERRUPT) 4051 notify_ring(&dev_priv->ring[VCS]); 4052 4053 for_each_pipe(dev_priv, pipe) { 4054 if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS && 4055 i915_handle_vblank(dev, pipe, pipe, iir)) 4056 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe); 4057 4058 if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS) 4059 blc_event = true; 4060 4061 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS) 4062 i9xx_pipe_crc_irq_handler(dev, pipe); 4063 4064 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS) 4065 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe); 4066 } 4067 4068 if (blc_event || (iir & I915_ASLE_INTERRUPT)) 4069 intel_opregion_asle_intr(dev); 4070 4071 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS) 4072 gmbus_irq_handler(dev); 4073 4074 /* With MSI, interrupts are only generated when iir 4075 * transitions from zero to nonzero. If another bit got 4076 * set while we were handling the existing iir bits, then 4077 * we would never get another interrupt. 4078 * 4079 * This is fine on non-MSI as well, as if we hit this path 4080 * we avoid exiting the interrupt handler only to generate 4081 * another one. 4082 * 4083 * Note that for MSI this could cause a stray interrupt report 4084 * if an interrupt landed in the time between writing IIR and 4085 * the posting read. This should be rare enough to never 4086 * trigger the 99% of 100,000 interrupts test for disabling 4087 * stray interrupts. 4088 */ 4089 iir = new_iir; 4090 } 4091 4092 return ret; 4093 } 4094 4095 static void i965_irq_uninstall(struct drm_device * dev) 4096 { 4097 struct drm_i915_private *dev_priv = dev->dev_private; 4098 int pipe; 4099 4100 if (!dev_priv) 4101 return; 4102 4103 I915_WRITE(PORT_HOTPLUG_EN, 0); 4104 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); 4105 4106 I915_WRITE(HWSTAM, 0xffffffff); 4107 for_each_pipe(dev_priv, pipe) 4108 I915_WRITE(PIPESTAT(pipe), 0); 4109 I915_WRITE(IMR, 0xffffffff); 4110 I915_WRITE(IER, 0x0); 4111 4112 for_each_pipe(dev_priv, pipe) 4113 I915_WRITE(PIPESTAT(pipe), 4114 I915_READ(PIPESTAT(pipe)) & 0x8000ffff); 4115 I915_WRITE(IIR, I915_READ(IIR)); 4116 } 4117 4118 /** 4119 * intel_irq_init - initializes irq support 4120 * @dev_priv: i915 device instance 4121 * 4122 * This function initializes all the irq support including work items, timers 4123 * and all the vtables. It does not setup the interrupt itself though. 4124 */ 4125 void intel_irq_init(struct drm_i915_private *dev_priv) 4126 { 4127 struct drm_device *dev = dev_priv->dev; 4128 4129 intel_hpd_init_work(dev_priv); 4130 4131 INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work); 4132 INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work); 4133 4134 /* Let's track the enabled rps events */ 4135 if (IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) 4136 /* WaGsvRC0ResidencyMethod:vlv */ 4137 dev_priv->pm_rps_events = GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED; 4138 else 4139 dev_priv->pm_rps_events = GEN6_PM_RPS_EVENTS; 4140 4141 INIT_DELAYED_WORK(&dev_priv->gpu_error.hangcheck_work, 4142 i915_hangcheck_elapsed); 4143 4144 pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY, PM_QOS_DEFAULT_VALUE); 4145 4146 if (IS_GEN2(dev_priv)) { 4147 dev->max_vblank_count = 0; 4148 dev->driver->get_vblank_counter = i8xx_get_vblank_counter; 4149 } else if (IS_G4X(dev_priv) || INTEL_INFO(dev_priv)->gen >= 5) { 4150 dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */ 4151 dev->driver->get_vblank_counter = gm45_get_vblank_counter; 4152 } else { 4153 dev->driver->get_vblank_counter = i915_get_vblank_counter; 4154 dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */ 4155 } 4156 4157 /* 4158 * Opt out of the vblank disable timer on everything except gen2. 4159 * Gen2 doesn't have a hardware frame counter and so depends on 4160 * vblank interrupts to produce sane vblank seuquence numbers. 4161 */ 4162 if (!IS_GEN2(dev_priv)) 4163 dev->vblank_disable_immediate = true; 4164 4165 dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp; 4166 dev->driver->get_scanout_position = i915_get_crtc_scanoutpos; 4167 4168 if (IS_CHERRYVIEW(dev_priv)) { 4169 dev->driver->irq_handler = cherryview_irq_handler; 4170 dev->driver->irq_preinstall = cherryview_irq_preinstall; 4171 dev->driver->irq_postinstall = cherryview_irq_postinstall; 4172 dev->driver->irq_uninstall = cherryview_irq_uninstall; 4173 dev->driver->enable_vblank = valleyview_enable_vblank; 4174 dev->driver->disable_vblank = valleyview_disable_vblank; 4175 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup; 4176 } else if (IS_VALLEYVIEW(dev_priv)) { 4177 dev->driver->irq_handler = valleyview_irq_handler; 4178 dev->driver->irq_preinstall = valleyview_irq_preinstall; 4179 dev->driver->irq_postinstall = valleyview_irq_postinstall; 4180 dev->driver->irq_uninstall = valleyview_irq_uninstall; 4181 dev->driver->enable_vblank = valleyview_enable_vblank; 4182 dev->driver->disable_vblank = valleyview_disable_vblank; 4183 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup; 4184 } else if (INTEL_INFO(dev_priv)->gen >= 8) { 4185 dev->driver->irq_handler = gen8_irq_handler; 4186 dev->driver->irq_preinstall = gen8_irq_reset; 4187 dev->driver->irq_postinstall = gen8_irq_postinstall; 4188 dev->driver->irq_uninstall = gen8_irq_uninstall; 4189 dev->driver->enable_vblank = gen8_enable_vblank; 4190 dev->driver->disable_vblank = gen8_disable_vblank; 4191 if (HAS_PCH_SPLIT(dev)) 4192 dev_priv->display.hpd_irq_setup = ibx_hpd_irq_setup; 4193 else 4194 dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup; 4195 } else if (HAS_PCH_SPLIT(dev)) { 4196 dev->driver->irq_handler = ironlake_irq_handler; 4197 dev->driver->irq_preinstall = ironlake_irq_reset; 4198 dev->driver->irq_postinstall = ironlake_irq_postinstall; 4199 dev->driver->irq_uninstall = ironlake_irq_uninstall; 4200 dev->driver->enable_vblank = ironlake_enable_vblank; 4201 dev->driver->disable_vblank = ironlake_disable_vblank; 4202 dev_priv->display.hpd_irq_setup = ibx_hpd_irq_setup; 4203 } else { 4204 if (INTEL_INFO(dev_priv)->gen == 2) { 4205 dev->driver->irq_preinstall = i8xx_irq_preinstall; 4206 dev->driver->irq_postinstall = i8xx_irq_postinstall; 4207 dev->driver->irq_handler = i8xx_irq_handler; 4208 dev->driver->irq_uninstall = i8xx_irq_uninstall; 4209 } else if (INTEL_INFO(dev_priv)->gen == 3) { 4210 dev->driver->irq_preinstall = i915_irq_preinstall; 4211 dev->driver->irq_postinstall = i915_irq_postinstall; 4212 dev->driver->irq_uninstall = i915_irq_uninstall; 4213 dev->driver->irq_handler = i915_irq_handler; 4214 } else { 4215 dev->driver->irq_preinstall = i965_irq_preinstall; 4216 dev->driver->irq_postinstall = i965_irq_postinstall; 4217 dev->driver->irq_uninstall = i965_irq_uninstall; 4218 dev->driver->irq_handler = i965_irq_handler; 4219 } 4220 if (I915_HAS_HOTPLUG(dev_priv)) 4221 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup; 4222 dev->driver->enable_vblank = i915_enable_vblank; 4223 dev->driver->disable_vblank = i915_disable_vblank; 4224 } 4225 } 4226 4227 /** 4228 * intel_irq_install - enables the hardware interrupt 4229 * @dev_priv: i915 device instance 4230 * 4231 * This function enables the hardware interrupt handling, but leaves the hotplug 4232 * handling still disabled. It is called after intel_irq_init(). 4233 * 4234 * In the driver load and resume code we need working interrupts in a few places 4235 * but don't want to deal with the hassle of concurrent probe and hotplug 4236 * workers. Hence the split into this two-stage approach. 4237 */ 4238 int intel_irq_install(struct drm_i915_private *dev_priv) 4239 { 4240 /* 4241 * We enable some interrupt sources in our postinstall hooks, so mark 4242 * interrupts as enabled _before_ actually enabling them to avoid 4243 * special cases in our ordering checks. 4244 */ 4245 dev_priv->pm.irqs_enabled = true; 4246 4247 return drm_irq_install(dev_priv->dev, dev_priv->dev->pdev->irq); 4248 } 4249 4250 /** 4251 * intel_irq_uninstall - finilizes all irq handling 4252 * @dev_priv: i915 device instance 4253 * 4254 * This stops interrupt and hotplug handling and unregisters and frees all 4255 * resources acquired in the init functions. 4256 */ 4257 void intel_irq_uninstall(struct drm_i915_private *dev_priv) 4258 { 4259 drm_irq_uninstall(dev_priv->dev); 4260 intel_hpd_cancel_work(dev_priv); 4261 dev_priv->pm.irqs_enabled = false; 4262 } 4263 4264 /** 4265 * intel_runtime_pm_disable_interrupts - runtime interrupt disabling 4266 * @dev_priv: i915 device instance 4267 * 4268 * This function is used to disable interrupts at runtime, both in the runtime 4269 * pm and the system suspend/resume code. 4270 */ 4271 void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv) 4272 { 4273 dev_priv->dev->driver->irq_uninstall(dev_priv->dev); 4274 dev_priv->pm.irqs_enabled = false; 4275 synchronize_irq(dev_priv->dev->irq); 4276 } 4277 4278 /** 4279 * intel_runtime_pm_enable_interrupts - runtime interrupt enabling 4280 * @dev_priv: i915 device instance 4281 * 4282 * This function is used to enable interrupts at runtime, both in the runtime 4283 * pm and the system suspend/resume code. 4284 */ 4285 void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv) 4286 { 4287 dev_priv->pm.irqs_enabled = true; 4288 dev_priv->dev->driver->irq_preinstall(dev_priv->dev); 4289 dev_priv->dev->driver->irq_postinstall(dev_priv->dev); 4290 } 4291