1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_irq.h" 7 8 #include <linux/sched/clock.h> 9 10 #include <drm/drm_managed.h> 11 12 #include "display/xe_display.h" 13 #include "regs/xe_irq_regs.h" 14 #include "xe_device.h" 15 #include "xe_drv.h" 16 #include "xe_gsc_proxy.h" 17 #include "xe_gt.h" 18 #include "xe_guc.h" 19 #include "xe_hw_engine.h" 20 #include "xe_hw_error.h" 21 #include "xe_i2c.h" 22 #include "xe_memirq.h" 23 #include "xe_mert.h" 24 #include "xe_mmio.h" 25 #include "xe_pxp.h" 26 #include "xe_sriov.h" 27 #include "xe_sysctrl.h" 28 #include "xe_tile.h" 29 30 /* 31 * Interrupt registers for a unit are always consecutive and ordered 32 * ISR, IMR, IIR, IER. 33 */ 34 #define IMR(offset) XE_REG(offset + 0x4) 35 #define IIR(offset) XE_REG(offset + 0x8) 36 #define IER(offset) XE_REG(offset + 0xc) 37 38 static int xe_irq_msix_init(struct xe_device *xe); 39 static void xe_irq_msix_free(struct xe_device *xe); 40 static int xe_irq_msix_request_irqs(struct xe_device *xe); 41 static void xe_irq_msix_synchronize_irq(struct xe_device *xe); 42 43 static void assert_iir_is_zero(struct xe_mmio *mmio, struct xe_reg reg) 44 { 45 u32 val = xe_mmio_read32(mmio, reg); 46 47 if (val == 0) 48 return; 49 50 drm_WARN(&mmio->tile->xe->drm, 1, 51 "Interrupt register 0x%x is not zero: 0x%08x\n", 52 reg.addr, val); 53 xe_mmio_write32(mmio, reg, 0xffffffff); 54 xe_mmio_read32(mmio, reg); 55 xe_mmio_write32(mmio, reg, 0xffffffff); 56 xe_mmio_read32(mmio, reg); 57 } 58 59 /* 60 * Unmask and enable the specified interrupts. Does not check current state, 61 * so any bits not specified here will become masked and disabled. 62 */ 63 static void unmask_and_enable(struct xe_tile *tile, u32 irqregs, u32 bits) 64 { 65 struct xe_mmio *mmio = &tile->mmio; 66 67 /* 68 * If we're just enabling an interrupt now, it shouldn't already 69 * be raised in the IIR. 70 */ 71 assert_iir_is_zero(mmio, IIR(irqregs)); 72 73 xe_mmio_write32(mmio, IER(irqregs), bits); 74 xe_mmio_write32(mmio, IMR(irqregs), ~bits); 75 76 /* Posting read */ 77 xe_mmio_read32(mmio, IMR(irqregs)); 78 } 79 80 /* Mask and disable all interrupts. */ 81 static void mask_and_disable(struct xe_tile *tile, u32 irqregs) 82 { 83 struct xe_mmio *mmio = &tile->mmio; 84 85 xe_mmio_write32(mmio, IMR(irqregs), ~0); 86 /* Posting read */ 87 xe_mmio_read32(mmio, IMR(irqregs)); 88 89 xe_mmio_write32(mmio, IER(irqregs), 0); 90 91 /* IIR can theoretically queue up two events. Be paranoid. */ 92 xe_mmio_write32(mmio, IIR(irqregs), ~0); 93 xe_mmio_read32(mmio, IIR(irqregs)); 94 xe_mmio_write32(mmio, IIR(irqregs), ~0); 95 xe_mmio_read32(mmio, IIR(irqregs)); 96 } 97 98 static u32 xelp_intr_disable(struct xe_device *xe) 99 { 100 struct xe_mmio *mmio = xe_root_tile_mmio(xe); 101 102 xe_mmio_write32(mmio, GFX_MSTR_IRQ, 0); 103 104 /* 105 * Now with master disabled, get a sample of level indications 106 * for this interrupt. Indications will be cleared on related acks. 107 * New indications can and will light up during processing, 108 * and will generate new interrupt after enabling master. 109 */ 110 return xe_mmio_read32(mmio, GFX_MSTR_IRQ); 111 } 112 113 static u32 114 gu_misc_irq_ack(struct xe_device *xe, const u32 master_ctl) 115 { 116 struct xe_mmio *mmio = xe_root_tile_mmio(xe); 117 u32 iir; 118 119 if (!(master_ctl & GU_MISC_IRQ)) 120 return 0; 121 122 iir = xe_mmio_read32(mmio, IIR(GU_MISC_IRQ_OFFSET)); 123 if (likely(iir)) 124 xe_mmio_write32(mmio, IIR(GU_MISC_IRQ_OFFSET), iir); 125 126 return iir; 127 } 128 129 static inline void xelp_intr_enable(struct xe_device *xe, bool stall) 130 { 131 struct xe_mmio *mmio = xe_root_tile_mmio(xe); 132 133 xe_mmio_write32(mmio, GFX_MSTR_IRQ, MASTER_IRQ); 134 if (stall) 135 xe_mmio_read32(mmio, GFX_MSTR_IRQ); 136 } 137 138 /* Enable/unmask the HWE interrupts for a specific GT's engines. */ 139 void xe_irq_enable_hwe(struct xe_gt *gt) 140 { 141 struct xe_device *xe = gt_to_xe(gt); 142 struct xe_mmio *mmio = >->mmio; 143 u32 common_mask, val, gsc_mask = 0, heci_mask = 0, 144 rcs_mask = 0, bcs_mask = 0, vcs_mask = 0, vecs_mask = 0, 145 ccs_mask = 0; 146 147 if (xe_device_uses_memirq(xe)) 148 return; 149 150 if (xe_device_uc_enabled(xe)) { 151 common_mask = GT_MI_USER_INTERRUPT | 152 GT_FLUSH_COMPLETE_INTERRUPT; 153 154 /* Enable Compute Walker Interrupt for non-MSIX platforms */ 155 if (GRAPHICS_VERx100(xe) >= 3511 && !xe_device_has_msix(xe)) { 156 rcs_mask |= GT_COMPUTE_WALKER_INTERRUPT; 157 ccs_mask |= GT_COMPUTE_WALKER_INTERRUPT; 158 } 159 } else { 160 common_mask = GT_MI_USER_INTERRUPT | 161 GT_CS_MASTER_ERROR_INTERRUPT | 162 GT_CONTEXT_SWITCH_INTERRUPT | 163 GT_WAIT_SEMAPHORE_INTERRUPT; 164 } 165 166 rcs_mask |= common_mask; 167 bcs_mask |= common_mask; 168 vcs_mask |= common_mask; 169 vecs_mask |= common_mask; 170 ccs_mask |= common_mask; 171 172 if (xe_gt_is_main_type(gt)) { 173 /* 174 * For enabling the interrupts, the information about fused off 175 * engines doesn't matter much, but this also allows to check if 176 * the engine is available architecturally in the platform 177 */ 178 u32 ccs_fuse_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_COMPUTE); 179 u32 bcs_fuse_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_COPY); 180 181 /* Enable interrupts for each engine class */ 182 xe_mmio_write32(mmio, RENDER_COPY_INTR_ENABLE, 183 REG_FIELD_PREP(ENGINE1_MASK, rcs_mask) | 184 REG_FIELD_PREP(ENGINE0_MASK, bcs_mask)); 185 if (ccs_fuse_mask) 186 xe_mmio_write32(mmio, CCS_RSVD_INTR_ENABLE, 187 REG_FIELD_PREP(ENGINE1_MASK, ccs_mask)); 188 189 /* Unmask interrupts for each engine instance */ 190 val = ~REG_FIELD_PREP(ENGINE1_MASK, rcs_mask); 191 xe_mmio_write32(mmio, RCS0_RSVD_INTR_MASK, val); 192 val = ~REG_FIELD_PREP(ENGINE1_MASK, bcs_mask); 193 xe_mmio_write32(mmio, BCS_RSVD_INTR_MASK, val); 194 195 val = ~(REG_FIELD_PREP(ENGINE1_MASK, bcs_mask) | 196 REG_FIELD_PREP(ENGINE0_MASK, bcs_mask)); 197 if (bcs_fuse_mask & (BIT(1)|BIT(2))) 198 xe_mmio_write32(mmio, XEHPC_BCS1_BCS2_INTR_MASK, val); 199 if (bcs_fuse_mask & (BIT(3)|BIT(4))) 200 xe_mmio_write32(mmio, XEHPC_BCS3_BCS4_INTR_MASK, val); 201 if (bcs_fuse_mask & (BIT(5)|BIT(6))) 202 xe_mmio_write32(mmio, XEHPC_BCS5_BCS6_INTR_MASK, val); 203 if (bcs_fuse_mask & (BIT(7)|BIT(8))) 204 xe_mmio_write32(mmio, XEHPC_BCS7_BCS8_INTR_MASK, val); 205 206 val = ~(REG_FIELD_PREP(ENGINE1_MASK, ccs_mask) | 207 REG_FIELD_PREP(ENGINE0_MASK, ccs_mask)); 208 if (ccs_fuse_mask & (BIT(0)|BIT(1))) 209 xe_mmio_write32(mmio, CCS0_CCS1_INTR_MASK, val); 210 if (ccs_fuse_mask & (BIT(2)|BIT(3))) 211 xe_mmio_write32(mmio, CCS2_CCS3_INTR_MASK, val); 212 } 213 214 if (xe_gt_is_media_type(gt) || MEDIA_VER(xe) < 13) { 215 u32 vcs_fuse_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_DECODE); 216 u32 vecs_fuse_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_ENHANCE); 217 u32 other_fuse_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_OTHER); 218 219 /* Enable interrupts for each engine class */ 220 xe_mmio_write32(mmio, VCS_VECS_INTR_ENABLE, 221 REG_FIELD_PREP(ENGINE1_MASK, vcs_mask) | 222 REG_FIELD_PREP(ENGINE0_MASK, vecs_mask)); 223 224 /* Unmask interrupts for each engine instance */ 225 val = ~(REG_FIELD_PREP(ENGINE1_MASK, vcs_mask) | 226 REG_FIELD_PREP(ENGINE0_MASK, vcs_mask)); 227 if (vcs_fuse_mask & (BIT(0) | BIT(1))) 228 xe_mmio_write32(mmio, VCS0_VCS1_INTR_MASK, val); 229 if (vcs_fuse_mask & (BIT(2) | BIT(3))) 230 xe_mmio_write32(mmio, VCS2_VCS3_INTR_MASK, val); 231 if (vcs_fuse_mask & (BIT(4) | BIT(5))) 232 xe_mmio_write32(mmio, VCS4_VCS5_INTR_MASK, val); 233 if (vcs_fuse_mask & (BIT(6) | BIT(7))) 234 xe_mmio_write32(mmio, VCS6_VCS7_INTR_MASK, val); 235 236 val = ~(REG_FIELD_PREP(ENGINE1_MASK, vecs_mask) | 237 REG_FIELD_PREP(ENGINE0_MASK, vecs_mask)); 238 if (vecs_fuse_mask & (BIT(0) | BIT(1))) 239 xe_mmio_write32(mmio, VECS0_VECS1_INTR_MASK, val); 240 if (vecs_fuse_mask & (BIT(2) | BIT(3))) 241 xe_mmio_write32(mmio, VECS2_VECS3_INTR_MASK, val); 242 243 /* 244 * the heci2 interrupt is enabled via the same register as the 245 * GSCCS interrupts, but it has its own mask register. 246 */ 247 if (other_fuse_mask) { 248 gsc_mask = common_mask | GSC_ER_COMPLETE; 249 heci_mask = GSC_IRQ_INTF(1); 250 } else if (xe->info.has_heci_gscfi) { 251 gsc_mask = GSC_IRQ_INTF(1); 252 } 253 254 if (gsc_mask) { 255 xe_mmio_write32(mmio, GUNIT_GSC_INTR_ENABLE, gsc_mask | heci_mask); 256 xe_mmio_write32(mmio, GUNIT_GSC_INTR_MASK, ~gsc_mask); 257 } 258 if (heci_mask) 259 xe_mmio_write32(mmio, HECI2_RSVD_INTR_MASK, ~(heci_mask << 16)); 260 261 if (xe_pxp_is_supported(xe)) { 262 u32 kcr_mask = KCR_PXP_STATE_TERMINATED_INTERRUPT | 263 KCR_APP_TERMINATED_PER_FW_REQ_INTERRUPT | 264 KCR_PXP_STATE_RESET_COMPLETE_INTERRUPT; 265 266 xe_mmio_write32(mmio, CRYPTO_RSVD_INTR_ENABLE, kcr_mask << 16); 267 xe_mmio_write32(mmio, CRYPTO_RSVD_INTR_MASK, ~(kcr_mask << 16)); 268 } 269 } 270 } 271 272 static u32 273 gt_engine_identity(struct xe_device *xe, 274 struct xe_mmio *mmio, 275 const unsigned int bank, 276 const unsigned int bit) 277 { 278 u32 timeout_ts; 279 u32 ident; 280 281 lockdep_assert_held(&xe->irq.lock); 282 283 xe_mmio_write32(mmio, IIR_REG_SELECTOR(bank), BIT(bit)); 284 285 /* 286 * NB: Specs do not specify how long to spin wait, 287 * so we do ~100us as an educated guess. 288 */ 289 timeout_ts = (local_clock() >> 10) + 100; 290 do { 291 ident = xe_mmio_read32(mmio, INTR_IDENTITY_REG(bank)); 292 } while (!(ident & INTR_DATA_VALID) && 293 !time_after32(local_clock() >> 10, timeout_ts)); 294 295 if (unlikely(!(ident & INTR_DATA_VALID))) { 296 drm_err(&xe->drm, "INTR_IDENTITY_REG%u:%u 0x%08x not valid!\n", 297 bank, bit, ident); 298 return 0; 299 } 300 301 xe_mmio_write32(mmio, INTR_IDENTITY_REG(bank), ident); 302 303 return ident; 304 } 305 306 #define OTHER_MEDIA_GUC_INSTANCE 16 307 308 static void 309 gt_other_irq_handler(struct xe_gt *gt, const u8 instance, const u16 iir) 310 { 311 if (instance == OTHER_GUC_INSTANCE && xe_gt_is_main_type(gt)) 312 return xe_guc_irq_handler(>->uc.guc, iir); 313 if (instance == OTHER_MEDIA_GUC_INSTANCE && xe_gt_is_media_type(gt)) 314 return xe_guc_irq_handler(>->uc.guc, iir); 315 if (instance == OTHER_GSC_HECI2_INSTANCE && xe_gt_is_media_type(gt)) 316 return xe_gsc_proxy_irq_handler(>->uc.gsc, iir); 317 318 if (instance != OTHER_GUC_INSTANCE && 319 instance != OTHER_MEDIA_GUC_INSTANCE) { 320 WARN_ONCE(1, "unhandled other interrupt instance=0x%x, iir=0x%x\n", 321 instance, iir); 322 } 323 } 324 325 static struct xe_gt *pick_engine_gt(struct xe_tile *tile, 326 enum xe_engine_class class, 327 unsigned int instance) 328 { 329 struct xe_device *xe = tile_to_xe(tile); 330 331 if (MEDIA_VER(xe) < 13) 332 return tile->primary_gt; 333 334 switch (class) { 335 case XE_ENGINE_CLASS_VIDEO_DECODE: 336 case XE_ENGINE_CLASS_VIDEO_ENHANCE: 337 return tile->media_gt; 338 case XE_ENGINE_CLASS_OTHER: 339 switch (instance) { 340 case OTHER_MEDIA_GUC_INSTANCE: 341 case OTHER_GSC_INSTANCE: 342 case OTHER_GSC_HECI2_INSTANCE: 343 return tile->media_gt; 344 default: 345 break; 346 } 347 fallthrough; 348 default: 349 return tile->primary_gt; 350 } 351 } 352 353 static void gt_irq_handler(struct xe_tile *tile, 354 u32 master_ctl, unsigned long *intr_dw, 355 u32 *identity) 356 { 357 struct xe_device *xe = tile_to_xe(tile); 358 struct xe_mmio *mmio = &tile->mmio; 359 unsigned int bank, bit; 360 u16 instance, intr_vec; 361 enum xe_engine_class class; 362 struct xe_hw_engine *hwe; 363 364 spin_lock(&xe->irq.lock); 365 366 for (bank = 0; bank < 2; bank++) { 367 if (!(master_ctl & GT_DW_IRQ(bank))) 368 continue; 369 370 intr_dw[bank] = xe_mmio_read32(mmio, GT_INTR_DW(bank)); 371 for_each_set_bit(bit, intr_dw + bank, 32) 372 identity[bit] = gt_engine_identity(xe, mmio, bank, bit); 373 xe_mmio_write32(mmio, GT_INTR_DW(bank), intr_dw[bank]); 374 375 for_each_set_bit(bit, intr_dw + bank, 32) { 376 struct xe_gt *engine_gt; 377 378 class = INTR_ENGINE_CLASS(identity[bit]); 379 instance = INTR_ENGINE_INSTANCE(identity[bit]); 380 intr_vec = INTR_ENGINE_INTR(identity[bit]); 381 382 engine_gt = pick_engine_gt(tile, class, instance); 383 384 hwe = xe_gt_hw_engine(engine_gt, class, instance, false); 385 if (hwe) { 386 xe_hw_engine_handle_irq(hwe, intr_vec); 387 continue; 388 } 389 390 if (class == XE_ENGINE_CLASS_OTHER) { 391 /* 392 * HECI GSCFI interrupts come from outside of GT. 393 * KCR irqs come from inside GT but are handled 394 * by the global PXP subsystem. 395 */ 396 if (xe->info.has_heci_gscfi && instance == OTHER_GSC_INSTANCE) 397 xe_heci_gsc_irq_handler(xe, intr_vec); 398 else if (instance == OTHER_KCR_INSTANCE) 399 xe_pxp_irq_handler(xe, intr_vec); 400 else 401 gt_other_irq_handler(engine_gt, instance, intr_vec); 402 } 403 } 404 } 405 406 spin_unlock(&xe->irq.lock); 407 } 408 409 /* 410 * Top-level interrupt handler for Xe_LP platforms (which did not have 411 * a "master tile" interrupt register. 412 */ 413 static irqreturn_t xelp_irq_handler(int irq, void *arg) 414 { 415 struct xe_device *xe = arg; 416 struct xe_tile *tile = xe_device_get_root_tile(xe); 417 u32 master_ctl, gu_misc_iir; 418 unsigned long intr_dw[2]; 419 u32 identity[32]; 420 421 if (!atomic_read(&xe->irq.enabled)) 422 return IRQ_NONE; 423 424 master_ctl = xelp_intr_disable(xe); 425 if (!master_ctl) { 426 xelp_intr_enable(xe, false); 427 return IRQ_NONE; 428 } 429 430 gt_irq_handler(tile, master_ctl, intr_dw, identity); 431 432 xe_display_irq_handler(xe, master_ctl); 433 434 gu_misc_iir = gu_misc_irq_ack(xe, master_ctl); 435 436 xelp_intr_enable(xe, false); 437 438 xe_display_irq_enable(xe, gu_misc_iir); 439 440 return IRQ_HANDLED; 441 } 442 443 static u32 dg1_intr_disable(struct xe_device *xe) 444 { 445 struct xe_mmio *mmio = xe_root_tile_mmio(xe); 446 u32 val; 447 448 /* First disable interrupts */ 449 xe_mmio_write32(mmio, DG1_MSTR_TILE_INTR, 0); 450 451 /* Get the indication levels and ack the master unit */ 452 val = xe_mmio_read32(mmio, DG1_MSTR_TILE_INTR); 453 if (unlikely(!val)) 454 return 0; 455 456 xe_mmio_write32(mmio, DG1_MSTR_TILE_INTR, val); 457 458 return val; 459 } 460 461 static void dg1_intr_enable(struct xe_device *xe, bool stall) 462 { 463 struct xe_mmio *mmio = xe_root_tile_mmio(xe); 464 465 xe_mmio_write32(mmio, DG1_MSTR_TILE_INTR, DG1_MSTR_IRQ); 466 if (stall) 467 xe_mmio_read32(mmio, DG1_MSTR_TILE_INTR); 468 } 469 470 /* 471 * Top-level interrupt handler for Xe_LP+ and beyond. These platforms have 472 * a "master tile" interrupt register which must be consulted before the 473 * "graphics master" interrupt register. 474 */ 475 static irqreturn_t dg1_irq_handler(int irq, void *arg) 476 { 477 struct xe_device *xe = arg; 478 struct xe_tile *tile; 479 u32 master_tile_ctl, master_ctl = 0, gu_misc_iir = 0; 480 unsigned long intr_dw[2]; 481 u32 identity[32]; 482 u8 id; 483 484 /* TODO: This really shouldn't be copied+pasted */ 485 486 if (!atomic_read(&xe->irq.enabled)) 487 return IRQ_NONE; 488 489 master_tile_ctl = dg1_intr_disable(xe); 490 if (!master_tile_ctl) { 491 dg1_intr_enable(xe, false); 492 return IRQ_NONE; 493 } 494 495 for_each_tile(tile, xe, id) { 496 struct xe_mmio *mmio = &tile->mmio; 497 498 if ((master_tile_ctl & DG1_MSTR_TILE(tile->id)) == 0) 499 continue; 500 501 master_ctl = xe_mmio_read32(mmio, GFX_MSTR_IRQ); 502 503 /* 504 * We might be in irq handler just when PCIe DPC is initiated 505 * and all MMIO reads will be returned with all 1's. Ignore this 506 * irq as device is inaccessible. 507 */ 508 if (master_ctl == REG_GENMASK(31, 0)) { 509 drm_dbg(&tile_to_xe(tile)->drm, 510 "Ignore this IRQ as device might be in DPC containment.\n"); 511 return IRQ_HANDLED; 512 } 513 514 xe_mmio_write32(mmio, GFX_MSTR_IRQ, master_ctl); 515 516 gt_irq_handler(tile, master_ctl, intr_dw, identity); 517 xe_hw_error_irq_handler(tile, master_ctl); 518 519 /* 520 * Display interrupts (including display backlight operations 521 * that get reported as Gunit GSE) would only be hooked up to 522 * the primary tile. 523 */ 524 if (id == 0) { 525 if (xe->info.has_heci_cscfi) 526 xe_heci_csc_irq_handler(xe, master_ctl); 527 xe_display_irq_handler(xe, master_ctl); 528 xe_i2c_irq_handler(xe, master_ctl); 529 xe_sysctrl_irq_handler(xe, master_ctl); 530 xe_mert_irq_handler(xe, master_ctl); 531 gu_misc_iir = gu_misc_irq_ack(xe, master_ctl); 532 } 533 } 534 535 dg1_intr_enable(xe, false); 536 xe_display_irq_enable(xe, gu_misc_iir); 537 538 return IRQ_HANDLED; 539 } 540 541 static void gt_irq_reset(struct xe_tile *tile) 542 { 543 struct xe_mmio *mmio = &tile->mmio; 544 u32 ccs_mask = ~0; 545 u32 bcs_mask = ~0; 546 547 if (tile->primary_gt) { 548 ccs_mask = xe_hw_engine_mask_per_class(tile->primary_gt, 549 XE_ENGINE_CLASS_COMPUTE); 550 bcs_mask = xe_hw_engine_mask_per_class(tile->primary_gt, 551 XE_ENGINE_CLASS_COPY); 552 } 553 554 /* Disable RCS, BCS, VCS and VECS class engines. */ 555 xe_mmio_write32(mmio, RENDER_COPY_INTR_ENABLE, 0); 556 xe_mmio_write32(mmio, VCS_VECS_INTR_ENABLE, 0); 557 if (ccs_mask) 558 xe_mmio_write32(mmio, CCS_RSVD_INTR_ENABLE, 0); 559 560 /* Restore masks irqs on RCS, BCS, VCS and VECS engines. */ 561 xe_mmio_write32(mmio, RCS0_RSVD_INTR_MASK, ~0); 562 xe_mmio_write32(mmio, BCS_RSVD_INTR_MASK, ~0); 563 if (bcs_mask & (BIT(1)|BIT(2))) 564 xe_mmio_write32(mmio, XEHPC_BCS1_BCS2_INTR_MASK, ~0); 565 if (bcs_mask & (BIT(3)|BIT(4))) 566 xe_mmio_write32(mmio, XEHPC_BCS3_BCS4_INTR_MASK, ~0); 567 if (bcs_mask & (BIT(5)|BIT(6))) 568 xe_mmio_write32(mmio, XEHPC_BCS5_BCS6_INTR_MASK, ~0); 569 if (bcs_mask & (BIT(7)|BIT(8))) 570 xe_mmio_write32(mmio, XEHPC_BCS7_BCS8_INTR_MASK, ~0); 571 xe_mmio_write32(mmio, VCS0_VCS1_INTR_MASK, ~0); 572 xe_mmio_write32(mmio, VCS2_VCS3_INTR_MASK, ~0); 573 xe_mmio_write32(mmio, VECS0_VECS1_INTR_MASK, ~0); 574 if (ccs_mask & (BIT(0)|BIT(1))) 575 xe_mmio_write32(mmio, CCS0_CCS1_INTR_MASK, ~0); 576 if (ccs_mask & (BIT(2)|BIT(3))) 577 xe_mmio_write32(mmio, CCS2_CCS3_INTR_MASK, ~0); 578 579 if ((tile->media_gt && 580 xe_hw_engine_mask_per_class(tile->media_gt, XE_ENGINE_CLASS_OTHER)) || 581 tile_to_xe(tile)->info.has_heci_gscfi) { 582 xe_mmio_write32(mmio, GUNIT_GSC_INTR_ENABLE, 0); 583 xe_mmio_write32(mmio, GUNIT_GSC_INTR_MASK, ~0); 584 xe_mmio_write32(mmio, HECI2_RSVD_INTR_MASK, ~0); 585 xe_mmio_write32(mmio, CRYPTO_RSVD_INTR_ENABLE, 0); 586 xe_mmio_write32(mmio, CRYPTO_RSVD_INTR_MASK, ~0); 587 } 588 589 xe_mmio_write32(mmio, GPM_WGBOXPERF_INTR_ENABLE, 0); 590 xe_mmio_write32(mmio, GPM_WGBOXPERF_INTR_MASK, ~0); 591 xe_mmio_write32(mmio, GUC_SG_INTR_ENABLE, 0); 592 xe_mmio_write32(mmio, GUC_SG_INTR_MASK, ~0); 593 } 594 595 static void xelp_irq_reset(struct xe_tile *tile) 596 { 597 xelp_intr_disable(tile_to_xe(tile)); 598 599 gt_irq_reset(tile); 600 601 if (IS_SRIOV_VF(tile_to_xe(tile))) 602 return; 603 604 mask_and_disable(tile, PCU_IRQ_OFFSET); 605 } 606 607 static void dg1_irq_reset(struct xe_tile *tile) 608 { 609 if (xe_tile_is_root(tile)) 610 dg1_intr_disable(tile_to_xe(tile)); 611 612 gt_irq_reset(tile); 613 614 if (IS_SRIOV_VF(tile_to_xe(tile))) 615 return; 616 617 mask_and_disable(tile, PCU_IRQ_OFFSET); 618 } 619 620 static void dg1_irq_reset_mstr(struct xe_tile *tile) 621 { 622 struct xe_mmio *mmio = &tile->mmio; 623 624 xe_mmio_write32(mmio, GFX_MSTR_IRQ, ~0); 625 } 626 627 static void vf_irq_reset(struct xe_device *xe) 628 { 629 struct xe_tile *tile; 630 unsigned int id; 631 632 xe_assert(xe, IS_SRIOV_VF(xe)); 633 634 if (GRAPHICS_VERx100(xe) < 1210) 635 xelp_intr_disable(xe); 636 else 637 xe_assert(xe, xe_device_has_memirq(xe)); 638 639 for_each_tile(tile, xe, id) { 640 if (xe_device_has_memirq(xe)) 641 xe_memirq_reset(&tile->memirq); 642 else 643 gt_irq_reset(tile); 644 } 645 } 646 647 static void xe_irq_reset(struct xe_device *xe) 648 { 649 struct xe_tile *tile; 650 u8 id; 651 652 if (IS_SRIOV_VF(xe)) 653 return vf_irq_reset(xe); 654 655 if (xe_device_uses_memirq(xe)) { 656 for_each_tile(tile, xe, id) 657 xe_memirq_reset(&tile->memirq); 658 } 659 660 for_each_tile(tile, xe, id) { 661 if (GRAPHICS_VERx100(xe) >= 1210) 662 dg1_irq_reset(tile); 663 else 664 xelp_irq_reset(tile); 665 } 666 667 tile = xe_device_get_root_tile(xe); 668 mask_and_disable(tile, GU_MISC_IRQ_OFFSET); 669 xe_display_irq_reset(xe); 670 xe_i2c_irq_reset(xe); 671 672 /* 673 * The tile's top-level status register should be the last one 674 * to be reset to avoid possible bit re-latching from lower 675 * level interrupts. 676 */ 677 if (GRAPHICS_VERx100(xe) >= 1210) { 678 for_each_tile(tile, xe, id) 679 dg1_irq_reset_mstr(tile); 680 } 681 } 682 683 static void vf_irq_postinstall(struct xe_device *xe) 684 { 685 struct xe_tile *tile; 686 unsigned int id; 687 688 for_each_tile(tile, xe, id) 689 if (xe_device_has_memirq(xe)) 690 xe_memirq_postinstall(&tile->memirq); 691 692 if (GRAPHICS_VERx100(xe) < 1210) 693 xelp_intr_enable(xe, true); 694 else 695 xe_assert(xe, xe_device_has_memirq(xe)); 696 } 697 698 static void xe_irq_postinstall(struct xe_device *xe) 699 { 700 if (IS_SRIOV_VF(xe)) 701 return vf_irq_postinstall(xe); 702 703 if (xe_device_uses_memirq(xe)) { 704 struct xe_tile *tile; 705 unsigned int id; 706 707 for_each_tile(tile, xe, id) 708 xe_memirq_postinstall(&tile->memirq); 709 } 710 711 xe_display_irq_postinstall(xe); 712 xe_i2c_irq_postinstall(xe); 713 714 /* 715 * ASLE backlight operations are reported via GUnit GSE interrupts 716 * on the root tile. 717 */ 718 unmask_and_enable(xe_device_get_root_tile(xe), 719 GU_MISC_IRQ_OFFSET, GU_MISC_GSE); 720 721 /* Enable top-level interrupts */ 722 if (GRAPHICS_VERx100(xe) >= 1210) 723 dg1_intr_enable(xe, true); 724 else 725 xelp_intr_enable(xe, true); 726 } 727 728 static irqreturn_t vf_mem_irq_handler(int irq, void *arg) 729 { 730 struct xe_device *xe = arg; 731 struct xe_tile *tile; 732 unsigned int id; 733 734 if (!atomic_read(&xe->irq.enabled)) 735 return IRQ_NONE; 736 737 for_each_tile(tile, xe, id) 738 xe_memirq_handler(&tile->memirq); 739 740 return IRQ_HANDLED; 741 } 742 743 static irq_handler_t xe_irq_handler(struct xe_device *xe) 744 { 745 if (IS_SRIOV_VF(xe) && xe_device_has_memirq(xe)) 746 return vf_mem_irq_handler; 747 748 if (GRAPHICS_VERx100(xe) >= 1210) 749 return dg1_irq_handler; 750 else 751 return xelp_irq_handler; 752 } 753 754 static int xe_irq_msi_request_irqs(struct xe_device *xe) 755 { 756 struct pci_dev *pdev = to_pci_dev(xe->drm.dev); 757 irq_handler_t irq_handler; 758 int irq, err; 759 760 irq_handler = xe_irq_handler(xe); 761 if (!irq_handler) { 762 drm_err(&xe->drm, "No supported interrupt handler"); 763 return -EINVAL; 764 } 765 766 irq = pci_irq_vector(pdev, 0); 767 err = request_irq(irq, irq_handler, IRQF_SHARED, DRIVER_NAME, xe); 768 if (err < 0) { 769 drm_err(&xe->drm, "Failed to request MSI IRQ %d\n", err); 770 return err; 771 } 772 773 return 0; 774 } 775 776 static void xe_irq_msi_free(struct xe_device *xe) 777 { 778 struct pci_dev *pdev = to_pci_dev(xe->drm.dev); 779 int irq; 780 781 irq = pci_irq_vector(pdev, 0); 782 free_irq(irq, xe); 783 } 784 785 static void irq_uninstall(void *arg) 786 { 787 struct xe_device *xe = arg; 788 789 if (!atomic_xchg(&xe->irq.enabled, 0)) 790 return; 791 792 xe_irq_reset(xe); 793 794 if (xe_device_has_msix(xe)) 795 xe_irq_msix_free(xe); 796 else 797 xe_irq_msi_free(xe); 798 } 799 800 int xe_irq_init(struct xe_device *xe) 801 { 802 spin_lock_init(&xe->irq.lock); 803 804 return xe_irq_msix_init(xe); 805 } 806 807 int xe_irq_install(struct xe_device *xe) 808 { 809 struct pci_dev *pdev = to_pci_dev(xe->drm.dev); 810 unsigned int irq_flags = PCI_IRQ_MSI; 811 int nvec = 1; 812 int err; 813 814 xe_hw_error_init(xe); 815 816 xe_irq_reset(xe); 817 818 if (xe_device_has_msix(xe)) { 819 nvec = xe->irq.msix.nvec; 820 irq_flags = PCI_IRQ_MSIX; 821 } 822 823 err = pci_alloc_irq_vectors(pdev, nvec, nvec, irq_flags); 824 if (err < 0) { 825 drm_err(&xe->drm, "Failed to allocate IRQ vectors: %d\n", err); 826 return err; 827 } 828 829 err = xe_device_has_msix(xe) ? xe_irq_msix_request_irqs(xe) : 830 xe_irq_msi_request_irqs(xe); 831 if (err) 832 return err; 833 834 atomic_set(&xe->irq.enabled, 1); 835 836 xe_irq_postinstall(xe); 837 838 return devm_add_action_or_reset(xe->drm.dev, irq_uninstall, xe); 839 } 840 841 static void xe_irq_msi_synchronize_irq(struct xe_device *xe) 842 { 843 synchronize_irq(to_pci_dev(xe->drm.dev)->irq); 844 } 845 846 void xe_irq_suspend(struct xe_device *xe) 847 { 848 atomic_set(&xe->irq.enabled, 0); /* no new irqs */ 849 850 /* flush irqs */ 851 if (xe_device_has_msix(xe)) 852 xe_irq_msix_synchronize_irq(xe); 853 else 854 xe_irq_msi_synchronize_irq(xe); 855 xe_irq_reset(xe); /* turn irqs off */ 856 } 857 858 void xe_irq_resume(struct xe_device *xe) 859 { 860 struct xe_gt *gt; 861 int id; 862 863 /* 864 * lock not needed: 865 * 1. no irq will arrive before the postinstall 866 * 2. display is not yet resumed 867 */ 868 atomic_set(&xe->irq.enabled, 1); 869 xe_irq_reset(xe); 870 xe_irq_postinstall(xe); /* turn irqs on */ 871 872 for_each_gt(gt, xe, id) 873 xe_irq_enable_hwe(gt); 874 } 875 876 /* MSI-X related definitions and functions below. */ 877 878 enum xe_irq_msix_static { 879 GUC2HOST_MSIX = 0, 880 DEFAULT_MSIX = XE_IRQ_DEFAULT_MSIX, 881 /* Must be last */ 882 NUM_OF_STATIC_MSIX, 883 }; 884 885 static int xe_irq_msix_init(struct xe_device *xe) 886 { 887 struct pci_dev *pdev = to_pci_dev(xe->drm.dev); 888 int nvec = pci_msix_vec_count(pdev); 889 890 if (nvec == -EINVAL) 891 return 0; /* MSI */ 892 893 if (nvec < 0) { 894 drm_err(&xe->drm, "Failed getting MSI-X vectors count: %d\n", nvec); 895 return nvec; 896 } 897 898 xe->irq.msix.nvec = nvec; 899 xa_init_flags(&xe->irq.msix.indexes, XA_FLAGS_ALLOC); 900 return 0; 901 } 902 903 static irqreturn_t xe_irq_msix_default_hwe_handler(int irq, void *arg) 904 { 905 unsigned int tile_id, gt_id; 906 struct xe_device *xe = arg; 907 struct xe_memirq *memirq; 908 struct xe_hw_engine *hwe; 909 enum xe_hw_engine_id id; 910 struct xe_tile *tile; 911 struct xe_gt *gt; 912 913 if (!atomic_read(&xe->irq.enabled)) 914 return IRQ_NONE; 915 916 for_each_tile(tile, xe, tile_id) { 917 memirq = &tile->memirq; 918 if (!memirq->bo) 919 continue; 920 921 for_each_gt(gt, xe, gt_id) { 922 if (gt->tile != tile) 923 continue; 924 925 for_each_hw_engine(hwe, gt, id) 926 xe_memirq_hwe_handler(memirq, hwe); 927 } 928 } 929 930 return IRQ_HANDLED; 931 } 932 933 static int xe_irq_msix_alloc_vector(struct xe_device *xe, void *irq_buf, 934 bool dynamic_msix, u16 *msix) 935 { 936 struct xa_limit limit; 937 int ret; 938 u32 id; 939 940 limit = (dynamic_msix) ? XA_LIMIT(NUM_OF_STATIC_MSIX, xe->irq.msix.nvec - 1) : 941 XA_LIMIT(*msix, *msix); 942 ret = xa_alloc(&xe->irq.msix.indexes, &id, irq_buf, limit, GFP_KERNEL); 943 if (ret) 944 return ret; 945 946 if (dynamic_msix) 947 *msix = id; 948 949 return 0; 950 } 951 952 static void xe_irq_msix_release_vector(struct xe_device *xe, u16 msix) 953 { 954 xa_erase(&xe->irq.msix.indexes, msix); 955 } 956 957 static int xe_irq_msix_request_irq_internal(struct xe_device *xe, irq_handler_t handler, 958 void *irq_buf, const char *name, u16 msix) 959 { 960 struct pci_dev *pdev = to_pci_dev(xe->drm.dev); 961 int ret, irq; 962 963 irq = pci_irq_vector(pdev, msix); 964 if (irq < 0) 965 return irq; 966 967 ret = request_irq(irq, handler, IRQF_SHARED, name, irq_buf); 968 if (ret < 0) 969 return ret; 970 971 return 0; 972 } 973 974 int xe_irq_msix_request_irq(struct xe_device *xe, irq_handler_t handler, void *irq_buf, 975 const char *name, bool dynamic_msix, u16 *msix) 976 { 977 int ret; 978 979 ret = xe_irq_msix_alloc_vector(xe, irq_buf, dynamic_msix, msix); 980 if (ret) 981 return ret; 982 983 ret = xe_irq_msix_request_irq_internal(xe, handler, irq_buf, name, *msix); 984 if (ret) { 985 drm_err(&xe->drm, "Failed to request IRQ for MSI-X %u\n", *msix); 986 xe_irq_msix_release_vector(xe, *msix); 987 return ret; 988 } 989 990 return 0; 991 } 992 993 void xe_irq_msix_free_irq(struct xe_device *xe, u16 msix) 994 { 995 struct pci_dev *pdev = to_pci_dev(xe->drm.dev); 996 int irq; 997 void *irq_buf; 998 999 irq_buf = xa_load(&xe->irq.msix.indexes, msix); 1000 if (!irq_buf) 1001 return; 1002 1003 irq = pci_irq_vector(pdev, msix); 1004 if (irq < 0) { 1005 drm_err(&xe->drm, "MSI-X %u can't be released, there is no matching IRQ\n", msix); 1006 return; 1007 } 1008 1009 free_irq(irq, irq_buf); 1010 xe_irq_msix_release_vector(xe, msix); 1011 } 1012 1013 int xe_irq_msix_request_irqs(struct xe_device *xe) 1014 { 1015 int err; 1016 u16 msix; 1017 1018 msix = GUC2HOST_MSIX; 1019 err = xe_irq_msix_request_irq(xe, xe_irq_handler(xe), xe, 1020 DRIVER_NAME "-guc2host", false, &msix); 1021 if (err) 1022 return err; 1023 1024 msix = DEFAULT_MSIX; 1025 err = xe_irq_msix_request_irq(xe, xe_irq_msix_default_hwe_handler, xe, 1026 DRIVER_NAME "-default-msix", false, &msix); 1027 if (err) { 1028 xe_irq_msix_free_irq(xe, GUC2HOST_MSIX); 1029 return err; 1030 } 1031 1032 return 0; 1033 } 1034 1035 void xe_irq_msix_free(struct xe_device *xe) 1036 { 1037 unsigned long msix; 1038 u32 *dummy; 1039 1040 xa_for_each(&xe->irq.msix.indexes, msix, dummy) 1041 xe_irq_msix_free_irq(xe, msix); 1042 xa_destroy(&xe->irq.msix.indexes); 1043 } 1044 1045 void xe_irq_msix_synchronize_irq(struct xe_device *xe) 1046 { 1047 struct pci_dev *pdev = to_pci_dev(xe->drm.dev); 1048 unsigned long msix; 1049 u32 *dummy; 1050 1051 xa_for_each(&xe->irq.msix.indexes, msix, dummy) 1052 synchronize_irq(pci_irq_vector(pdev, msix)); 1053 } 1054