1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_irq.h" 7 8 #include <linux/sched/clock.h> 9 10 #include <drm/drm_managed.h> 11 12 #include "display/xe_display.h" 13 #include "regs/xe_guc_regs.h" 14 #include "regs/xe_irq_regs.h" 15 #include "xe_device.h" 16 #include "xe_drv.h" 17 #include "xe_gsc_proxy.h" 18 #include "xe_gt.h" 19 #include "xe_guc.h" 20 #include "xe_hw_engine.h" 21 #include "xe_memirq.h" 22 #include "xe_mmio.h" 23 #include "xe_pxp.h" 24 #include "xe_sriov.h" 25 26 /* 27 * Interrupt registers for a unit are always consecutive and ordered 28 * ISR, IMR, IIR, IER. 29 */ 30 #define IMR(offset) XE_REG(offset + 0x4) 31 #define IIR(offset) XE_REG(offset + 0x8) 32 #define IER(offset) XE_REG(offset + 0xc) 33 34 static int xe_irq_msix_init(struct xe_device *xe); 35 static void xe_irq_msix_free(struct xe_device *xe); 36 static int xe_irq_msix_request_irqs(struct xe_device *xe); 37 static void xe_irq_msix_synchronize_irq(struct xe_device *xe); 38 39 static void assert_iir_is_zero(struct xe_mmio *mmio, struct xe_reg reg) 40 { 41 u32 val = xe_mmio_read32(mmio, reg); 42 43 if (val == 0) 44 return; 45 46 drm_WARN(&mmio->tile->xe->drm, 1, 47 "Interrupt register 0x%x is not zero: 0x%08x\n", 48 reg.addr, val); 49 xe_mmio_write32(mmio, reg, 0xffffffff); 50 xe_mmio_read32(mmio, reg); 51 xe_mmio_write32(mmio, reg, 0xffffffff); 52 xe_mmio_read32(mmio, reg); 53 } 54 55 /* 56 * Unmask and enable the specified interrupts. Does not check current state, 57 * so any bits not specified here will become masked and disabled. 58 */ 59 static void unmask_and_enable(struct xe_tile *tile, u32 irqregs, u32 bits) 60 { 61 struct xe_mmio *mmio = &tile->mmio; 62 63 /* 64 * If we're just enabling an interrupt now, it shouldn't already 65 * be raised in the IIR. 66 */ 67 assert_iir_is_zero(mmio, IIR(irqregs)); 68 69 xe_mmio_write32(mmio, IER(irqregs), bits); 70 xe_mmio_write32(mmio, IMR(irqregs), ~bits); 71 72 /* Posting read */ 73 xe_mmio_read32(mmio, IMR(irqregs)); 74 } 75 76 /* Mask and disable all interrupts. */ 77 static void mask_and_disable(struct xe_tile *tile, u32 irqregs) 78 { 79 struct xe_mmio *mmio = &tile->mmio; 80 81 xe_mmio_write32(mmio, IMR(irqregs), ~0); 82 /* Posting read */ 83 xe_mmio_read32(mmio, IMR(irqregs)); 84 85 xe_mmio_write32(mmio, IER(irqregs), 0); 86 87 /* IIR can theoretically queue up two events. Be paranoid. */ 88 xe_mmio_write32(mmio, IIR(irqregs), ~0); 89 xe_mmio_read32(mmio, IIR(irqregs)); 90 xe_mmio_write32(mmio, IIR(irqregs), ~0); 91 xe_mmio_read32(mmio, IIR(irqregs)); 92 } 93 94 static u32 xelp_intr_disable(struct xe_device *xe) 95 { 96 struct xe_mmio *mmio = xe_root_tile_mmio(xe); 97 98 xe_mmio_write32(mmio, GFX_MSTR_IRQ, 0); 99 100 /* 101 * Now with master disabled, get a sample of level indications 102 * for this interrupt. Indications will be cleared on related acks. 103 * New indications can and will light up during processing, 104 * and will generate new interrupt after enabling master. 105 */ 106 return xe_mmio_read32(mmio, GFX_MSTR_IRQ); 107 } 108 109 static u32 110 gu_misc_irq_ack(struct xe_device *xe, const u32 master_ctl) 111 { 112 struct xe_mmio *mmio = xe_root_tile_mmio(xe); 113 u32 iir; 114 115 if (!(master_ctl & GU_MISC_IRQ)) 116 return 0; 117 118 iir = xe_mmio_read32(mmio, IIR(GU_MISC_IRQ_OFFSET)); 119 if (likely(iir)) 120 xe_mmio_write32(mmio, IIR(GU_MISC_IRQ_OFFSET), iir); 121 122 return iir; 123 } 124 125 static inline void xelp_intr_enable(struct xe_device *xe, bool stall) 126 { 127 struct xe_mmio *mmio = xe_root_tile_mmio(xe); 128 129 xe_mmio_write32(mmio, GFX_MSTR_IRQ, MASTER_IRQ); 130 if (stall) 131 xe_mmio_read32(mmio, GFX_MSTR_IRQ); 132 } 133 134 /* Enable/unmask the HWE interrupts for a specific GT's engines. */ 135 void xe_irq_enable_hwe(struct xe_gt *gt) 136 { 137 struct xe_device *xe = gt_to_xe(gt); 138 struct xe_mmio *mmio = >->mmio; 139 u32 ccs_mask, bcs_mask; 140 u32 irqs, dmask, smask; 141 u32 gsc_mask = 0; 142 u32 heci_mask = 0; 143 144 if (xe_device_uses_memirq(xe)) 145 return; 146 147 if (xe_device_uc_enabled(xe)) { 148 irqs = GT_RENDER_USER_INTERRUPT | 149 GT_RENDER_PIPECTL_NOTIFY_INTERRUPT; 150 } else { 151 irqs = GT_RENDER_USER_INTERRUPT | 152 GT_CS_MASTER_ERROR_INTERRUPT | 153 GT_CONTEXT_SWITCH_INTERRUPT | 154 GT_WAIT_SEMAPHORE_INTERRUPT; 155 } 156 157 ccs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_COMPUTE); 158 bcs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_COPY); 159 160 dmask = irqs << 16 | irqs; 161 smask = irqs << 16; 162 163 if (!xe_gt_is_media_type(gt)) { 164 /* Enable interrupts for each engine class */ 165 xe_mmio_write32(mmio, RENDER_COPY_INTR_ENABLE, dmask); 166 if (ccs_mask) 167 xe_mmio_write32(mmio, CCS_RSVD_INTR_ENABLE, smask); 168 169 /* Unmask interrupts for each engine instance */ 170 xe_mmio_write32(mmio, RCS0_RSVD_INTR_MASK, ~smask); 171 xe_mmio_write32(mmio, BCS_RSVD_INTR_MASK, ~smask); 172 if (bcs_mask & (BIT(1)|BIT(2))) 173 xe_mmio_write32(mmio, XEHPC_BCS1_BCS2_INTR_MASK, ~dmask); 174 if (bcs_mask & (BIT(3)|BIT(4))) 175 xe_mmio_write32(mmio, XEHPC_BCS3_BCS4_INTR_MASK, ~dmask); 176 if (bcs_mask & (BIT(5)|BIT(6))) 177 xe_mmio_write32(mmio, XEHPC_BCS5_BCS6_INTR_MASK, ~dmask); 178 if (bcs_mask & (BIT(7)|BIT(8))) 179 xe_mmio_write32(mmio, XEHPC_BCS7_BCS8_INTR_MASK, ~dmask); 180 if (ccs_mask & (BIT(0)|BIT(1))) 181 xe_mmio_write32(mmio, CCS0_CCS1_INTR_MASK, ~dmask); 182 if (ccs_mask & (BIT(2)|BIT(3))) 183 xe_mmio_write32(mmio, CCS2_CCS3_INTR_MASK, ~dmask); 184 } 185 186 if (xe_gt_is_media_type(gt) || MEDIA_VER(xe) < 13) { 187 /* Enable interrupts for each engine class */ 188 xe_mmio_write32(mmio, VCS_VECS_INTR_ENABLE, dmask); 189 190 /* Unmask interrupts for each engine instance */ 191 xe_mmio_write32(mmio, VCS0_VCS1_INTR_MASK, ~dmask); 192 xe_mmio_write32(mmio, VCS2_VCS3_INTR_MASK, ~dmask); 193 xe_mmio_write32(mmio, VECS0_VECS1_INTR_MASK, ~dmask); 194 195 /* 196 * the heci2 interrupt is enabled via the same register as the 197 * GSCCS interrupts, but it has its own mask register. 198 */ 199 if (xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_OTHER)) { 200 gsc_mask = irqs | GSC_ER_COMPLETE; 201 heci_mask = GSC_IRQ_INTF(1); 202 } else if (xe->info.has_heci_gscfi) { 203 gsc_mask = GSC_IRQ_INTF(1); 204 } 205 206 if (gsc_mask) { 207 xe_mmio_write32(mmio, GUNIT_GSC_INTR_ENABLE, gsc_mask | heci_mask); 208 xe_mmio_write32(mmio, GUNIT_GSC_INTR_MASK, ~gsc_mask); 209 } 210 if (heci_mask) 211 xe_mmio_write32(mmio, HECI2_RSVD_INTR_MASK, ~(heci_mask << 16)); 212 213 if (xe_pxp_is_supported(xe)) { 214 u32 kcr_mask = KCR_PXP_STATE_TERMINATED_INTERRUPT | 215 KCR_APP_TERMINATED_PER_FW_REQ_INTERRUPT | 216 KCR_PXP_STATE_RESET_COMPLETE_INTERRUPT; 217 218 xe_mmio_write32(mmio, CRYPTO_RSVD_INTR_ENABLE, kcr_mask << 16); 219 xe_mmio_write32(mmio, CRYPTO_RSVD_INTR_MASK, ~(kcr_mask << 16)); 220 } 221 } 222 } 223 224 static u32 225 gt_engine_identity(struct xe_device *xe, 226 struct xe_mmio *mmio, 227 const unsigned int bank, 228 const unsigned int bit) 229 { 230 u32 timeout_ts; 231 u32 ident; 232 233 lockdep_assert_held(&xe->irq.lock); 234 235 xe_mmio_write32(mmio, IIR_REG_SELECTOR(bank), BIT(bit)); 236 237 /* 238 * NB: Specs do not specify how long to spin wait, 239 * so we do ~100us as an educated guess. 240 */ 241 timeout_ts = (local_clock() >> 10) + 100; 242 do { 243 ident = xe_mmio_read32(mmio, INTR_IDENTITY_REG(bank)); 244 } while (!(ident & INTR_DATA_VALID) && 245 !time_after32(local_clock() >> 10, timeout_ts)); 246 247 if (unlikely(!(ident & INTR_DATA_VALID))) { 248 drm_err(&xe->drm, "INTR_IDENTITY_REG%u:%u 0x%08x not valid!\n", 249 bank, bit, ident); 250 return 0; 251 } 252 253 xe_mmio_write32(mmio, INTR_IDENTITY_REG(bank), ident); 254 255 return ident; 256 } 257 258 #define OTHER_MEDIA_GUC_INSTANCE 16 259 260 static void 261 gt_other_irq_handler(struct xe_gt *gt, const u8 instance, const u16 iir) 262 { 263 if (instance == OTHER_GUC_INSTANCE && !xe_gt_is_media_type(gt)) 264 return xe_guc_irq_handler(>->uc.guc, iir); 265 if (instance == OTHER_MEDIA_GUC_INSTANCE && xe_gt_is_media_type(gt)) 266 return xe_guc_irq_handler(>->uc.guc, iir); 267 if (instance == OTHER_GSC_HECI2_INSTANCE && xe_gt_is_media_type(gt)) 268 return xe_gsc_proxy_irq_handler(>->uc.gsc, iir); 269 270 if (instance != OTHER_GUC_INSTANCE && 271 instance != OTHER_MEDIA_GUC_INSTANCE) { 272 WARN_ONCE(1, "unhandled other interrupt instance=0x%x, iir=0x%x\n", 273 instance, iir); 274 } 275 } 276 277 static struct xe_gt *pick_engine_gt(struct xe_tile *tile, 278 enum xe_engine_class class, 279 unsigned int instance) 280 { 281 struct xe_device *xe = tile_to_xe(tile); 282 283 if (MEDIA_VER(xe) < 13) 284 return tile->primary_gt; 285 286 switch (class) { 287 case XE_ENGINE_CLASS_VIDEO_DECODE: 288 case XE_ENGINE_CLASS_VIDEO_ENHANCE: 289 return tile->media_gt; 290 case XE_ENGINE_CLASS_OTHER: 291 switch (instance) { 292 case OTHER_MEDIA_GUC_INSTANCE: 293 case OTHER_GSC_INSTANCE: 294 case OTHER_GSC_HECI2_INSTANCE: 295 return tile->media_gt; 296 default: 297 break; 298 } 299 fallthrough; 300 default: 301 return tile->primary_gt; 302 } 303 } 304 305 static void gt_irq_handler(struct xe_tile *tile, 306 u32 master_ctl, unsigned long *intr_dw, 307 u32 *identity) 308 { 309 struct xe_device *xe = tile_to_xe(tile); 310 struct xe_mmio *mmio = &tile->mmio; 311 unsigned int bank, bit; 312 u16 instance, intr_vec; 313 enum xe_engine_class class; 314 struct xe_hw_engine *hwe; 315 316 spin_lock(&xe->irq.lock); 317 318 for (bank = 0; bank < 2; bank++) { 319 if (!(master_ctl & GT_DW_IRQ(bank))) 320 continue; 321 322 intr_dw[bank] = xe_mmio_read32(mmio, GT_INTR_DW(bank)); 323 for_each_set_bit(bit, intr_dw + bank, 32) 324 identity[bit] = gt_engine_identity(xe, mmio, bank, bit); 325 xe_mmio_write32(mmio, GT_INTR_DW(bank), intr_dw[bank]); 326 327 for_each_set_bit(bit, intr_dw + bank, 32) { 328 struct xe_gt *engine_gt; 329 330 class = INTR_ENGINE_CLASS(identity[bit]); 331 instance = INTR_ENGINE_INSTANCE(identity[bit]); 332 intr_vec = INTR_ENGINE_INTR(identity[bit]); 333 334 engine_gt = pick_engine_gt(tile, class, instance); 335 336 hwe = xe_gt_hw_engine(engine_gt, class, instance, false); 337 if (hwe) { 338 xe_hw_engine_handle_irq(hwe, intr_vec); 339 continue; 340 } 341 342 if (class == XE_ENGINE_CLASS_OTHER) { 343 /* 344 * HECI GSCFI interrupts come from outside of GT. 345 * KCR irqs come from inside GT but are handled 346 * by the global PXP subsystem. 347 */ 348 if (xe->info.has_heci_gscfi && instance == OTHER_GSC_INSTANCE) 349 xe_heci_gsc_irq_handler(xe, intr_vec); 350 else if (instance == OTHER_KCR_INSTANCE) 351 xe_pxp_irq_handler(xe, intr_vec); 352 else 353 gt_other_irq_handler(engine_gt, instance, intr_vec); 354 } 355 } 356 } 357 358 spin_unlock(&xe->irq.lock); 359 } 360 361 /* 362 * Top-level interrupt handler for Xe_LP platforms (which did not have 363 * a "master tile" interrupt register. 364 */ 365 static irqreturn_t xelp_irq_handler(int irq, void *arg) 366 { 367 struct xe_device *xe = arg; 368 struct xe_tile *tile = xe_device_get_root_tile(xe); 369 u32 master_ctl, gu_misc_iir; 370 unsigned long intr_dw[2]; 371 u32 identity[32]; 372 373 if (!atomic_read(&xe->irq.enabled)) 374 return IRQ_NONE; 375 376 master_ctl = xelp_intr_disable(xe); 377 if (!master_ctl) { 378 xelp_intr_enable(xe, false); 379 return IRQ_NONE; 380 } 381 382 gt_irq_handler(tile, master_ctl, intr_dw, identity); 383 384 xe_display_irq_handler(xe, master_ctl); 385 386 gu_misc_iir = gu_misc_irq_ack(xe, master_ctl); 387 388 xelp_intr_enable(xe, false); 389 390 xe_display_irq_enable(xe, gu_misc_iir); 391 392 return IRQ_HANDLED; 393 } 394 395 static u32 dg1_intr_disable(struct xe_device *xe) 396 { 397 struct xe_mmio *mmio = xe_root_tile_mmio(xe); 398 u32 val; 399 400 /* First disable interrupts */ 401 xe_mmio_write32(mmio, DG1_MSTR_TILE_INTR, 0); 402 403 /* Get the indication levels and ack the master unit */ 404 val = xe_mmio_read32(mmio, DG1_MSTR_TILE_INTR); 405 if (unlikely(!val)) 406 return 0; 407 408 xe_mmio_write32(mmio, DG1_MSTR_TILE_INTR, val); 409 410 return val; 411 } 412 413 static void dg1_intr_enable(struct xe_device *xe, bool stall) 414 { 415 struct xe_mmio *mmio = xe_root_tile_mmio(xe); 416 417 xe_mmio_write32(mmio, DG1_MSTR_TILE_INTR, DG1_MSTR_IRQ); 418 if (stall) 419 xe_mmio_read32(mmio, DG1_MSTR_TILE_INTR); 420 } 421 422 /* 423 * Top-level interrupt handler for Xe_LP+ and beyond. These platforms have 424 * a "master tile" interrupt register which must be consulted before the 425 * "graphics master" interrupt register. 426 */ 427 static irqreturn_t dg1_irq_handler(int irq, void *arg) 428 { 429 struct xe_device *xe = arg; 430 struct xe_tile *tile; 431 u32 master_tile_ctl, master_ctl = 0, gu_misc_iir = 0; 432 unsigned long intr_dw[2]; 433 u32 identity[32]; 434 u8 id; 435 436 /* TODO: This really shouldn't be copied+pasted */ 437 438 if (!atomic_read(&xe->irq.enabled)) 439 return IRQ_NONE; 440 441 master_tile_ctl = dg1_intr_disable(xe); 442 if (!master_tile_ctl) { 443 dg1_intr_enable(xe, false); 444 return IRQ_NONE; 445 } 446 447 for_each_tile(tile, xe, id) { 448 struct xe_mmio *mmio = &tile->mmio; 449 450 if ((master_tile_ctl & DG1_MSTR_TILE(tile->id)) == 0) 451 continue; 452 453 master_ctl = xe_mmio_read32(mmio, GFX_MSTR_IRQ); 454 455 /* 456 * We might be in irq handler just when PCIe DPC is initiated 457 * and all MMIO reads will be returned with all 1's. Ignore this 458 * irq as device is inaccessible. 459 */ 460 if (master_ctl == REG_GENMASK(31, 0)) { 461 drm_dbg(&tile_to_xe(tile)->drm, 462 "Ignore this IRQ as device might be in DPC containment.\n"); 463 return IRQ_HANDLED; 464 } 465 466 xe_mmio_write32(mmio, GFX_MSTR_IRQ, master_ctl); 467 468 gt_irq_handler(tile, master_ctl, intr_dw, identity); 469 470 /* 471 * Display interrupts (including display backlight operations 472 * that get reported as Gunit GSE) would only be hooked up to 473 * the primary tile. 474 */ 475 if (id == 0) { 476 if (xe->info.has_heci_cscfi) 477 xe_heci_csc_irq_handler(xe, master_ctl); 478 xe_display_irq_handler(xe, master_ctl); 479 gu_misc_iir = gu_misc_irq_ack(xe, master_ctl); 480 } 481 } 482 483 dg1_intr_enable(xe, false); 484 xe_display_irq_enable(xe, gu_misc_iir); 485 486 return IRQ_HANDLED; 487 } 488 489 static void gt_irq_reset(struct xe_tile *tile) 490 { 491 struct xe_mmio *mmio = &tile->mmio; 492 493 u32 ccs_mask = xe_hw_engine_mask_per_class(tile->primary_gt, 494 XE_ENGINE_CLASS_COMPUTE); 495 u32 bcs_mask = xe_hw_engine_mask_per_class(tile->primary_gt, 496 XE_ENGINE_CLASS_COPY); 497 498 /* Disable RCS, BCS, VCS and VECS class engines. */ 499 xe_mmio_write32(mmio, RENDER_COPY_INTR_ENABLE, 0); 500 xe_mmio_write32(mmio, VCS_VECS_INTR_ENABLE, 0); 501 if (ccs_mask) 502 xe_mmio_write32(mmio, CCS_RSVD_INTR_ENABLE, 0); 503 504 /* Restore masks irqs on RCS, BCS, VCS and VECS engines. */ 505 xe_mmio_write32(mmio, RCS0_RSVD_INTR_MASK, ~0); 506 xe_mmio_write32(mmio, BCS_RSVD_INTR_MASK, ~0); 507 if (bcs_mask & (BIT(1)|BIT(2))) 508 xe_mmio_write32(mmio, XEHPC_BCS1_BCS2_INTR_MASK, ~0); 509 if (bcs_mask & (BIT(3)|BIT(4))) 510 xe_mmio_write32(mmio, XEHPC_BCS3_BCS4_INTR_MASK, ~0); 511 if (bcs_mask & (BIT(5)|BIT(6))) 512 xe_mmio_write32(mmio, XEHPC_BCS5_BCS6_INTR_MASK, ~0); 513 if (bcs_mask & (BIT(7)|BIT(8))) 514 xe_mmio_write32(mmio, XEHPC_BCS7_BCS8_INTR_MASK, ~0); 515 xe_mmio_write32(mmio, VCS0_VCS1_INTR_MASK, ~0); 516 xe_mmio_write32(mmio, VCS2_VCS3_INTR_MASK, ~0); 517 xe_mmio_write32(mmio, VECS0_VECS1_INTR_MASK, ~0); 518 if (ccs_mask & (BIT(0)|BIT(1))) 519 xe_mmio_write32(mmio, CCS0_CCS1_INTR_MASK, ~0); 520 if (ccs_mask & (BIT(2)|BIT(3))) 521 xe_mmio_write32(mmio, CCS2_CCS3_INTR_MASK, ~0); 522 523 if ((tile->media_gt && 524 xe_hw_engine_mask_per_class(tile->media_gt, XE_ENGINE_CLASS_OTHER)) || 525 tile_to_xe(tile)->info.has_heci_gscfi) { 526 xe_mmio_write32(mmio, GUNIT_GSC_INTR_ENABLE, 0); 527 xe_mmio_write32(mmio, GUNIT_GSC_INTR_MASK, ~0); 528 xe_mmio_write32(mmio, HECI2_RSVD_INTR_MASK, ~0); 529 xe_mmio_write32(mmio, CRYPTO_RSVD_INTR_ENABLE, 0); 530 xe_mmio_write32(mmio, CRYPTO_RSVD_INTR_MASK, ~0); 531 } 532 533 xe_mmio_write32(mmio, GPM_WGBOXPERF_INTR_ENABLE, 0); 534 xe_mmio_write32(mmio, GPM_WGBOXPERF_INTR_MASK, ~0); 535 xe_mmio_write32(mmio, GUC_SG_INTR_ENABLE, 0); 536 xe_mmio_write32(mmio, GUC_SG_INTR_MASK, ~0); 537 } 538 539 static void xelp_irq_reset(struct xe_tile *tile) 540 { 541 xelp_intr_disable(tile_to_xe(tile)); 542 543 gt_irq_reset(tile); 544 545 if (IS_SRIOV_VF(tile_to_xe(tile))) 546 return; 547 548 mask_and_disable(tile, PCU_IRQ_OFFSET); 549 } 550 551 static void dg1_irq_reset(struct xe_tile *tile) 552 { 553 if (tile->id == 0) 554 dg1_intr_disable(tile_to_xe(tile)); 555 556 gt_irq_reset(tile); 557 558 if (IS_SRIOV_VF(tile_to_xe(tile))) 559 return; 560 561 mask_and_disable(tile, PCU_IRQ_OFFSET); 562 } 563 564 static void dg1_irq_reset_mstr(struct xe_tile *tile) 565 { 566 struct xe_mmio *mmio = &tile->mmio; 567 568 xe_mmio_write32(mmio, GFX_MSTR_IRQ, ~0); 569 } 570 571 static void vf_irq_reset(struct xe_device *xe) 572 { 573 struct xe_tile *tile; 574 unsigned int id; 575 576 xe_assert(xe, IS_SRIOV_VF(xe)); 577 578 if (GRAPHICS_VERx100(xe) < 1210) 579 xelp_intr_disable(xe); 580 else 581 xe_assert(xe, xe_device_has_memirq(xe)); 582 583 for_each_tile(tile, xe, id) { 584 if (xe_device_has_memirq(xe)) 585 xe_memirq_reset(&tile->memirq); 586 else 587 gt_irq_reset(tile); 588 } 589 } 590 591 static void xe_irq_reset(struct xe_device *xe) 592 { 593 struct xe_tile *tile; 594 u8 id; 595 596 if (IS_SRIOV_VF(xe)) 597 return vf_irq_reset(xe); 598 599 if (xe_device_uses_memirq(xe)) { 600 for_each_tile(tile, xe, id) 601 xe_memirq_reset(&tile->memirq); 602 } 603 604 for_each_tile(tile, xe, id) { 605 if (GRAPHICS_VERx100(xe) >= 1210) 606 dg1_irq_reset(tile); 607 else 608 xelp_irq_reset(tile); 609 } 610 611 tile = xe_device_get_root_tile(xe); 612 mask_and_disable(tile, GU_MISC_IRQ_OFFSET); 613 xe_display_irq_reset(xe); 614 615 /* 616 * The tile's top-level status register should be the last one 617 * to be reset to avoid possible bit re-latching from lower 618 * level interrupts. 619 */ 620 if (GRAPHICS_VERx100(xe) >= 1210) { 621 for_each_tile(tile, xe, id) 622 dg1_irq_reset_mstr(tile); 623 } 624 } 625 626 static void vf_irq_postinstall(struct xe_device *xe) 627 { 628 struct xe_tile *tile; 629 unsigned int id; 630 631 for_each_tile(tile, xe, id) 632 if (xe_device_has_memirq(xe)) 633 xe_memirq_postinstall(&tile->memirq); 634 635 if (GRAPHICS_VERx100(xe) < 1210) 636 xelp_intr_enable(xe, true); 637 else 638 xe_assert(xe, xe_device_has_memirq(xe)); 639 } 640 641 static void xe_irq_postinstall(struct xe_device *xe) 642 { 643 if (IS_SRIOV_VF(xe)) 644 return vf_irq_postinstall(xe); 645 646 if (xe_device_uses_memirq(xe)) { 647 struct xe_tile *tile; 648 unsigned int id; 649 650 for_each_tile(tile, xe, id) 651 xe_memirq_postinstall(&tile->memirq); 652 } 653 654 xe_display_irq_postinstall(xe, xe_root_mmio_gt(xe)); 655 656 /* 657 * ASLE backlight operations are reported via GUnit GSE interrupts 658 * on the root tile. 659 */ 660 unmask_and_enable(xe_device_get_root_tile(xe), 661 GU_MISC_IRQ_OFFSET, GU_MISC_GSE); 662 663 /* Enable top-level interrupts */ 664 if (GRAPHICS_VERx100(xe) >= 1210) 665 dg1_intr_enable(xe, true); 666 else 667 xelp_intr_enable(xe, true); 668 } 669 670 static irqreturn_t vf_mem_irq_handler(int irq, void *arg) 671 { 672 struct xe_device *xe = arg; 673 struct xe_tile *tile; 674 unsigned int id; 675 676 if (!atomic_read(&xe->irq.enabled)) 677 return IRQ_NONE; 678 679 for_each_tile(tile, xe, id) 680 xe_memirq_handler(&tile->memirq); 681 682 return IRQ_HANDLED; 683 } 684 685 static irq_handler_t xe_irq_handler(struct xe_device *xe) 686 { 687 if (IS_SRIOV_VF(xe) && xe_device_has_memirq(xe)) 688 return vf_mem_irq_handler; 689 690 if (GRAPHICS_VERx100(xe) >= 1210) 691 return dg1_irq_handler; 692 else 693 return xelp_irq_handler; 694 } 695 696 static int xe_irq_msi_request_irqs(struct xe_device *xe) 697 { 698 struct pci_dev *pdev = to_pci_dev(xe->drm.dev); 699 irq_handler_t irq_handler; 700 int irq, err; 701 702 irq_handler = xe_irq_handler(xe); 703 if (!irq_handler) { 704 drm_err(&xe->drm, "No supported interrupt handler"); 705 return -EINVAL; 706 } 707 708 irq = pci_irq_vector(pdev, 0); 709 err = request_irq(irq, irq_handler, IRQF_SHARED, DRIVER_NAME, xe); 710 if (err < 0) { 711 drm_err(&xe->drm, "Failed to request MSI IRQ %d\n", err); 712 return err; 713 } 714 715 return 0; 716 } 717 718 static void xe_irq_msi_free(struct xe_device *xe) 719 { 720 struct pci_dev *pdev = to_pci_dev(xe->drm.dev); 721 int irq; 722 723 irq = pci_irq_vector(pdev, 0); 724 free_irq(irq, xe); 725 } 726 727 static void irq_uninstall(void *arg) 728 { 729 struct xe_device *xe = arg; 730 731 if (!atomic_xchg(&xe->irq.enabled, 0)) 732 return; 733 734 xe_irq_reset(xe); 735 736 if (xe_device_has_msix(xe)) 737 xe_irq_msix_free(xe); 738 else 739 xe_irq_msi_free(xe); 740 } 741 742 int xe_irq_init(struct xe_device *xe) 743 { 744 spin_lock_init(&xe->irq.lock); 745 746 return xe_irq_msix_init(xe); 747 } 748 749 int xe_irq_install(struct xe_device *xe) 750 { 751 struct pci_dev *pdev = to_pci_dev(xe->drm.dev); 752 unsigned int irq_flags = PCI_IRQ_MSI; 753 int nvec = 1; 754 int err; 755 756 xe_irq_reset(xe); 757 758 if (xe_device_has_msix(xe)) { 759 nvec = xe->irq.msix.nvec; 760 irq_flags = PCI_IRQ_MSIX; 761 } 762 763 err = pci_alloc_irq_vectors(pdev, nvec, nvec, irq_flags); 764 if (err < 0) { 765 drm_err(&xe->drm, "Failed to allocate IRQ vectors: %d\n", err); 766 return err; 767 } 768 769 err = xe_device_has_msix(xe) ? xe_irq_msix_request_irqs(xe) : 770 xe_irq_msi_request_irqs(xe); 771 if (err) 772 return err; 773 774 atomic_set(&xe->irq.enabled, 1); 775 776 xe_irq_postinstall(xe); 777 778 return devm_add_action_or_reset(xe->drm.dev, irq_uninstall, xe); 779 } 780 781 static void xe_irq_msi_synchronize_irq(struct xe_device *xe) 782 { 783 synchronize_irq(to_pci_dev(xe->drm.dev)->irq); 784 } 785 786 void xe_irq_suspend(struct xe_device *xe) 787 { 788 atomic_set(&xe->irq.enabled, 0); /* no new irqs */ 789 790 /* flush irqs */ 791 if (xe_device_has_msix(xe)) 792 xe_irq_msix_synchronize_irq(xe); 793 else 794 xe_irq_msi_synchronize_irq(xe); 795 xe_irq_reset(xe); /* turn irqs off */ 796 } 797 798 void xe_irq_resume(struct xe_device *xe) 799 { 800 struct xe_gt *gt; 801 int id; 802 803 /* 804 * lock not needed: 805 * 1. no irq will arrive before the postinstall 806 * 2. display is not yet resumed 807 */ 808 atomic_set(&xe->irq.enabled, 1); 809 xe_irq_reset(xe); 810 xe_irq_postinstall(xe); /* turn irqs on */ 811 812 for_each_gt(gt, xe, id) 813 xe_irq_enable_hwe(gt); 814 } 815 816 /* MSI-X related definitions and functions below. */ 817 818 enum xe_irq_msix_static { 819 GUC2HOST_MSIX = 0, 820 DEFAULT_MSIX = XE_IRQ_DEFAULT_MSIX, 821 /* Must be last */ 822 NUM_OF_STATIC_MSIX, 823 }; 824 825 static int xe_irq_msix_init(struct xe_device *xe) 826 { 827 struct pci_dev *pdev = to_pci_dev(xe->drm.dev); 828 int nvec = pci_msix_vec_count(pdev); 829 830 if (nvec == -EINVAL) 831 return 0; /* MSI */ 832 833 if (nvec < 0) { 834 drm_err(&xe->drm, "Failed getting MSI-X vectors count: %d\n", nvec); 835 return nvec; 836 } 837 838 xe->irq.msix.nvec = nvec; 839 xa_init_flags(&xe->irq.msix.indexes, XA_FLAGS_ALLOC); 840 return 0; 841 } 842 843 static irqreturn_t guc2host_irq_handler(int irq, void *arg) 844 { 845 struct xe_device *xe = arg; 846 struct xe_tile *tile; 847 u8 id; 848 849 if (!atomic_read(&xe->irq.enabled)) 850 return IRQ_NONE; 851 852 for_each_tile(tile, xe, id) 853 xe_guc_irq_handler(&tile->primary_gt->uc.guc, 854 GUC_INTR_GUC2HOST); 855 856 return IRQ_HANDLED; 857 } 858 859 static irqreturn_t xe_irq_msix_default_hwe_handler(int irq, void *arg) 860 { 861 unsigned int tile_id, gt_id; 862 struct xe_device *xe = arg; 863 struct xe_memirq *memirq; 864 struct xe_hw_engine *hwe; 865 enum xe_hw_engine_id id; 866 struct xe_tile *tile; 867 struct xe_gt *gt; 868 869 if (!atomic_read(&xe->irq.enabled)) 870 return IRQ_NONE; 871 872 for_each_tile(tile, xe, tile_id) { 873 memirq = &tile->memirq; 874 if (!memirq->bo) 875 continue; 876 877 for_each_gt(gt, xe, gt_id) { 878 if (gt->tile != tile) 879 continue; 880 881 for_each_hw_engine(hwe, gt, id) 882 xe_memirq_hwe_handler(memirq, hwe); 883 } 884 } 885 886 return IRQ_HANDLED; 887 } 888 889 static int xe_irq_msix_alloc_vector(struct xe_device *xe, void *irq_buf, 890 bool dynamic_msix, u16 *msix) 891 { 892 struct xa_limit limit; 893 int ret; 894 u32 id; 895 896 limit = (dynamic_msix) ? XA_LIMIT(NUM_OF_STATIC_MSIX, xe->irq.msix.nvec - 1) : 897 XA_LIMIT(*msix, *msix); 898 ret = xa_alloc(&xe->irq.msix.indexes, &id, irq_buf, limit, GFP_KERNEL); 899 if (ret) 900 return ret; 901 902 if (dynamic_msix) 903 *msix = id; 904 905 return 0; 906 } 907 908 static void xe_irq_msix_release_vector(struct xe_device *xe, u16 msix) 909 { 910 xa_erase(&xe->irq.msix.indexes, msix); 911 } 912 913 static int xe_irq_msix_request_irq_internal(struct xe_device *xe, irq_handler_t handler, 914 void *irq_buf, const char *name, u16 msix) 915 { 916 struct pci_dev *pdev = to_pci_dev(xe->drm.dev); 917 int ret, irq; 918 919 irq = pci_irq_vector(pdev, msix); 920 if (irq < 0) 921 return irq; 922 923 ret = request_irq(irq, handler, IRQF_SHARED, name, irq_buf); 924 if (ret < 0) 925 return ret; 926 927 return 0; 928 } 929 930 int xe_irq_msix_request_irq(struct xe_device *xe, irq_handler_t handler, void *irq_buf, 931 const char *name, bool dynamic_msix, u16 *msix) 932 { 933 int ret; 934 935 ret = xe_irq_msix_alloc_vector(xe, irq_buf, dynamic_msix, msix); 936 if (ret) 937 return ret; 938 939 ret = xe_irq_msix_request_irq_internal(xe, handler, irq_buf, name, *msix); 940 if (ret) { 941 drm_err(&xe->drm, "Failed to request IRQ for MSI-X %u\n", *msix); 942 xe_irq_msix_release_vector(xe, *msix); 943 return ret; 944 } 945 946 return 0; 947 } 948 949 void xe_irq_msix_free_irq(struct xe_device *xe, u16 msix) 950 { 951 struct pci_dev *pdev = to_pci_dev(xe->drm.dev); 952 int irq; 953 void *irq_buf; 954 955 irq_buf = xa_load(&xe->irq.msix.indexes, msix); 956 if (!irq_buf) 957 return; 958 959 irq = pci_irq_vector(pdev, msix); 960 if (irq < 0) { 961 drm_err(&xe->drm, "MSI-X %u can't be released, there is no matching IRQ\n", msix); 962 return; 963 } 964 965 free_irq(irq, irq_buf); 966 xe_irq_msix_release_vector(xe, msix); 967 } 968 969 int xe_irq_msix_request_irqs(struct xe_device *xe) 970 { 971 int err; 972 u16 msix; 973 974 msix = GUC2HOST_MSIX; 975 err = xe_irq_msix_request_irq(xe, guc2host_irq_handler, xe, 976 DRIVER_NAME "-guc2host", false, &msix); 977 if (err) 978 return err; 979 980 msix = DEFAULT_MSIX; 981 err = xe_irq_msix_request_irq(xe, xe_irq_msix_default_hwe_handler, xe, 982 DRIVER_NAME "-default-msix", false, &msix); 983 if (err) { 984 xe_irq_msix_free_irq(xe, GUC2HOST_MSIX); 985 return err; 986 } 987 988 return 0; 989 } 990 991 void xe_irq_msix_free(struct xe_device *xe) 992 { 993 unsigned long msix; 994 u32 *dummy; 995 996 xa_for_each(&xe->irq.msix.indexes, msix, dummy) 997 xe_irq_msix_free_irq(xe, msix); 998 xa_destroy(&xe->irq.msix.indexes); 999 } 1000 1001 void xe_irq_msix_synchronize_irq(struct xe_device *xe) 1002 { 1003 struct pci_dev *pdev = to_pci_dev(xe->drm.dev); 1004 unsigned long msix; 1005 u32 *dummy; 1006 1007 xa_for_each(&xe->irq.msix.indexes, msix, dummy) 1008 synchronize_irq(pci_irq_vector(pdev, msix)); 1009 } 1010