1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2021 Intel Corporation
4 */
5
6 #include "xe_irq.h"
7
8 #include <linux/sched/clock.h>
9
10 #include <drm/drm_managed.h>
11
12 #include "display/xe_display.h"
13 #include "regs/xe_irq_regs.h"
14 #include "xe_device.h"
15 #include "xe_drv.h"
16 #include "xe_gsc_proxy.h"
17 #include "xe_gt.h"
18 #include "xe_guc.h"
19 #include "xe_hw_engine.h"
20 #include "xe_memirq.h"
21 #include "xe_mmio.h"
22 #include "xe_sriov.h"
23
24 /*
25 * Interrupt registers for a unit are always consecutive and ordered
26 * ISR, IMR, IIR, IER.
27 */
28 #define IMR(offset) XE_REG(offset + 0x4)
29 #define IIR(offset) XE_REG(offset + 0x8)
30 #define IER(offset) XE_REG(offset + 0xc)
31
assert_iir_is_zero(struct xe_mmio * mmio,struct xe_reg reg)32 static void assert_iir_is_zero(struct xe_mmio *mmio, struct xe_reg reg)
33 {
34 u32 val = xe_mmio_read32(mmio, reg);
35
36 if (val == 0)
37 return;
38
39 drm_WARN(&mmio->tile->xe->drm, 1,
40 "Interrupt register 0x%x is not zero: 0x%08x\n",
41 reg.addr, val);
42 xe_mmio_write32(mmio, reg, 0xffffffff);
43 xe_mmio_read32(mmio, reg);
44 xe_mmio_write32(mmio, reg, 0xffffffff);
45 xe_mmio_read32(mmio, reg);
46 }
47
48 /*
49 * Unmask and enable the specified interrupts. Does not check current state,
50 * so any bits not specified here will become masked and disabled.
51 */
unmask_and_enable(struct xe_tile * tile,u32 irqregs,u32 bits)52 static void unmask_and_enable(struct xe_tile *tile, u32 irqregs, u32 bits)
53 {
54 struct xe_mmio *mmio = &tile->mmio;
55
56 /*
57 * If we're just enabling an interrupt now, it shouldn't already
58 * be raised in the IIR.
59 */
60 assert_iir_is_zero(mmio, IIR(irqregs));
61
62 xe_mmio_write32(mmio, IER(irqregs), bits);
63 xe_mmio_write32(mmio, IMR(irqregs), ~bits);
64
65 /* Posting read */
66 xe_mmio_read32(mmio, IMR(irqregs));
67 }
68
69 /* Mask and disable all interrupts. */
mask_and_disable(struct xe_tile * tile,u32 irqregs)70 static void mask_and_disable(struct xe_tile *tile, u32 irqregs)
71 {
72 struct xe_mmio *mmio = &tile->mmio;
73
74 xe_mmio_write32(mmio, IMR(irqregs), ~0);
75 /* Posting read */
76 xe_mmio_read32(mmio, IMR(irqregs));
77
78 xe_mmio_write32(mmio, IER(irqregs), 0);
79
80 /* IIR can theoretically queue up two events. Be paranoid. */
81 xe_mmio_write32(mmio, IIR(irqregs), ~0);
82 xe_mmio_read32(mmio, IIR(irqregs));
83 xe_mmio_write32(mmio, IIR(irqregs), ~0);
84 xe_mmio_read32(mmio, IIR(irqregs));
85 }
86
xelp_intr_disable(struct xe_device * xe)87 static u32 xelp_intr_disable(struct xe_device *xe)
88 {
89 struct xe_mmio *mmio = xe_root_tile_mmio(xe);
90
91 xe_mmio_write32(mmio, GFX_MSTR_IRQ, 0);
92
93 /*
94 * Now with master disabled, get a sample of level indications
95 * for this interrupt. Indications will be cleared on related acks.
96 * New indications can and will light up during processing,
97 * and will generate new interrupt after enabling master.
98 */
99 return xe_mmio_read32(mmio, GFX_MSTR_IRQ);
100 }
101
102 static u32
gu_misc_irq_ack(struct xe_device * xe,const u32 master_ctl)103 gu_misc_irq_ack(struct xe_device *xe, const u32 master_ctl)
104 {
105 struct xe_mmio *mmio = xe_root_tile_mmio(xe);
106 u32 iir;
107
108 if (!(master_ctl & GU_MISC_IRQ))
109 return 0;
110
111 iir = xe_mmio_read32(mmio, IIR(GU_MISC_IRQ_OFFSET));
112 if (likely(iir))
113 xe_mmio_write32(mmio, IIR(GU_MISC_IRQ_OFFSET), iir);
114
115 return iir;
116 }
117
xelp_intr_enable(struct xe_device * xe,bool stall)118 static inline void xelp_intr_enable(struct xe_device *xe, bool stall)
119 {
120 struct xe_mmio *mmio = xe_root_tile_mmio(xe);
121
122 xe_mmio_write32(mmio, GFX_MSTR_IRQ, MASTER_IRQ);
123 if (stall)
124 xe_mmio_read32(mmio, GFX_MSTR_IRQ);
125 }
126
127 /* Enable/unmask the HWE interrupts for a specific GT's engines. */
xe_irq_enable_hwe(struct xe_gt * gt)128 void xe_irq_enable_hwe(struct xe_gt *gt)
129 {
130 struct xe_device *xe = gt_to_xe(gt);
131 struct xe_mmio *mmio = >->mmio;
132 u32 ccs_mask, bcs_mask;
133 u32 irqs, dmask, smask;
134 u32 gsc_mask = 0;
135 u32 heci_mask = 0;
136
137 if (xe_device_uses_memirq(xe))
138 return;
139
140 if (xe_device_uc_enabled(xe)) {
141 irqs = GT_RENDER_USER_INTERRUPT |
142 GT_RENDER_PIPECTL_NOTIFY_INTERRUPT;
143 } else {
144 irqs = GT_RENDER_USER_INTERRUPT |
145 GT_CS_MASTER_ERROR_INTERRUPT |
146 GT_CONTEXT_SWITCH_INTERRUPT |
147 GT_WAIT_SEMAPHORE_INTERRUPT;
148 }
149
150 ccs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_COMPUTE);
151 bcs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_COPY);
152
153 dmask = irqs << 16 | irqs;
154 smask = irqs << 16;
155
156 if (!xe_gt_is_media_type(gt)) {
157 /* Enable interrupts for each engine class */
158 xe_mmio_write32(mmio, RENDER_COPY_INTR_ENABLE, dmask);
159 if (ccs_mask)
160 xe_mmio_write32(mmio, CCS_RSVD_INTR_ENABLE, smask);
161
162 /* Unmask interrupts for each engine instance */
163 xe_mmio_write32(mmio, RCS0_RSVD_INTR_MASK, ~smask);
164 xe_mmio_write32(mmio, BCS_RSVD_INTR_MASK, ~smask);
165 if (bcs_mask & (BIT(1)|BIT(2)))
166 xe_mmio_write32(mmio, XEHPC_BCS1_BCS2_INTR_MASK, ~dmask);
167 if (bcs_mask & (BIT(3)|BIT(4)))
168 xe_mmio_write32(mmio, XEHPC_BCS3_BCS4_INTR_MASK, ~dmask);
169 if (bcs_mask & (BIT(5)|BIT(6)))
170 xe_mmio_write32(mmio, XEHPC_BCS5_BCS6_INTR_MASK, ~dmask);
171 if (bcs_mask & (BIT(7)|BIT(8)))
172 xe_mmio_write32(mmio, XEHPC_BCS7_BCS8_INTR_MASK, ~dmask);
173 if (ccs_mask & (BIT(0)|BIT(1)))
174 xe_mmio_write32(mmio, CCS0_CCS1_INTR_MASK, ~dmask);
175 if (ccs_mask & (BIT(2)|BIT(3)))
176 xe_mmio_write32(mmio, CCS2_CCS3_INTR_MASK, ~dmask);
177 }
178
179 if (xe_gt_is_media_type(gt) || MEDIA_VER(xe) < 13) {
180 /* Enable interrupts for each engine class */
181 xe_mmio_write32(mmio, VCS_VECS_INTR_ENABLE, dmask);
182
183 /* Unmask interrupts for each engine instance */
184 xe_mmio_write32(mmio, VCS0_VCS1_INTR_MASK, ~dmask);
185 xe_mmio_write32(mmio, VCS2_VCS3_INTR_MASK, ~dmask);
186 xe_mmio_write32(mmio, VECS0_VECS1_INTR_MASK, ~dmask);
187
188 /*
189 * the heci2 interrupt is enabled via the same register as the
190 * GSCCS interrupts, but it has its own mask register.
191 */
192 if (xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_OTHER)) {
193 gsc_mask = irqs | GSC_ER_COMPLETE;
194 heci_mask = GSC_IRQ_INTF(1);
195 } else if (HAS_HECI_GSCFI(xe)) {
196 gsc_mask = GSC_IRQ_INTF(1);
197 }
198
199 if (gsc_mask) {
200 xe_mmio_write32(mmio, GUNIT_GSC_INTR_ENABLE, gsc_mask | heci_mask);
201 xe_mmio_write32(mmio, GUNIT_GSC_INTR_MASK, ~gsc_mask);
202 }
203 if (heci_mask)
204 xe_mmio_write32(mmio, HECI2_RSVD_INTR_MASK, ~(heci_mask << 16));
205 }
206 }
207
208 static u32
gt_engine_identity(struct xe_device * xe,struct xe_mmio * mmio,const unsigned int bank,const unsigned int bit)209 gt_engine_identity(struct xe_device *xe,
210 struct xe_mmio *mmio,
211 const unsigned int bank,
212 const unsigned int bit)
213 {
214 u32 timeout_ts;
215 u32 ident;
216
217 lockdep_assert_held(&xe->irq.lock);
218
219 xe_mmio_write32(mmio, IIR_REG_SELECTOR(bank), BIT(bit));
220
221 /*
222 * NB: Specs do not specify how long to spin wait,
223 * so we do ~100us as an educated guess.
224 */
225 timeout_ts = (local_clock() >> 10) + 100;
226 do {
227 ident = xe_mmio_read32(mmio, INTR_IDENTITY_REG(bank));
228 } while (!(ident & INTR_DATA_VALID) &&
229 !time_after32(local_clock() >> 10, timeout_ts));
230
231 if (unlikely(!(ident & INTR_DATA_VALID))) {
232 drm_err(&xe->drm, "INTR_IDENTITY_REG%u:%u 0x%08x not valid!\n",
233 bank, bit, ident);
234 return 0;
235 }
236
237 xe_mmio_write32(mmio, INTR_IDENTITY_REG(bank), ident);
238
239 return ident;
240 }
241
242 #define OTHER_MEDIA_GUC_INSTANCE 16
243
244 static void
gt_other_irq_handler(struct xe_gt * gt,const u8 instance,const u16 iir)245 gt_other_irq_handler(struct xe_gt *gt, const u8 instance, const u16 iir)
246 {
247 if (instance == OTHER_GUC_INSTANCE && !xe_gt_is_media_type(gt))
248 return xe_guc_irq_handler(>->uc.guc, iir);
249 if (instance == OTHER_MEDIA_GUC_INSTANCE && xe_gt_is_media_type(gt))
250 return xe_guc_irq_handler(>->uc.guc, iir);
251 if (instance == OTHER_GSC_HECI2_INSTANCE && xe_gt_is_media_type(gt))
252 return xe_gsc_proxy_irq_handler(>->uc.gsc, iir);
253
254 if (instance != OTHER_GUC_INSTANCE &&
255 instance != OTHER_MEDIA_GUC_INSTANCE) {
256 WARN_ONCE(1, "unhandled other interrupt instance=0x%x, iir=0x%x\n",
257 instance, iir);
258 }
259 }
260
pick_engine_gt(struct xe_tile * tile,enum xe_engine_class class,unsigned int instance)261 static struct xe_gt *pick_engine_gt(struct xe_tile *tile,
262 enum xe_engine_class class,
263 unsigned int instance)
264 {
265 struct xe_device *xe = tile_to_xe(tile);
266
267 if (MEDIA_VER(xe) < 13)
268 return tile->primary_gt;
269
270 switch (class) {
271 case XE_ENGINE_CLASS_VIDEO_DECODE:
272 case XE_ENGINE_CLASS_VIDEO_ENHANCE:
273 return tile->media_gt;
274 case XE_ENGINE_CLASS_OTHER:
275 switch (instance) {
276 case OTHER_MEDIA_GUC_INSTANCE:
277 case OTHER_GSC_INSTANCE:
278 case OTHER_GSC_HECI2_INSTANCE:
279 return tile->media_gt;
280 default:
281 break;
282 }
283 fallthrough;
284 default:
285 return tile->primary_gt;
286 }
287 }
288
gt_irq_handler(struct xe_tile * tile,u32 master_ctl,unsigned long * intr_dw,u32 * identity)289 static void gt_irq_handler(struct xe_tile *tile,
290 u32 master_ctl, unsigned long *intr_dw,
291 u32 *identity)
292 {
293 struct xe_device *xe = tile_to_xe(tile);
294 struct xe_mmio *mmio = &tile->mmio;
295 unsigned int bank, bit;
296 u16 instance, intr_vec;
297 enum xe_engine_class class;
298 struct xe_hw_engine *hwe;
299
300 spin_lock(&xe->irq.lock);
301
302 for (bank = 0; bank < 2; bank++) {
303 if (!(master_ctl & GT_DW_IRQ(bank)))
304 continue;
305
306 intr_dw[bank] = xe_mmio_read32(mmio, GT_INTR_DW(bank));
307 for_each_set_bit(bit, intr_dw + bank, 32)
308 identity[bit] = gt_engine_identity(xe, mmio, bank, bit);
309 xe_mmio_write32(mmio, GT_INTR_DW(bank), intr_dw[bank]);
310
311 for_each_set_bit(bit, intr_dw + bank, 32) {
312 struct xe_gt *engine_gt;
313
314 class = INTR_ENGINE_CLASS(identity[bit]);
315 instance = INTR_ENGINE_INSTANCE(identity[bit]);
316 intr_vec = INTR_ENGINE_INTR(identity[bit]);
317
318 engine_gt = pick_engine_gt(tile, class, instance);
319
320 hwe = xe_gt_hw_engine(engine_gt, class, instance, false);
321 if (hwe) {
322 xe_hw_engine_handle_irq(hwe, intr_vec);
323 continue;
324 }
325
326 if (class == XE_ENGINE_CLASS_OTHER) {
327 /* HECI GSCFI interrupts come from outside of GT */
328 if (HAS_HECI_GSCFI(xe) && instance == OTHER_GSC_INSTANCE)
329 xe_heci_gsc_irq_handler(xe, intr_vec);
330 else
331 gt_other_irq_handler(engine_gt, instance, intr_vec);
332 }
333 }
334 }
335
336 spin_unlock(&xe->irq.lock);
337 }
338
339 /*
340 * Top-level interrupt handler for Xe_LP platforms (which did not have
341 * a "master tile" interrupt register.
342 */
xelp_irq_handler(int irq,void * arg)343 static irqreturn_t xelp_irq_handler(int irq, void *arg)
344 {
345 struct xe_device *xe = arg;
346 struct xe_tile *tile = xe_device_get_root_tile(xe);
347 u32 master_ctl, gu_misc_iir;
348 unsigned long intr_dw[2];
349 u32 identity[32];
350
351 spin_lock(&xe->irq.lock);
352 if (!xe->irq.enabled) {
353 spin_unlock(&xe->irq.lock);
354 return IRQ_NONE;
355 }
356 spin_unlock(&xe->irq.lock);
357
358 master_ctl = xelp_intr_disable(xe);
359 if (!master_ctl) {
360 xelp_intr_enable(xe, false);
361 return IRQ_NONE;
362 }
363
364 gt_irq_handler(tile, master_ctl, intr_dw, identity);
365
366 xe_display_irq_handler(xe, master_ctl);
367
368 gu_misc_iir = gu_misc_irq_ack(xe, master_ctl);
369
370 xelp_intr_enable(xe, false);
371
372 xe_display_irq_enable(xe, gu_misc_iir);
373
374 return IRQ_HANDLED;
375 }
376
dg1_intr_disable(struct xe_device * xe)377 static u32 dg1_intr_disable(struct xe_device *xe)
378 {
379 struct xe_mmio *mmio = xe_root_tile_mmio(xe);
380 u32 val;
381
382 /* First disable interrupts */
383 xe_mmio_write32(mmio, DG1_MSTR_TILE_INTR, 0);
384
385 /* Get the indication levels and ack the master unit */
386 val = xe_mmio_read32(mmio, DG1_MSTR_TILE_INTR);
387 if (unlikely(!val))
388 return 0;
389
390 xe_mmio_write32(mmio, DG1_MSTR_TILE_INTR, val);
391
392 return val;
393 }
394
dg1_intr_enable(struct xe_device * xe,bool stall)395 static void dg1_intr_enable(struct xe_device *xe, bool stall)
396 {
397 struct xe_mmio *mmio = xe_root_tile_mmio(xe);
398
399 xe_mmio_write32(mmio, DG1_MSTR_TILE_INTR, DG1_MSTR_IRQ);
400 if (stall)
401 xe_mmio_read32(mmio, DG1_MSTR_TILE_INTR);
402 }
403
404 /*
405 * Top-level interrupt handler for Xe_LP+ and beyond. These platforms have
406 * a "master tile" interrupt register which must be consulted before the
407 * "graphics master" interrupt register.
408 */
dg1_irq_handler(int irq,void * arg)409 static irqreturn_t dg1_irq_handler(int irq, void *arg)
410 {
411 struct xe_device *xe = arg;
412 struct xe_tile *tile;
413 u32 master_tile_ctl, master_ctl = 0, gu_misc_iir = 0;
414 unsigned long intr_dw[2];
415 u32 identity[32];
416 u8 id;
417
418 /* TODO: This really shouldn't be copied+pasted */
419
420 spin_lock(&xe->irq.lock);
421 if (!xe->irq.enabled) {
422 spin_unlock(&xe->irq.lock);
423 return IRQ_NONE;
424 }
425 spin_unlock(&xe->irq.lock);
426
427 master_tile_ctl = dg1_intr_disable(xe);
428 if (!master_tile_ctl) {
429 dg1_intr_enable(xe, false);
430 return IRQ_NONE;
431 }
432
433 for_each_tile(tile, xe, id) {
434 struct xe_mmio *mmio = &tile->mmio;
435
436 if ((master_tile_ctl & DG1_MSTR_TILE(tile->id)) == 0)
437 continue;
438
439 master_ctl = xe_mmio_read32(mmio, GFX_MSTR_IRQ);
440
441 /*
442 * We might be in irq handler just when PCIe DPC is initiated
443 * and all MMIO reads will be returned with all 1's. Ignore this
444 * irq as device is inaccessible.
445 */
446 if (master_ctl == REG_GENMASK(31, 0)) {
447 drm_dbg(&tile_to_xe(tile)->drm,
448 "Ignore this IRQ as device might be in DPC containment.\n");
449 return IRQ_HANDLED;
450 }
451
452 xe_mmio_write32(mmio, GFX_MSTR_IRQ, master_ctl);
453
454 gt_irq_handler(tile, master_ctl, intr_dw, identity);
455
456 /*
457 * Display interrupts (including display backlight operations
458 * that get reported as Gunit GSE) would only be hooked up to
459 * the primary tile.
460 */
461 if (id == 0) {
462 if (HAS_HECI_CSCFI(xe))
463 xe_heci_csc_irq_handler(xe, master_ctl);
464 xe_display_irq_handler(xe, master_ctl);
465 gu_misc_iir = gu_misc_irq_ack(xe, master_ctl);
466 }
467 }
468
469 dg1_intr_enable(xe, false);
470 xe_display_irq_enable(xe, gu_misc_iir);
471
472 return IRQ_HANDLED;
473 }
474
gt_irq_reset(struct xe_tile * tile)475 static void gt_irq_reset(struct xe_tile *tile)
476 {
477 struct xe_mmio *mmio = &tile->mmio;
478
479 u32 ccs_mask = xe_hw_engine_mask_per_class(tile->primary_gt,
480 XE_ENGINE_CLASS_COMPUTE);
481 u32 bcs_mask = xe_hw_engine_mask_per_class(tile->primary_gt,
482 XE_ENGINE_CLASS_COPY);
483
484 /* Disable RCS, BCS, VCS and VECS class engines. */
485 xe_mmio_write32(mmio, RENDER_COPY_INTR_ENABLE, 0);
486 xe_mmio_write32(mmio, VCS_VECS_INTR_ENABLE, 0);
487 if (ccs_mask)
488 xe_mmio_write32(mmio, CCS_RSVD_INTR_ENABLE, 0);
489
490 /* Restore masks irqs on RCS, BCS, VCS and VECS engines. */
491 xe_mmio_write32(mmio, RCS0_RSVD_INTR_MASK, ~0);
492 xe_mmio_write32(mmio, BCS_RSVD_INTR_MASK, ~0);
493 if (bcs_mask & (BIT(1)|BIT(2)))
494 xe_mmio_write32(mmio, XEHPC_BCS1_BCS2_INTR_MASK, ~0);
495 if (bcs_mask & (BIT(3)|BIT(4)))
496 xe_mmio_write32(mmio, XEHPC_BCS3_BCS4_INTR_MASK, ~0);
497 if (bcs_mask & (BIT(5)|BIT(6)))
498 xe_mmio_write32(mmio, XEHPC_BCS5_BCS6_INTR_MASK, ~0);
499 if (bcs_mask & (BIT(7)|BIT(8)))
500 xe_mmio_write32(mmio, XEHPC_BCS7_BCS8_INTR_MASK, ~0);
501 xe_mmio_write32(mmio, VCS0_VCS1_INTR_MASK, ~0);
502 xe_mmio_write32(mmio, VCS2_VCS3_INTR_MASK, ~0);
503 xe_mmio_write32(mmio, VECS0_VECS1_INTR_MASK, ~0);
504 if (ccs_mask & (BIT(0)|BIT(1)))
505 xe_mmio_write32(mmio, CCS0_CCS1_INTR_MASK, ~0);
506 if (ccs_mask & (BIT(2)|BIT(3)))
507 xe_mmio_write32(mmio, CCS2_CCS3_INTR_MASK, ~0);
508
509 if ((tile->media_gt &&
510 xe_hw_engine_mask_per_class(tile->media_gt, XE_ENGINE_CLASS_OTHER)) ||
511 HAS_HECI_GSCFI(tile_to_xe(tile))) {
512 xe_mmio_write32(mmio, GUNIT_GSC_INTR_ENABLE, 0);
513 xe_mmio_write32(mmio, GUNIT_GSC_INTR_MASK, ~0);
514 xe_mmio_write32(mmio, HECI2_RSVD_INTR_MASK, ~0);
515 }
516
517 xe_mmio_write32(mmio, GPM_WGBOXPERF_INTR_ENABLE, 0);
518 xe_mmio_write32(mmio, GPM_WGBOXPERF_INTR_MASK, ~0);
519 xe_mmio_write32(mmio, GUC_SG_INTR_ENABLE, 0);
520 xe_mmio_write32(mmio, GUC_SG_INTR_MASK, ~0);
521 }
522
xelp_irq_reset(struct xe_tile * tile)523 static void xelp_irq_reset(struct xe_tile *tile)
524 {
525 xelp_intr_disable(tile_to_xe(tile));
526
527 gt_irq_reset(tile);
528
529 if (IS_SRIOV_VF(tile_to_xe(tile)))
530 return;
531
532 mask_and_disable(tile, PCU_IRQ_OFFSET);
533 }
534
dg1_irq_reset(struct xe_tile * tile)535 static void dg1_irq_reset(struct xe_tile *tile)
536 {
537 if (tile->id == 0)
538 dg1_intr_disable(tile_to_xe(tile));
539
540 gt_irq_reset(tile);
541
542 if (IS_SRIOV_VF(tile_to_xe(tile)))
543 return;
544
545 mask_and_disable(tile, PCU_IRQ_OFFSET);
546 }
547
dg1_irq_reset_mstr(struct xe_tile * tile)548 static void dg1_irq_reset_mstr(struct xe_tile *tile)
549 {
550 struct xe_mmio *mmio = &tile->mmio;
551
552 xe_mmio_write32(mmio, GFX_MSTR_IRQ, ~0);
553 }
554
vf_irq_reset(struct xe_device * xe)555 static void vf_irq_reset(struct xe_device *xe)
556 {
557 struct xe_tile *tile;
558 unsigned int id;
559
560 xe_assert(xe, IS_SRIOV_VF(xe));
561
562 if (GRAPHICS_VERx100(xe) < 1210)
563 xelp_intr_disable(xe);
564 else
565 xe_assert(xe, xe_device_has_memirq(xe));
566
567 for_each_tile(tile, xe, id) {
568 if (xe_device_has_memirq(xe))
569 xe_memirq_reset(&tile->memirq);
570 else
571 gt_irq_reset(tile);
572 }
573 }
574
xe_irq_reset(struct xe_device * xe)575 static void xe_irq_reset(struct xe_device *xe)
576 {
577 struct xe_tile *tile;
578 u8 id;
579
580 if (IS_SRIOV_VF(xe))
581 return vf_irq_reset(xe);
582
583 for_each_tile(tile, xe, id) {
584 if (GRAPHICS_VERx100(xe) >= 1210)
585 dg1_irq_reset(tile);
586 else
587 xelp_irq_reset(tile);
588 }
589
590 tile = xe_device_get_root_tile(xe);
591 mask_and_disable(tile, GU_MISC_IRQ_OFFSET);
592 xe_display_irq_reset(xe);
593
594 /*
595 * The tile's top-level status register should be the last one
596 * to be reset to avoid possible bit re-latching from lower
597 * level interrupts.
598 */
599 if (GRAPHICS_VERx100(xe) >= 1210) {
600 for_each_tile(tile, xe, id)
601 dg1_irq_reset_mstr(tile);
602 }
603 }
604
vf_irq_postinstall(struct xe_device * xe)605 static void vf_irq_postinstall(struct xe_device *xe)
606 {
607 struct xe_tile *tile;
608 unsigned int id;
609
610 for_each_tile(tile, xe, id)
611 if (xe_device_has_memirq(xe))
612 xe_memirq_postinstall(&tile->memirq);
613
614 if (GRAPHICS_VERx100(xe) < 1210)
615 xelp_intr_enable(xe, true);
616 else
617 xe_assert(xe, xe_device_has_memirq(xe));
618 }
619
xe_irq_postinstall(struct xe_device * xe)620 static void xe_irq_postinstall(struct xe_device *xe)
621 {
622 if (IS_SRIOV_VF(xe))
623 return vf_irq_postinstall(xe);
624
625 xe_display_irq_postinstall(xe, xe_root_mmio_gt(xe));
626
627 /*
628 * ASLE backlight operations are reported via GUnit GSE interrupts
629 * on the root tile.
630 */
631 unmask_and_enable(xe_device_get_root_tile(xe),
632 GU_MISC_IRQ_OFFSET, GU_MISC_GSE);
633
634 /* Enable top-level interrupts */
635 if (GRAPHICS_VERx100(xe) >= 1210)
636 dg1_intr_enable(xe, true);
637 else
638 xelp_intr_enable(xe, true);
639 }
640
vf_mem_irq_handler(int irq,void * arg)641 static irqreturn_t vf_mem_irq_handler(int irq, void *arg)
642 {
643 struct xe_device *xe = arg;
644 struct xe_tile *tile;
645 unsigned int id;
646
647 spin_lock(&xe->irq.lock);
648 if (!xe->irq.enabled) {
649 spin_unlock(&xe->irq.lock);
650 return IRQ_NONE;
651 }
652 spin_unlock(&xe->irq.lock);
653
654 for_each_tile(tile, xe, id)
655 xe_memirq_handler(&tile->memirq);
656
657 return IRQ_HANDLED;
658 }
659
xe_irq_handler(struct xe_device * xe)660 static irq_handler_t xe_irq_handler(struct xe_device *xe)
661 {
662 if (IS_SRIOV_VF(xe) && xe_device_has_memirq(xe))
663 return vf_mem_irq_handler;
664
665 if (GRAPHICS_VERx100(xe) >= 1210)
666 return dg1_irq_handler;
667 else
668 return xelp_irq_handler;
669 }
670
irq_uninstall(void * arg)671 static void irq_uninstall(void *arg)
672 {
673 struct xe_device *xe = arg;
674 struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
675 int irq;
676
677 if (!xe->irq.enabled)
678 return;
679
680 xe->irq.enabled = false;
681 xe_irq_reset(xe);
682
683 irq = pci_irq_vector(pdev, 0);
684 free_irq(irq, xe);
685 }
686
xe_irq_install(struct xe_device * xe)687 int xe_irq_install(struct xe_device *xe)
688 {
689 struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
690 unsigned int irq_flags = PCI_IRQ_MSIX;
691 irq_handler_t irq_handler;
692 int err, irq, nvec;
693
694 irq_handler = xe_irq_handler(xe);
695 if (!irq_handler) {
696 drm_err(&xe->drm, "No supported interrupt handler");
697 return -EINVAL;
698 }
699
700 xe_irq_reset(xe);
701
702 nvec = pci_msix_vec_count(pdev);
703 if (nvec <= 0) {
704 if (nvec == -EINVAL) {
705 /* MSIX capability is not supported in the device, using MSI */
706 irq_flags = PCI_IRQ_MSI;
707 nvec = 1;
708 } else {
709 drm_err(&xe->drm, "MSIX: Failed getting count\n");
710 return nvec;
711 }
712 }
713
714 err = pci_alloc_irq_vectors(pdev, nvec, nvec, irq_flags);
715 if (err < 0) {
716 drm_err(&xe->drm, "MSI/MSIX: Failed to enable support %d\n", err);
717 return err;
718 }
719
720 irq = pci_irq_vector(pdev, 0);
721 err = request_irq(irq, irq_handler, IRQF_SHARED, DRIVER_NAME, xe);
722 if (err < 0) {
723 drm_err(&xe->drm, "Failed to request MSI/MSIX IRQ %d\n", err);
724 return err;
725 }
726
727 xe->irq.enabled = true;
728
729 xe_irq_postinstall(xe);
730
731 err = devm_add_action_or_reset(xe->drm.dev, irq_uninstall, xe);
732 if (err)
733 goto free_irq_handler;
734
735 return 0;
736
737 free_irq_handler:
738 free_irq(irq, xe);
739
740 return err;
741 }
742
xe_irq_suspend(struct xe_device * xe)743 void xe_irq_suspend(struct xe_device *xe)
744 {
745 int irq = to_pci_dev(xe->drm.dev)->irq;
746
747 spin_lock_irq(&xe->irq.lock);
748 xe->irq.enabled = false; /* no new irqs */
749 spin_unlock_irq(&xe->irq.lock);
750
751 synchronize_irq(irq); /* flush irqs */
752 xe_irq_reset(xe); /* turn irqs off */
753 }
754
xe_irq_resume(struct xe_device * xe)755 void xe_irq_resume(struct xe_device *xe)
756 {
757 struct xe_gt *gt;
758 int id;
759
760 /*
761 * lock not needed:
762 * 1. no irq will arrive before the postinstall
763 * 2. display is not yet resumed
764 */
765 xe->irq.enabled = true;
766 xe_irq_reset(xe);
767 xe_irq_postinstall(xe); /* turn irqs on */
768
769 for_each_gt(gt, xe, id)
770 xe_irq_enable_hwe(gt);
771 }
772