28 #define IMR(offset)				XE_REG(offset + 0x4)
29 #define IIR(offset)				XE_REG(offset + 0x8)
30 #define IER(offset)				XE_REG(offset + 0xc)
36 	if (val == 0)  in assert_iir_is_zero()
39 	drm_WARN(&mmio->tile->xe->drm, 1,  in assert_iir_is_zero()
40 		 "Interrupt register 0x%x is not zero: 0x%08x\n",  in assert_iir_is_zero()
42 	xe_mmio_write32(mmio, reg, 0xffffffff);  in assert_iir_is_zero()
44 	xe_mmio_write32(mmio, reg, 0xffffffff);  in assert_iir_is_zero()
74 	xe_mmio_write32(mmio, IMR(irqregs), ~0);  in mask_and_disable()
78 	xe_mmio_write32(mmio, IER(irqregs), 0);  in mask_and_disable()
81 	xe_mmio_write32(mmio, IIR(irqregs), ~0);  in mask_and_disable()
83 	xe_mmio_write32(mmio, IIR(irqregs), ~0);  in mask_and_disable()
87 static u32 xelp_intr_disable(struct xe_device *xe)  in xelp_intr_disable()  argument
89 	struct xe_mmio *mmio = xe_root_tile_mmio(xe);  in xelp_intr_disable()
91 	xe_mmio_write32(mmio, GFX_MSTR_IRQ, 0);  in xelp_intr_disable()
103 gu_misc_irq_ack(struct xe_device *xe, const u32 master_ctl)  in gu_misc_irq_ack()  argument
105 	struct xe_mmio *mmio = xe_root_tile_mmio(xe);  in gu_misc_irq_ack()
109 		return 0;  in gu_misc_irq_ack()
118 static inline void xelp_intr_enable(struct xe_device *xe, bool stall)  in xelp_intr_enable()  argument
120 	struct xe_mmio *mmio = xe_root_tile_mmio(xe);  in xelp_intr_enable()
130 	struct xe_device *xe = gt_to_xe(gt);  in xe_irq_enable_hwe()  local
134 	u32 gsc_mask = 0;  in xe_irq_enable_hwe()
135 	u32 heci_mask = 0;  in xe_irq_enable_hwe()
137 	if (xe_device_uses_memirq(xe))  in xe_irq_enable_hwe()
140 	if (xe_device_uc_enabled(xe)) {  in xe_irq_enable_hwe()
173 		if (ccs_mask & (BIT(0)|BIT(1)))  in xe_irq_enable_hwe()
179 	if (xe_gt_is_media_type(gt) || MEDIA_VER(xe) < 13) {  in xe_irq_enable_hwe()
195 		} else if (HAS_HECI_GSCFI(xe)) {  in xe_irq_enable_hwe()
209 gt_engine_identity(struct xe_device *xe,  in gt_engine_identity()  argument
217 	lockdep_assert_held(&xe->irq.lock);  in gt_engine_identity()
232 		drm_err(&xe->drm, "INTR_IDENTITY_REG%u:%u 0x%08x not valid!\n",  in gt_engine_identity()
234 		return 0;  in gt_engine_identity()
256 		WARN_ONCE(1, "unhandled other interrupt instance=0x%x, iir=0x%x\n",  in gt_other_irq_handler()
265 	struct xe_device *xe = tile_to_xe(tile);  in pick_engine_gt()  local
267 	if (MEDIA_VER(xe) < 13)  in pick_engine_gt()
293 	struct xe_device *xe = tile_to_xe(tile);  in gt_irq_handler()  local
300 	spin_lock(&xe->irq.lock);  in gt_irq_handler()
302 	for (bank = 0; bank < 2; bank++) {  in gt_irq_handler()
308 			identity[bit] = gt_engine_identity(xe, mmio, bank, bit);  in gt_irq_handler()
328 				if (HAS_HECI_GSCFI(xe) && instance == OTHER_GSC_INSTANCE)  in gt_irq_handler()
329 					xe_heci_gsc_irq_handler(xe, intr_vec);  in gt_irq_handler()
336 	spin_unlock(&xe->irq.lock);  in gt_irq_handler()
345 	struct xe_device *xe = arg;  in xelp_irq_handler()  local
346 	struct xe_tile *tile = xe_device_get_root_tile(xe);  in xelp_irq_handler()
351 	spin_lock(&xe->irq.lock);  in xelp_irq_handler()
352 	if (!xe->irq.enabled) {  in xelp_irq_handler()
353 		spin_unlock(&xe->irq.lock);  in xelp_irq_handler()
356 	spin_unlock(&xe->irq.lock);  in xelp_irq_handler()
358 	master_ctl = xelp_intr_disable(xe);  in xelp_irq_handler()
360 		xelp_intr_enable(xe, false);  in xelp_irq_handler()
366 	xe_display_irq_handler(xe, master_ctl);  in xelp_irq_handler()
368 	gu_misc_iir = gu_misc_irq_ack(xe, master_ctl);  in xelp_irq_handler()
370 	xelp_intr_enable(xe, false);  in xelp_irq_handler()
372 	xe_display_irq_enable(xe, gu_misc_iir);  in xelp_irq_handler()
377 static u32 dg1_intr_disable(struct xe_device *xe)  in dg1_intr_disable()  argument
379 	struct xe_mmio *mmio = xe_root_tile_mmio(xe);  in dg1_intr_disable()
383 	xe_mmio_write32(mmio, DG1_MSTR_TILE_INTR, 0);  in dg1_intr_disable()
388 		return 0;  in dg1_intr_disable()
395 static void dg1_intr_enable(struct xe_device *xe, bool stall)  in dg1_intr_enable()  argument
397 	struct xe_mmio *mmio = xe_root_tile_mmio(xe);  in dg1_intr_enable()
411 	struct xe_device *xe = arg;  in dg1_irq_handler()  local
413 	u32 master_tile_ctl, master_ctl = 0, gu_misc_iir = 0;  in dg1_irq_handler()
420 	spin_lock(&xe->irq.lock);  in dg1_irq_handler()
421 	if (!xe->irq.enabled) {  in dg1_irq_handler()
422 		spin_unlock(&xe->irq.lock);  in dg1_irq_handler()
425 	spin_unlock(&xe->irq.lock);  in dg1_irq_handler()
427 	master_tile_ctl = dg1_intr_disable(xe);  in dg1_irq_handler()
429 		dg1_intr_enable(xe, false);  in dg1_irq_handler()
433 	for_each_tile(tile, xe, id) {  in dg1_irq_handler()
436 		if ((master_tile_ctl & DG1_MSTR_TILE(tile->id)) == 0)  in dg1_irq_handler()
446 		if (master_ctl == REG_GENMASK(31, 0)) {  in dg1_irq_handler()
461 		if (id == 0) {  in dg1_irq_handler()
462 			if (HAS_HECI_CSCFI(xe))  in dg1_irq_handler()
463 				xe_heci_csc_irq_handler(xe, master_ctl);  in dg1_irq_handler()
464 			xe_display_irq_handler(xe, master_ctl);  in dg1_irq_handler()
465 			gu_misc_iir = gu_misc_irq_ack(xe, master_ctl);  in dg1_irq_handler()
469 	dg1_intr_enable(xe, false);  in dg1_irq_handler()
470 	xe_display_irq_enable(xe, gu_misc_iir);  in dg1_irq_handler()
485 	xe_mmio_write32(mmio, RENDER_COPY_INTR_ENABLE, 0);  in gt_irq_reset()
486 	xe_mmio_write32(mmio, VCS_VECS_INTR_ENABLE, 0);  in gt_irq_reset()
488 		xe_mmio_write32(mmio, CCS_RSVD_INTR_ENABLE, 0);  in gt_irq_reset()
491 	xe_mmio_write32(mmio, RCS0_RSVD_INTR_MASK,	~0);  in gt_irq_reset()
492 	xe_mmio_write32(mmio, BCS_RSVD_INTR_MASK,	~0);  in gt_irq_reset()
494 		xe_mmio_write32(mmio, XEHPC_BCS1_BCS2_INTR_MASK, ~0);  in gt_irq_reset()
496 		xe_mmio_write32(mmio, XEHPC_BCS3_BCS4_INTR_MASK, ~0);  in gt_irq_reset()
498 		xe_mmio_write32(mmio, XEHPC_BCS5_BCS6_INTR_MASK, ~0);  in gt_irq_reset()
500 		xe_mmio_write32(mmio, XEHPC_BCS7_BCS8_INTR_MASK, ~0);  in gt_irq_reset()
501 	xe_mmio_write32(mmio, VCS0_VCS1_INTR_MASK,	~0);  in gt_irq_reset()
502 	xe_mmio_write32(mmio, VCS2_VCS3_INTR_MASK,	~0);  in gt_irq_reset()
503 	xe_mmio_write32(mmio, VECS0_VECS1_INTR_MASK,	~0);  in gt_irq_reset()
504 	if (ccs_mask & (BIT(0)|BIT(1)))  in gt_irq_reset()
505 		xe_mmio_write32(mmio, CCS0_CCS1_INTR_MASK, ~0);  in gt_irq_reset()
507 		xe_mmio_write32(mmio, CCS2_CCS3_INTR_MASK, ~0);  in gt_irq_reset()
512 		xe_mmio_write32(mmio, GUNIT_GSC_INTR_ENABLE, 0);  in gt_irq_reset()
513 		xe_mmio_write32(mmio, GUNIT_GSC_INTR_MASK, ~0);  in gt_irq_reset()
514 		xe_mmio_write32(mmio, HECI2_RSVD_INTR_MASK, ~0);  in gt_irq_reset()
517 	xe_mmio_write32(mmio, GPM_WGBOXPERF_INTR_ENABLE, 0);  in gt_irq_reset()
518 	xe_mmio_write32(mmio, GPM_WGBOXPERF_INTR_MASK,  ~0);  in gt_irq_reset()
519 	xe_mmio_write32(mmio, GUC_SG_INTR_ENABLE,	 0);  in gt_irq_reset()
520 	xe_mmio_write32(mmio, GUC_SG_INTR_MASK,		~0);  in gt_irq_reset()
537 	if (tile->id == 0)  in dg1_irq_reset()
552 	xe_mmio_write32(mmio, GFX_MSTR_IRQ, ~0);  in dg1_irq_reset_mstr()
555 static void vf_irq_reset(struct xe_device *xe)  in vf_irq_reset()  argument
560 	xe_assert(xe, IS_SRIOV_VF(xe));  in vf_irq_reset()
562 	if (GRAPHICS_VERx100(xe) < 1210)  in vf_irq_reset()
563 		xelp_intr_disable(xe);  in vf_irq_reset()
565 		xe_assert(xe, xe_device_has_memirq(xe));  in vf_irq_reset()
567 	for_each_tile(tile, xe, id) {  in vf_irq_reset()
568 		if (xe_device_has_memirq(xe))  in vf_irq_reset()
575 static void xe_irq_reset(struct xe_device *xe)  in xe_irq_reset()  argument
580 	if (IS_SRIOV_VF(xe))  in xe_irq_reset()
581 		return vf_irq_reset(xe);  in xe_irq_reset()
583 	for_each_tile(tile, xe, id) {  in xe_irq_reset()
584 		if (GRAPHICS_VERx100(xe) >= 1210)  in xe_irq_reset()
590 	tile = xe_device_get_root_tile(xe);  in xe_irq_reset()
592 	xe_display_irq_reset(xe);  in xe_irq_reset()
599 	if (GRAPHICS_VERx100(xe) >= 1210) {  in xe_irq_reset()
600 		for_each_tile(tile, xe, id)  in xe_irq_reset()
605 static void vf_irq_postinstall(struct xe_device *xe)  in vf_irq_postinstall()  argument
610 	for_each_tile(tile, xe, id)  in vf_irq_postinstall()
611 		if (xe_device_has_memirq(xe))  in vf_irq_postinstall()
614 	if (GRAPHICS_VERx100(xe) < 1210)  in vf_irq_postinstall()
615 		xelp_intr_enable(xe, true);  in vf_irq_postinstall()
617 		xe_assert(xe, xe_device_has_memirq(xe));  in vf_irq_postinstall()
620 static void xe_irq_postinstall(struct xe_device *xe)  in xe_irq_postinstall()  argument
622 	if (IS_SRIOV_VF(xe))  in xe_irq_postinstall()
623 		return vf_irq_postinstall(xe);  in xe_irq_postinstall()
625 	xe_display_irq_postinstall(xe, xe_root_mmio_gt(xe));  in xe_irq_postinstall()
631 	unmask_and_enable(xe_device_get_root_tile(xe),  in xe_irq_postinstall()
635 	if (GRAPHICS_VERx100(xe) >= 1210)  in xe_irq_postinstall()
636 		dg1_intr_enable(xe, true);  in xe_irq_postinstall()
638 		xelp_intr_enable(xe, true);  in xe_irq_postinstall()
643 	struct xe_device *xe = arg;  in vf_mem_irq_handler()  local
647 	spin_lock(&xe->irq.lock);  in vf_mem_irq_handler()
648 	if (!xe->irq.enabled) {  in vf_mem_irq_handler()
649 		spin_unlock(&xe->irq.lock);  in vf_mem_irq_handler()
652 	spin_unlock(&xe->irq.lock);  in vf_mem_irq_handler()
654 	for_each_tile(tile, xe, id)  in vf_mem_irq_handler()
660 static irq_handler_t xe_irq_handler(struct xe_device *xe)  in xe_irq_handler()  argument
662 	if (IS_SRIOV_VF(xe) && xe_device_has_memirq(xe))  in xe_irq_handler()
665 	if (GRAPHICS_VERx100(xe) >= 1210)  in xe_irq_handler()
673 	struct xe_device *xe = arg;  in irq_uninstall()  local
674 	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);  in irq_uninstall()
677 	if (!xe->irq.enabled)  in irq_uninstall()
680 	xe->irq.enabled = false;  in irq_uninstall()
681 	xe_irq_reset(xe);  in irq_uninstall()
683 	irq = pci_irq_vector(pdev, 0);  in irq_uninstall()
684 	free_irq(irq, xe);  in irq_uninstall()
687 int xe_irq_install(struct xe_device *xe)  in xe_irq_install()  argument
689 	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);  in xe_irq_install()
694 	irq_handler = xe_irq_handler(xe);  in xe_irq_install()
696 		drm_err(&xe->drm, "No supported interrupt handler");  in xe_irq_install()
700 	xe_irq_reset(xe);  in xe_irq_install()
703 	if (nvec <= 0) {  in xe_irq_install()
709 			drm_err(&xe->drm, "MSIX: Failed getting count\n");  in xe_irq_install()
715 	if (err < 0) {  in xe_irq_install()
716 		drm_err(&xe->drm, "MSI/MSIX: Failed to enable support %d\n", err);  in xe_irq_install()
720 	irq = pci_irq_vector(pdev, 0);  in xe_irq_install()
721 	err = request_irq(irq, irq_handler, IRQF_SHARED, DRIVER_NAME, xe);  in xe_irq_install()
722 	if (err < 0) {  in xe_irq_install()
723 		drm_err(&xe->drm, "Failed to request MSI/MSIX IRQ %d\n", err);  in xe_irq_install()
727 	xe->irq.enabled = true;  in xe_irq_install()
729 	xe_irq_postinstall(xe);  in xe_irq_install()
731 	err = devm_add_action_or_reset(xe->drm.dev, irq_uninstall, xe);  in xe_irq_install()
735 	return 0;  in xe_irq_install()
738 	free_irq(irq, xe);  in xe_irq_install()
743 void xe_irq_suspend(struct xe_device *xe)  in xe_irq_suspend()  argument
745 	int irq = to_pci_dev(xe->drm.dev)->irq;  in xe_irq_suspend()
747 	spin_lock_irq(&xe->irq.lock);  in xe_irq_suspend()
748 	xe->irq.enabled = false; /* no new irqs */  in xe_irq_suspend()
749 	spin_unlock_irq(&xe->irq.lock);  in xe_irq_suspend()
752 	xe_irq_reset(xe); /* turn irqs off */  in xe_irq_suspend()
755 void xe_irq_resume(struct xe_device *xe)  in xe_irq_resume()  argument
765 	xe->irq.enabled = true;  in xe_irq_resume()
766 	xe_irq_reset(xe);  in xe_irq_resume()
767 	xe_irq_postinstall(xe); /* turn irqs on */  in xe_irq_resume()
769 	for_each_gt(gt, xe, id)  in xe_irq_resume()