1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2023 Intel Corporation 4 */ 5 6 #include "i915_drv.h" 7 #include "i915_perf_oa_regs.h" 8 #include "intel_engine_pm.h" 9 #include "intel_gt.h" 10 #include "intel_gt_mcr.h" 11 #include "intel_gt_pm.h" 12 #include "intel_gt_print.h" 13 #include "intel_gt_regs.h" 14 #include "intel_tlb.h" 15 #include "uc/intel_guc.h" 16 17 /* 18 * HW architecture suggest typical invalidation time at 40us, 19 * with pessimistic cases up to 100us and a recommendation to 20 * cap at 1ms. We go a bit higher just in case. 21 */ 22 #define TLB_INVAL_TIMEOUT_US 100 23 #define TLB_INVAL_TIMEOUT_MS 4 24 25 /* 26 * On Xe_HP the TLB invalidation registers are located at the same MMIO offsets 27 * but are now considered MCR registers. Since they exist within a GAM range, 28 * the primary instance of the register rolls up the status from each unit. 29 */ 30 static int wait_for_invalidate(struct intel_engine_cs *engine) 31 { 32 if (engine->tlb_inv.mcr) 33 return intel_gt_mcr_wait_for_reg(engine->gt, 34 engine->tlb_inv.reg.mcr_reg, 35 engine->tlb_inv.done, 36 0, 37 TLB_INVAL_TIMEOUT_US, 38 TLB_INVAL_TIMEOUT_MS); 39 else 40 return __intel_wait_for_register_fw(engine->gt->uncore, 41 engine->tlb_inv.reg.reg, 42 engine->tlb_inv.done, 43 0, 44 TLB_INVAL_TIMEOUT_US, 45 TLB_INVAL_TIMEOUT_MS, 46 NULL); 47 } 48 49 static void mmio_invalidate_full(struct intel_gt *gt) 50 { 51 struct drm_i915_private *i915 = gt->i915; 52 struct intel_uncore *uncore = gt->uncore; 53 struct intel_engine_cs *engine; 54 intel_engine_mask_t awake, tmp; 55 enum intel_engine_id id; 56 unsigned long flags; 57 58 if (GRAPHICS_VER(i915) < 8) 59 return; 60 61 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); 62 63 intel_gt_mcr_lock(gt, &flags); 64 spin_lock(&uncore->lock); /* serialise invalidate with GT reset */ 65 66 awake = 0; 67 for_each_engine(engine, gt, id) { 68 if (!intel_engine_pm_is_awake(engine)) 69 continue; 70 71 if (engine->tlb_inv.mcr) 72 intel_gt_mcr_multicast_write_fw(gt, 73 engine->tlb_inv.reg.mcr_reg, 74 engine->tlb_inv.request); 75 else 76 intel_uncore_write_fw(uncore, 77 engine->tlb_inv.reg.reg, 78 engine->tlb_inv.request); 79 80 awake |= engine->mask; 81 } 82 83 GT_TRACE(gt, "invalidated engines %08x\n", awake); 84 85 /* Wa_2207587034:tgl,dg1,rkl,adl-s,adl-p */ 86 if (awake && 87 (IS_TIGERLAKE(i915) || 88 IS_DG1(i915) || 89 IS_ROCKETLAKE(i915) || 90 IS_ALDERLAKE_S(i915) || 91 IS_ALDERLAKE_P(i915))) 92 intel_uncore_write_fw(uncore, GEN12_OA_TLB_INV_CR, 1); 93 94 spin_unlock(&uncore->lock); 95 intel_gt_mcr_unlock(gt, flags); 96 97 for_each_engine_masked(engine, gt, awake, tmp) { 98 if (wait_for_invalidate(engine)) 99 gt_err_ratelimited(gt, 100 "%s TLB invalidation did not complete in %ums!\n", 101 engine->name, TLB_INVAL_TIMEOUT_MS); 102 } 103 104 /* 105 * Use delayed put since a) we mostly expect a flurry of TLB 106 * invalidations so it is good to avoid paying the forcewake cost and 107 * b) it works around a bug in Icelake which cannot cope with too rapid 108 * transitions. 109 */ 110 intel_uncore_forcewake_put_delayed(uncore, FORCEWAKE_ALL); 111 } 112 113 static bool tlb_seqno_passed(const struct intel_gt *gt, u32 seqno) 114 { 115 u32 cur = intel_gt_tlb_seqno(gt); 116 117 /* Only skip if a *full* TLB invalidate barrier has passed */ 118 return (s32)(cur - ALIGN(seqno, 2)) > 0; 119 } 120 121 void intel_gt_invalidate_tlb_full(struct intel_gt *gt, u32 seqno) 122 { 123 intel_wakeref_t wakeref; 124 125 if (I915_SELFTEST_ONLY(gt->awake == -ENODEV)) 126 return; 127 128 if (intel_gt_is_wedged(gt)) 129 return; 130 131 if (tlb_seqno_passed(gt, seqno)) 132 return; 133 134 with_intel_gt_pm_if_awake(gt, wakeref) { 135 struct intel_guc *guc = gt_to_guc(gt); 136 137 mutex_lock(>->tlb.invalidate_lock); 138 if (tlb_seqno_passed(gt, seqno)) 139 goto unlock; 140 141 if (HAS_GUC_TLB_INVALIDATION(gt->i915)) { 142 /* 143 * Only perform GuC TLB invalidation if GuC is ready. 144 * The only time GuC could not be ready is on GT reset, 145 * which would clobber all the TLBs anyways, making 146 * any TLB invalidation path here unnecessary. 147 */ 148 if (intel_guc_is_ready(guc)) 149 intel_guc_invalidate_tlb_engines(guc); 150 } else { 151 mmio_invalidate_full(gt); 152 } 153 154 write_seqcount_invalidate(>->tlb.seqno); 155 unlock: 156 mutex_unlock(>->tlb.invalidate_lock); 157 } 158 } 159 160 void intel_gt_init_tlb(struct intel_gt *gt) 161 { 162 mutex_init(>->tlb.invalidate_lock); 163 seqcount_mutex_init(>->tlb.seqno, >->tlb.invalidate_lock); 164 } 165 166 void intel_gt_fini_tlb(struct intel_gt *gt) 167 { 168 mutex_destroy(>->tlb.invalidate_lock); 169 } 170 171 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 172 #include "selftest_tlb.c" 173 #endif 174