29 static long tlb_timeout_jiffies(struct xe_gt *gt)  in tlb_timeout_jiffies()  argument
35 	long delay = xe_guc_ct_queue_proc_time_jiffies(&gt->uc.guc.ct);  in tlb_timeout_jiffies()
61 	struct xe_gt *gt = container_of(work, struct xe_gt,  in xe_gt_tlb_fence_timeout()  local
63 	struct xe_device *xe = gt_to_xe(gt);  in xe_gt_tlb_fence_timeout()
66 	spin_lock_irq(&gt->tlb_invalidation.pending_lock);  in xe_gt_tlb_fence_timeout()
68 				 &gt->tlb_invalidation.pending_fences, link) {  in xe_gt_tlb_fence_timeout()
72 		if (msecs_to_jiffies(since_inval_ms) < tlb_timeout_jiffies(gt))  in xe_gt_tlb_fence_timeout()
76 		xe_gt_err(gt, "TLB invalidation fence timeout, seqno=%d recv=%d",  in xe_gt_tlb_fence_timeout()
77 			  fence->seqno, gt->tlb_invalidation.seqno_recv);  in xe_gt_tlb_fence_timeout()
82 	if (!list_empty(&gt->tlb_invalidation.pending_fences))  in xe_gt_tlb_fence_timeout()
84 				   &gt->tlb_invalidation.fence_tdr,  in xe_gt_tlb_fence_timeout()
85 				   tlb_timeout_jiffies(gt));  in xe_gt_tlb_fence_timeout()
86 	spin_unlock_irq(&gt->tlb_invalidation.pending_lock);  in xe_gt_tlb_fence_timeout()
90  * xe_gt_tlb_invalidation_init - Initialize GT TLB invalidation state
91  * @gt: graphics tile
93  * Initialize GT TLB invalidation state, purely software initialization, should
98 int xe_gt_tlb_invalidation_init(struct xe_gt *gt)  in xe_gt_tlb_invalidation_init()  argument
100 	gt->tlb_invalidation.seqno = 1;  in xe_gt_tlb_invalidation_init()
101 	INIT_LIST_HEAD(&gt->tlb_invalidation.pending_fences);  in xe_gt_tlb_invalidation_init()
102 	spin_lock_init(&gt->tlb_invalidation.pending_lock);  in xe_gt_tlb_invalidation_init()
103 	spin_lock_init(&gt->tlb_invalidation.lock);  in xe_gt_tlb_invalidation_init()
104 	INIT_DELAYED_WORK(&gt->tlb_invalidation.fence_tdr,  in xe_gt_tlb_invalidation_init()
111  * xe_gt_tlb_invalidation_reset - Initialize GT TLB invalidation reset
112  * @gt: graphics tile
114  * Signal any pending invalidation fences, should be called during a GT reset
116 void xe_gt_tlb_invalidation_reset(struct xe_gt *gt)  in xe_gt_tlb_invalidation_reset()  argument
126 	mutex_lock(&gt->uc.guc.ct.lock);  in xe_gt_tlb_invalidation_reset()
127 	spin_lock_irq(&gt->tlb_invalidation.pending_lock);  in xe_gt_tlb_invalidation_reset()
128 	cancel_delayed_work(&gt->tlb_invalidation.fence_tdr);  in xe_gt_tlb_invalidation_reset()
134 	 * completed a full GT reset.  in xe_gt_tlb_invalidation_reset()
136 	if (gt->tlb_invalidation.seqno == 1)  in xe_gt_tlb_invalidation_reset()
139 		pending_seqno = gt->tlb_invalidation.seqno - 1;  in xe_gt_tlb_invalidation_reset()
140 	WRITE_ONCE(gt->tlb_invalidation.seqno_recv, pending_seqno);  in xe_gt_tlb_invalidation_reset()
143 				 &gt->tlb_invalidation.pending_fences, link)  in xe_gt_tlb_invalidation_reset()
144 		invalidation_fence_signal(gt_to_xe(gt), fence);  in xe_gt_tlb_invalidation_reset()
145 	spin_unlock_irq(&gt->tlb_invalidation.pending_lock);  in xe_gt_tlb_invalidation_reset()
146 	mutex_unlock(&gt->uc.guc.ct.lock);  in xe_gt_tlb_invalidation_reset()
149 static bool tlb_invalidation_seqno_past(struct xe_gt *gt, int seqno)  in tlb_invalidation_seqno_past()  argument
151 	int seqno_recv = READ_ONCE(gt->tlb_invalidation.seqno_recv);  in tlb_invalidation_seqno_past()
166 	struct xe_gt *gt = guc_to_gt(guc);  in send_tlb_invalidation()  local
167 	struct xe_device *xe = gt_to_xe(gt);  in send_tlb_invalidation()
171 	xe_gt_assert(gt, fence);  in send_tlb_invalidation()
180 	seqno = gt->tlb_invalidation.seqno;  in send_tlb_invalidation()
187 		spin_lock_irq(&gt->tlb_invalidation.pending_lock);  in send_tlb_invalidation()
194 		if (tlb_invalidation_seqno_past(gt, seqno)) {  in send_tlb_invalidation()
199 				      &gt->tlb_invalidation.pending_fences);  in send_tlb_invalidation()
201 			if (list_is_singular(&gt->tlb_invalidation.pending_fences))  in send_tlb_invalidation()
203 						   &gt->tlb_invalidation.fence_tdr,  in send_tlb_invalidation()
204 						   tlb_timeout_jiffies(gt));  in send_tlb_invalidation()
206 		spin_unlock_irq(&gt->tlb_invalidation.pending_lock);  in send_tlb_invalidation()
211 		gt->tlb_invalidation.seqno = (gt->tlb_invalidation.seqno + 1) %  in send_tlb_invalidation()
213 		if (!gt->tlb_invalidation.seqno)  in send_tlb_invalidation()
214 			gt->tlb_invalidation.seqno = 1;  in send_tlb_invalidation()
217 	xe_gt_stats_incr(gt, XE_GT_STATS_ID_TLB_INVAL, 1);  in send_tlb_invalidation()
227  * xe_gt_tlb_invalidation_guc - Issue a TLB invalidation on this GT for the GuC
228  * @gt: graphics tile
237 static int xe_gt_tlb_invalidation_guc(struct xe_gt *gt,  in xe_gt_tlb_invalidation_guc()  argument
246 	return send_tlb_invalidation(&gt->uc.guc, fence, action,  in xe_gt_tlb_invalidation_guc()
251  * xe_gt_tlb_invalidation_ggtt - Issue a TLB invalidation on this GT for the GGTT
252  * @gt: graphics tile
259 int xe_gt_tlb_invalidation_ggtt(struct xe_gt *gt)  in xe_gt_tlb_invalidation_ggtt()  argument
261 	struct xe_device *xe = gt_to_xe(gt);  in xe_gt_tlb_invalidation_ggtt()
263 	if (xe_guc_ct_enabled(&gt->uc.guc.ct) &&  in xe_gt_tlb_invalidation_ggtt()
264 	    gt->uc.guc.submission_state.enabled) {  in xe_gt_tlb_invalidation_ggtt()
268 		xe_gt_tlb_invalidation_fence_init(gt, &fence, true);  in xe_gt_tlb_invalidation_ggtt()
269 		ret = xe_gt_tlb_invalidation_guc(gt, &fence);  in xe_gt_tlb_invalidation_ggtt()
280 		xe_gt_WARN_ON(gt, xe_force_wake_get(gt_to_fw(gt), XE_FW_GT));  in xe_gt_tlb_invalidation_ggtt()
282 			xe_mmio_write32(gt, PVC_GUC_TLB_INV_DESC1,  in xe_gt_tlb_invalidation_ggtt()
284 			xe_mmio_write32(gt, PVC_GUC_TLB_INV_DESC0,  in xe_gt_tlb_invalidation_ggtt()
287 			xe_mmio_write32(gt, GUC_TLB_INV_CR,  in xe_gt_tlb_invalidation_ggtt()
290 		xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);  in xe_gt_tlb_invalidation_ggtt()
297  * xe_gt_tlb_invalidation_range - Issue a TLB invalidation on this GT for an
300  * @gt: graphics tile
313 int xe_gt_tlb_invalidation_range(struct xe_gt *gt,  in xe_gt_tlb_invalidation_range()  argument
317 	struct xe_device *xe = gt_to_xe(gt);  in xe_gt_tlb_invalidation_range()
322 	xe_gt_assert(gt, fence);  in xe_gt_tlb_invalidation_range()
325 	if (gt_to_xe(gt)->info.force_execlist) {  in xe_gt_tlb_invalidation_range()
366 		xe_gt_assert(gt, length >= SZ_4K);  in xe_gt_tlb_invalidation_range()
367 		xe_gt_assert(gt, is_power_of_2(length));  in xe_gt_tlb_invalidation_range()
368 		xe_gt_assert(gt, !(length & GENMASK(ilog2(SZ_16M) - 1,  in xe_gt_tlb_invalidation_range()
370 		xe_gt_assert(gt, IS_ALIGNED(start, length));  in xe_gt_tlb_invalidation_range()
379 	xe_gt_assert(gt, len <= MAX_TLB_INVALIDATION_LEN);  in xe_gt_tlb_invalidation_range()
381 	return send_tlb_invalidation(&gt->uc.guc, fence, action, len);  in xe_gt_tlb_invalidation_range()
385  * xe_gt_tlb_invalidation_vma - Issue a TLB invalidation on this GT for a VMA
386  * @gt: graphics tile
397 int xe_gt_tlb_invalidation_vma(struct xe_gt *gt,  in xe_gt_tlb_invalidation_vma()  argument
401 	xe_gt_assert(gt, vma);  in xe_gt_tlb_invalidation_vma()
403 	return xe_gt_tlb_invalidation_range(gt, fence, xe_vma_start(vma),  in xe_gt_tlb_invalidation_vma()
422 	struct xe_gt *gt = guc_to_gt(guc);  in xe_guc_tlb_invalidation_done_handler()  local
423 	struct xe_device *xe = gt_to_xe(gt);  in xe_guc_tlb_invalidation_done_handler()
445 	spin_lock_irqsave(&gt->tlb_invalidation.pending_lock, flags);  in xe_guc_tlb_invalidation_done_handler()
446 	if (tlb_invalidation_seqno_past(gt, msg[0])) {  in xe_guc_tlb_invalidation_done_handler()
447 		spin_unlock_irqrestore(&gt->tlb_invalidation.pending_lock, flags);  in xe_guc_tlb_invalidation_done_handler()
451 	WRITE_ONCE(gt->tlb_invalidation.seqno_recv, msg[0]);  in xe_guc_tlb_invalidation_done_handler()
454 				 &gt->tlb_invalidation.pending_fences, link) {  in xe_guc_tlb_invalidation_done_handler()
457 		if (!tlb_invalidation_seqno_past(gt, fence->seqno))  in xe_guc_tlb_invalidation_done_handler()
463 	if (!list_empty(&gt->tlb_invalidation.pending_fences))  in xe_guc_tlb_invalidation_done_handler()
465 				 &gt->tlb_invalidation.fence_tdr,  in xe_guc_tlb_invalidation_done_handler()
466 				 tlb_timeout_jiffies(gt));  in xe_guc_tlb_invalidation_done_handler()
468 		cancel_delayed_work(&gt->tlb_invalidation.fence_tdr);  in xe_guc_tlb_invalidation_done_handler()
470 	spin_unlock_irqrestore(&gt->tlb_invalidation.pending_lock, flags);  in xe_guc_tlb_invalidation_done_handler()
494  * @gt: GT
501 void xe_gt_tlb_invalidation_fence_init(struct xe_gt *gt,  in xe_gt_tlb_invalidation_fence_init()  argument
505 	xe_pm_runtime_get_noresume(gt_to_xe(gt));  in xe_gt_tlb_invalidation_fence_init()
507 	spin_lock_irq(&gt->tlb_invalidation.lock);  in xe_gt_tlb_invalidation_fence_init()
509 		       &gt->tlb_invalidation.lock,  in xe_gt_tlb_invalidation_fence_init()
511 	spin_unlock_irq(&gt->tlb_invalidation.lock);  in xe_gt_tlb_invalidation_fence_init()
517 	fence->gt = gt;  in xe_gt_tlb_invalidation_fence_init()
528 	xe_pm_runtime_put(gt_to_xe(fence->gt));  in xe_gt_tlb_invalidation_fence_fini()