xref: /linux/drivers/gpu/drm/xe/xe_gt_pagefault.c (revision 3c2fe27971c3c9cc27de6e369385f6428db6c0b5)
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2022 Intel Corporation
4  */
5 
6 #include "xe_gt_pagefault.h"
7 
8 #include <linux/bitfield.h>
9 #include <linux/circ_buf.h>
10 
11 #include <drm/drm_exec.h>
12 #include <drm/drm_managed.h>
13 
14 #include "abi/guc_actions_abi.h"
15 #include "xe_bo.h"
16 #include "xe_gt.h"
17 #include "xe_gt_stats.h"
18 #include "xe_gt_tlb_invalidation.h"
19 #include "xe_guc.h"
20 #include "xe_guc_ct.h"
21 #include "xe_migrate.h"
22 #include "xe_svm.h"
23 #include "xe_trace_bo.h"
24 #include "xe_vm.h"
25 
26 struct pagefault {
27 	u64 page_addr;
28 	u32 asid;
29 	u16 pdata;
30 	u8 vfid;
31 	u8 access_type;
32 	u8 fault_type;
33 	u8 fault_level;
34 	u8 engine_class;
35 	u8 engine_instance;
36 	u8 fault_unsuccessful;
37 	bool trva_fault;
38 };
39 
40 enum access_type {
41 	ACCESS_TYPE_READ = 0,
42 	ACCESS_TYPE_WRITE = 1,
43 	ACCESS_TYPE_ATOMIC = 2,
44 	ACCESS_TYPE_RESERVED = 3,
45 };
46 
47 enum fault_type {
48 	NOT_PRESENT = 0,
49 	WRITE_ACCESS_VIOLATION = 1,
50 	ATOMIC_ACCESS_VIOLATION = 2,
51 };
52 
53 struct acc {
54 	u64 va_range_base;
55 	u32 asid;
56 	u32 sub_granularity;
57 	u8 granularity;
58 	u8 vfid;
59 	u8 access_type;
60 	u8 engine_class;
61 	u8 engine_instance;
62 };
63 
access_is_atomic(enum access_type access_type)64 static bool access_is_atomic(enum access_type access_type)
65 {
66 	return access_type == ACCESS_TYPE_ATOMIC;
67 }
68 
vma_is_valid(struct xe_tile * tile,struct xe_vma * vma)69 static bool vma_is_valid(struct xe_tile *tile, struct xe_vma *vma)
70 {
71 	return BIT(tile->id) & vma->tile_present &&
72 		!(BIT(tile->id) & vma->tile_invalidated);
73 }
74 
vma_matches(struct xe_vma * vma,u64 page_addr)75 static bool vma_matches(struct xe_vma *vma, u64 page_addr)
76 {
77 	if (page_addr > xe_vma_end(vma) - 1 ||
78 	    page_addr + SZ_4K - 1 < xe_vma_start(vma))
79 		return false;
80 
81 	return true;
82 }
83 
lookup_vma(struct xe_vm * vm,u64 page_addr)84 static struct xe_vma *lookup_vma(struct xe_vm *vm, u64 page_addr)
85 {
86 	struct xe_vma *vma = NULL;
87 
88 	if (vm->usm.last_fault_vma) {   /* Fast lookup */
89 		if (vma_matches(vm->usm.last_fault_vma, page_addr))
90 			vma = vm->usm.last_fault_vma;
91 	}
92 	if (!vma)
93 		vma = xe_vm_find_overlapping_vma(vm, page_addr, SZ_4K);
94 
95 	return vma;
96 }
97 
xe_pf_begin(struct drm_exec * exec,struct xe_vma * vma,bool atomic,unsigned int id)98 static int xe_pf_begin(struct drm_exec *exec, struct xe_vma *vma,
99 		       bool atomic, unsigned int id)
100 {
101 	struct xe_bo *bo = xe_vma_bo(vma);
102 	struct xe_vm *vm = xe_vma_vm(vma);
103 	int err;
104 
105 	err = xe_vm_lock_vma(exec, vma);
106 	if (err)
107 		return err;
108 
109 	if (atomic && IS_DGFX(vm->xe)) {
110 		if (xe_vma_is_userptr(vma)) {
111 			err = -EACCES;
112 			return err;
113 		}
114 
115 		/* Migrate to VRAM, move should invalidate the VMA first */
116 		err = xe_bo_migrate(bo, XE_PL_VRAM0 + id);
117 		if (err)
118 			return err;
119 	} else if (bo) {
120 		/* Create backing store if needed */
121 		err = xe_bo_validate(bo, vm, true);
122 		if (err)
123 			return err;
124 	}
125 
126 	return 0;
127 }
128 
handle_vma_pagefault(struct xe_gt * gt,struct xe_vma * vma,bool atomic)129 static int handle_vma_pagefault(struct xe_gt *gt, struct xe_vma *vma,
130 				bool atomic)
131 {
132 	struct xe_vm *vm = xe_vma_vm(vma);
133 	struct xe_tile *tile = gt_to_tile(gt);
134 	struct drm_exec exec;
135 	struct dma_fence *fence;
136 	ktime_t end = 0;
137 	int err;
138 
139 	lockdep_assert_held_write(&vm->lock);
140 
141 	xe_gt_stats_incr(gt, XE_GT_STATS_ID_VMA_PAGEFAULT_COUNT, 1);
142 	xe_gt_stats_incr(gt, XE_GT_STATS_ID_VMA_PAGEFAULT_KB, xe_vma_size(vma) / 1024);
143 
144 	trace_xe_vma_pagefault(vma);
145 
146 	/* Check if VMA is valid */
147 	if (vma_is_valid(tile, vma) && !atomic)
148 		return 0;
149 
150 retry_userptr:
151 	if (xe_vma_is_userptr(vma) &&
152 	    xe_vma_userptr_check_repin(to_userptr_vma(vma))) {
153 		struct xe_userptr_vma *uvma = to_userptr_vma(vma);
154 
155 		err = xe_vma_userptr_pin_pages(uvma);
156 		if (err)
157 			return err;
158 	}
159 
160 	/* Lock VM and BOs dma-resv */
161 	drm_exec_init(&exec, 0, 0);
162 	drm_exec_until_all_locked(&exec) {
163 		err = xe_pf_begin(&exec, vma, atomic, tile->id);
164 		drm_exec_retry_on_contention(&exec);
165 		if (xe_vm_validate_should_retry(&exec, err, &end))
166 			err = -EAGAIN;
167 		if (err)
168 			goto unlock_dma_resv;
169 
170 		/* Bind VMA only to the GT that has faulted */
171 		trace_xe_vma_pf_bind(vma);
172 		fence = xe_vma_rebind(vm, vma, BIT(tile->id));
173 		if (IS_ERR(fence)) {
174 			err = PTR_ERR(fence);
175 			if (xe_vm_validate_should_retry(&exec, err, &end))
176 				err = -EAGAIN;
177 			goto unlock_dma_resv;
178 		}
179 	}
180 
181 	dma_fence_wait(fence, false);
182 	dma_fence_put(fence);
183 	vma->tile_invalidated &= ~BIT(tile->id);
184 
185 unlock_dma_resv:
186 	drm_exec_fini(&exec);
187 	if (err == -EAGAIN)
188 		goto retry_userptr;
189 
190 	return err;
191 }
192 
asid_to_vm(struct xe_device * xe,u32 asid)193 static struct xe_vm *asid_to_vm(struct xe_device *xe, u32 asid)
194 {
195 	struct xe_vm *vm;
196 
197 	down_read(&xe->usm.lock);
198 	vm = xa_load(&xe->usm.asid_to_vm, asid);
199 	if (vm && xe_vm_in_fault_mode(vm))
200 		xe_vm_get(vm);
201 	else
202 		vm = ERR_PTR(-EINVAL);
203 	up_read(&xe->usm.lock);
204 
205 	return vm;
206 }
207 
handle_pagefault(struct xe_gt * gt,struct pagefault * pf)208 static int handle_pagefault(struct xe_gt *gt, struct pagefault *pf)
209 {
210 	struct xe_device *xe = gt_to_xe(gt);
211 	struct xe_vm *vm;
212 	struct xe_vma *vma = NULL;
213 	int err;
214 	bool atomic;
215 
216 	/* SW isn't expected to handle TRTT faults */
217 	if (pf->trva_fault)
218 		return -EFAULT;
219 
220 	vm = asid_to_vm(xe, pf->asid);
221 	if (IS_ERR(vm))
222 		return PTR_ERR(vm);
223 
224 	/*
225 	 * TODO: Change to read lock? Using write lock for simplicity.
226 	 */
227 	down_write(&vm->lock);
228 
229 	if (xe_vm_is_closed(vm)) {
230 		err = -ENOENT;
231 		goto unlock_vm;
232 	}
233 
234 	vma = lookup_vma(vm, pf->page_addr);
235 	if (!vma) {
236 		err = -EINVAL;
237 		goto unlock_vm;
238 	}
239 
240 	atomic = access_is_atomic(pf->access_type);
241 
242 	if (xe_vma_is_cpu_addr_mirror(vma))
243 		err = xe_svm_handle_pagefault(vm, vma, gt,
244 					      pf->page_addr, atomic);
245 	else
246 		err = handle_vma_pagefault(gt, vma, atomic);
247 
248 unlock_vm:
249 	if (!err)
250 		vm->usm.last_fault_vma = vma;
251 	up_write(&vm->lock);
252 	xe_vm_put(vm);
253 
254 	return err;
255 }
256 
send_pagefault_reply(struct xe_guc * guc,struct xe_guc_pagefault_reply * reply)257 static int send_pagefault_reply(struct xe_guc *guc,
258 				struct xe_guc_pagefault_reply *reply)
259 {
260 	u32 action[] = {
261 		XE_GUC_ACTION_PAGE_FAULT_RES_DESC,
262 		reply->dw0,
263 		reply->dw1,
264 	};
265 
266 	return xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 0, 0);
267 }
268 
print_pagefault(struct xe_device * xe,struct pagefault * pf)269 static void print_pagefault(struct xe_device *xe, struct pagefault *pf)
270 {
271 	drm_dbg(&xe->drm, "\n\tASID: %d\n"
272 		 "\tVFID: %d\n"
273 		 "\tPDATA: 0x%04x\n"
274 		 "\tFaulted Address: 0x%08x%08x\n"
275 		 "\tFaultType: %d\n"
276 		 "\tAccessType: %d\n"
277 		 "\tFaultLevel: %d\n"
278 		 "\tEngineClass: %d %s\n"
279 		 "\tEngineInstance: %d\n",
280 		 pf->asid, pf->vfid, pf->pdata, upper_32_bits(pf->page_addr),
281 		 lower_32_bits(pf->page_addr),
282 		 pf->fault_type, pf->access_type, pf->fault_level,
283 		 pf->engine_class, xe_hw_engine_class_to_str(pf->engine_class),
284 		 pf->engine_instance);
285 }
286 
287 #define PF_MSG_LEN_DW	4
288 
get_pagefault(struct pf_queue * pf_queue,struct pagefault * pf)289 static bool get_pagefault(struct pf_queue *pf_queue, struct pagefault *pf)
290 {
291 	const struct xe_guc_pagefault_desc *desc;
292 	bool ret = false;
293 
294 	spin_lock_irq(&pf_queue->lock);
295 	if (pf_queue->tail != pf_queue->head) {
296 		desc = (const struct xe_guc_pagefault_desc *)
297 			(pf_queue->data + pf_queue->tail);
298 
299 		pf->fault_level = FIELD_GET(PFD_FAULT_LEVEL, desc->dw0);
300 		pf->trva_fault = FIELD_GET(XE2_PFD_TRVA_FAULT, desc->dw0);
301 		pf->engine_class = FIELD_GET(PFD_ENG_CLASS, desc->dw0);
302 		pf->engine_instance = FIELD_GET(PFD_ENG_INSTANCE, desc->dw0);
303 		pf->pdata = FIELD_GET(PFD_PDATA_HI, desc->dw1) <<
304 			PFD_PDATA_HI_SHIFT;
305 		pf->pdata |= FIELD_GET(PFD_PDATA_LO, desc->dw0);
306 		pf->asid = FIELD_GET(PFD_ASID, desc->dw1);
307 		pf->vfid = FIELD_GET(PFD_VFID, desc->dw2);
308 		pf->access_type = FIELD_GET(PFD_ACCESS_TYPE, desc->dw2);
309 		pf->fault_type = FIELD_GET(PFD_FAULT_TYPE, desc->dw2);
310 		pf->page_addr = (u64)(FIELD_GET(PFD_VIRTUAL_ADDR_HI, desc->dw3)) <<
311 			PFD_VIRTUAL_ADDR_HI_SHIFT;
312 		pf->page_addr |= FIELD_GET(PFD_VIRTUAL_ADDR_LO, desc->dw2) <<
313 			PFD_VIRTUAL_ADDR_LO_SHIFT;
314 
315 		pf_queue->tail = (pf_queue->tail + PF_MSG_LEN_DW) %
316 			pf_queue->num_dw;
317 		ret = true;
318 	}
319 	spin_unlock_irq(&pf_queue->lock);
320 
321 	return ret;
322 }
323 
pf_queue_full(struct pf_queue * pf_queue)324 static bool pf_queue_full(struct pf_queue *pf_queue)
325 {
326 	lockdep_assert_held(&pf_queue->lock);
327 
328 	return CIRC_SPACE(pf_queue->head, pf_queue->tail,
329 			  pf_queue->num_dw) <=
330 		PF_MSG_LEN_DW;
331 }
332 
xe_guc_pagefault_handler(struct xe_guc * guc,u32 * msg,u32 len)333 int xe_guc_pagefault_handler(struct xe_guc *guc, u32 *msg, u32 len)
334 {
335 	struct xe_gt *gt = guc_to_gt(guc);
336 	struct xe_device *xe = gt_to_xe(gt);
337 	struct pf_queue *pf_queue;
338 	unsigned long flags;
339 	u32 asid;
340 	bool full;
341 
342 	if (unlikely(len != PF_MSG_LEN_DW))
343 		return -EPROTO;
344 
345 	asid = FIELD_GET(PFD_ASID, msg[1]);
346 	pf_queue = gt->usm.pf_queue + (asid % NUM_PF_QUEUE);
347 
348 	/*
349 	 * The below logic doesn't work unless PF_QUEUE_NUM_DW % PF_MSG_LEN_DW == 0
350 	 */
351 	xe_gt_assert(gt, !(pf_queue->num_dw % PF_MSG_LEN_DW));
352 
353 	spin_lock_irqsave(&pf_queue->lock, flags);
354 	full = pf_queue_full(pf_queue);
355 	if (!full) {
356 		memcpy(pf_queue->data + pf_queue->head, msg, len * sizeof(u32));
357 		pf_queue->head = (pf_queue->head + len) %
358 			pf_queue->num_dw;
359 		queue_work(gt->usm.pf_wq, &pf_queue->worker);
360 	} else {
361 		drm_warn(&xe->drm, "PF Queue full, shouldn't be possible");
362 	}
363 	spin_unlock_irqrestore(&pf_queue->lock, flags);
364 
365 	return full ? -ENOSPC : 0;
366 }
367 
368 #define USM_QUEUE_MAX_RUNTIME_MS	20
369 
pf_queue_work_func(struct work_struct * w)370 static void pf_queue_work_func(struct work_struct *w)
371 {
372 	struct pf_queue *pf_queue = container_of(w, struct pf_queue, worker);
373 	struct xe_gt *gt = pf_queue->gt;
374 	struct xe_device *xe = gt_to_xe(gt);
375 	struct xe_guc_pagefault_reply reply = {};
376 	struct pagefault pf = {};
377 	unsigned long threshold;
378 	int ret;
379 
380 	threshold = jiffies + msecs_to_jiffies(USM_QUEUE_MAX_RUNTIME_MS);
381 
382 	while (get_pagefault(pf_queue, &pf)) {
383 		ret = handle_pagefault(gt, &pf);
384 		if (unlikely(ret)) {
385 			print_pagefault(xe, &pf);
386 			pf.fault_unsuccessful = 1;
387 			drm_dbg(&xe->drm, "Fault response: Unsuccessful %d\n", ret);
388 		}
389 
390 		reply.dw0 = FIELD_PREP(PFR_VALID, 1) |
391 			FIELD_PREP(PFR_SUCCESS, pf.fault_unsuccessful) |
392 			FIELD_PREP(PFR_REPLY, PFR_ACCESS) |
393 			FIELD_PREP(PFR_DESC_TYPE, FAULT_RESPONSE_DESC) |
394 			FIELD_PREP(PFR_ASID, pf.asid);
395 
396 		reply.dw1 = FIELD_PREP(PFR_VFID, pf.vfid) |
397 			FIELD_PREP(PFR_ENG_INSTANCE, pf.engine_instance) |
398 			FIELD_PREP(PFR_ENG_CLASS, pf.engine_class) |
399 			FIELD_PREP(PFR_PDATA, pf.pdata);
400 
401 		send_pagefault_reply(&gt->uc.guc, &reply);
402 
403 		if (time_after(jiffies, threshold) &&
404 		    pf_queue->tail != pf_queue->head) {
405 			queue_work(gt->usm.pf_wq, w);
406 			break;
407 		}
408 	}
409 }
410 
411 static void acc_queue_work_func(struct work_struct *w);
412 
pagefault_fini(void * arg)413 static void pagefault_fini(void *arg)
414 {
415 	struct xe_gt *gt = arg;
416 	struct xe_device *xe = gt_to_xe(gt);
417 
418 	if (!xe->info.has_usm)
419 		return;
420 
421 	destroy_workqueue(gt->usm.acc_wq);
422 	destroy_workqueue(gt->usm.pf_wq);
423 }
424 
xe_alloc_pf_queue(struct xe_gt * gt,struct pf_queue * pf_queue)425 static int xe_alloc_pf_queue(struct xe_gt *gt, struct pf_queue *pf_queue)
426 {
427 	struct xe_device *xe = gt_to_xe(gt);
428 	xe_dss_mask_t all_dss;
429 	int num_dss, num_eus;
430 
431 	bitmap_or(all_dss, gt->fuse_topo.g_dss_mask, gt->fuse_topo.c_dss_mask,
432 		  XE_MAX_DSS_FUSE_BITS);
433 
434 	num_dss = bitmap_weight(all_dss, XE_MAX_DSS_FUSE_BITS);
435 	num_eus = bitmap_weight(gt->fuse_topo.eu_mask_per_dss,
436 				XE_MAX_EU_FUSE_BITS) * num_dss;
437 
438 	/*
439 	 * user can issue separate page faults per EU and per CS
440 	 *
441 	 * XXX: Multiplier required as compute UMD are getting PF queue errors
442 	 * without it. Follow on why this multiplier is required.
443 	 */
444 #define PF_MULTIPLIER	8
445 	pf_queue->num_dw =
446 		(num_eus + XE_NUM_HW_ENGINES) * PF_MSG_LEN_DW * PF_MULTIPLIER;
447 	pf_queue->num_dw = roundup_pow_of_two(pf_queue->num_dw);
448 #undef PF_MULTIPLIER
449 
450 	pf_queue->gt = gt;
451 	pf_queue->data = devm_kcalloc(xe->drm.dev, pf_queue->num_dw,
452 				      sizeof(u32), GFP_KERNEL);
453 	if (!pf_queue->data)
454 		return -ENOMEM;
455 
456 	spin_lock_init(&pf_queue->lock);
457 	INIT_WORK(&pf_queue->worker, pf_queue_work_func);
458 
459 	return 0;
460 }
461 
xe_gt_pagefault_init(struct xe_gt * gt)462 int xe_gt_pagefault_init(struct xe_gt *gt)
463 {
464 	struct xe_device *xe = gt_to_xe(gt);
465 	int i, ret = 0;
466 
467 	if (!xe->info.has_usm)
468 		return 0;
469 
470 	for (i = 0; i < NUM_PF_QUEUE; ++i) {
471 		ret = xe_alloc_pf_queue(gt, &gt->usm.pf_queue[i]);
472 		if (ret)
473 			return ret;
474 	}
475 	for (i = 0; i < NUM_ACC_QUEUE; ++i) {
476 		gt->usm.acc_queue[i].gt = gt;
477 		spin_lock_init(&gt->usm.acc_queue[i].lock);
478 		INIT_WORK(&gt->usm.acc_queue[i].worker, acc_queue_work_func);
479 	}
480 
481 	gt->usm.pf_wq = alloc_workqueue("xe_gt_page_fault_work_queue",
482 					WQ_UNBOUND | WQ_HIGHPRI, NUM_PF_QUEUE);
483 	if (!gt->usm.pf_wq)
484 		return -ENOMEM;
485 
486 	gt->usm.acc_wq = alloc_workqueue("xe_gt_access_counter_work_queue",
487 					 WQ_UNBOUND | WQ_HIGHPRI,
488 					 NUM_ACC_QUEUE);
489 	if (!gt->usm.acc_wq) {
490 		destroy_workqueue(gt->usm.pf_wq);
491 		return -ENOMEM;
492 	}
493 
494 	return devm_add_action_or_reset(xe->drm.dev, pagefault_fini, gt);
495 }
496 
xe_gt_pagefault_reset(struct xe_gt * gt)497 void xe_gt_pagefault_reset(struct xe_gt *gt)
498 {
499 	struct xe_device *xe = gt_to_xe(gt);
500 	int i;
501 
502 	if (!xe->info.has_usm)
503 		return;
504 
505 	for (i = 0; i < NUM_PF_QUEUE; ++i) {
506 		spin_lock_irq(&gt->usm.pf_queue[i].lock);
507 		gt->usm.pf_queue[i].head = 0;
508 		gt->usm.pf_queue[i].tail = 0;
509 		spin_unlock_irq(&gt->usm.pf_queue[i].lock);
510 	}
511 
512 	for (i = 0; i < NUM_ACC_QUEUE; ++i) {
513 		spin_lock(&gt->usm.acc_queue[i].lock);
514 		gt->usm.acc_queue[i].head = 0;
515 		gt->usm.acc_queue[i].tail = 0;
516 		spin_unlock(&gt->usm.acc_queue[i].lock);
517 	}
518 }
519 
granularity_in_byte(int val)520 static int granularity_in_byte(int val)
521 {
522 	switch (val) {
523 	case 0:
524 		return SZ_128K;
525 	case 1:
526 		return SZ_2M;
527 	case 2:
528 		return SZ_16M;
529 	case 3:
530 		return SZ_64M;
531 	default:
532 		return 0;
533 	}
534 }
535 
sub_granularity_in_byte(int val)536 static int sub_granularity_in_byte(int val)
537 {
538 	return (granularity_in_byte(val) / 32);
539 }
540 
print_acc(struct xe_device * xe,struct acc * acc)541 static void print_acc(struct xe_device *xe, struct acc *acc)
542 {
543 	drm_warn(&xe->drm, "Access counter request:\n"
544 		 "\tType: %s\n"
545 		 "\tASID: %d\n"
546 		 "\tVFID: %d\n"
547 		 "\tEngine: %d:%d\n"
548 		 "\tGranularity: 0x%x KB Region/ %d KB sub-granularity\n"
549 		 "\tSub_Granularity Vector: 0x%08x\n"
550 		 "\tVA Range base: 0x%016llx\n",
551 		 acc->access_type ? "AC_NTFY_VAL" : "AC_TRIG_VAL",
552 		 acc->asid, acc->vfid, acc->engine_class, acc->engine_instance,
553 		 granularity_in_byte(acc->granularity) / SZ_1K,
554 		 sub_granularity_in_byte(acc->granularity) / SZ_1K,
555 		 acc->sub_granularity, acc->va_range_base);
556 }
557 
get_acc_vma(struct xe_vm * vm,struct acc * acc)558 static struct xe_vma *get_acc_vma(struct xe_vm *vm, struct acc *acc)
559 {
560 	u64 page_va = acc->va_range_base + (ffs(acc->sub_granularity) - 1) *
561 		sub_granularity_in_byte(acc->granularity);
562 
563 	return xe_vm_find_overlapping_vma(vm, page_va, SZ_4K);
564 }
565 
handle_acc(struct xe_gt * gt,struct acc * acc)566 static int handle_acc(struct xe_gt *gt, struct acc *acc)
567 {
568 	struct xe_device *xe = gt_to_xe(gt);
569 	struct xe_tile *tile = gt_to_tile(gt);
570 	struct drm_exec exec;
571 	struct xe_vm *vm;
572 	struct xe_vma *vma;
573 	int ret = 0;
574 
575 	/* We only support ACC_TRIGGER at the moment */
576 	if (acc->access_type != ACC_TRIGGER)
577 		return -EINVAL;
578 
579 	vm = asid_to_vm(xe, acc->asid);
580 	if (IS_ERR(vm))
581 		return PTR_ERR(vm);
582 
583 	down_read(&vm->lock);
584 
585 	/* Lookup VMA */
586 	vma = get_acc_vma(vm, acc);
587 	if (!vma) {
588 		ret = -EINVAL;
589 		goto unlock_vm;
590 	}
591 
592 	trace_xe_vma_acc(vma);
593 
594 	/* Userptr or null can't be migrated, nothing to do */
595 	if (xe_vma_has_no_bo(vma))
596 		goto unlock_vm;
597 
598 	/* Lock VM and BOs dma-resv */
599 	drm_exec_init(&exec, 0, 0);
600 	drm_exec_until_all_locked(&exec) {
601 		ret = xe_pf_begin(&exec, vma, true, tile->id);
602 		drm_exec_retry_on_contention(&exec);
603 		if (ret)
604 			break;
605 	}
606 
607 	drm_exec_fini(&exec);
608 unlock_vm:
609 	up_read(&vm->lock);
610 	xe_vm_put(vm);
611 
612 	return ret;
613 }
614 
615 #define make_u64(hi__, low__)  ((u64)(hi__) << 32 | (u64)(low__))
616 
617 #define ACC_MSG_LEN_DW        4
618 
get_acc(struct acc_queue * acc_queue,struct acc * acc)619 static bool get_acc(struct acc_queue *acc_queue, struct acc *acc)
620 {
621 	const struct xe_guc_acc_desc *desc;
622 	bool ret = false;
623 
624 	spin_lock(&acc_queue->lock);
625 	if (acc_queue->tail != acc_queue->head) {
626 		desc = (const struct xe_guc_acc_desc *)
627 			(acc_queue->data + acc_queue->tail);
628 
629 		acc->granularity = FIELD_GET(ACC_GRANULARITY, desc->dw2);
630 		acc->sub_granularity = FIELD_GET(ACC_SUBG_HI, desc->dw1) << 31 |
631 			FIELD_GET(ACC_SUBG_LO, desc->dw0);
632 		acc->engine_class = FIELD_GET(ACC_ENG_CLASS, desc->dw1);
633 		acc->engine_instance = FIELD_GET(ACC_ENG_INSTANCE, desc->dw1);
634 		acc->asid =  FIELD_GET(ACC_ASID, desc->dw1);
635 		acc->vfid =  FIELD_GET(ACC_VFID, desc->dw2);
636 		acc->access_type = FIELD_GET(ACC_TYPE, desc->dw0);
637 		acc->va_range_base = make_u64(desc->dw3 & ACC_VIRTUAL_ADDR_RANGE_HI,
638 					      desc->dw2 & ACC_VIRTUAL_ADDR_RANGE_LO);
639 
640 		acc_queue->tail = (acc_queue->tail + ACC_MSG_LEN_DW) %
641 				  ACC_QUEUE_NUM_DW;
642 		ret = true;
643 	}
644 	spin_unlock(&acc_queue->lock);
645 
646 	return ret;
647 }
648 
acc_queue_work_func(struct work_struct * w)649 static void acc_queue_work_func(struct work_struct *w)
650 {
651 	struct acc_queue *acc_queue = container_of(w, struct acc_queue, worker);
652 	struct xe_gt *gt = acc_queue->gt;
653 	struct xe_device *xe = gt_to_xe(gt);
654 	struct acc acc = {};
655 	unsigned long threshold;
656 	int ret;
657 
658 	threshold = jiffies + msecs_to_jiffies(USM_QUEUE_MAX_RUNTIME_MS);
659 
660 	while (get_acc(acc_queue, &acc)) {
661 		ret = handle_acc(gt, &acc);
662 		if (unlikely(ret)) {
663 			print_acc(xe, &acc);
664 			drm_warn(&xe->drm, "ACC: Unsuccessful %d\n", ret);
665 		}
666 
667 		if (time_after(jiffies, threshold) &&
668 		    acc_queue->tail != acc_queue->head) {
669 			queue_work(gt->usm.acc_wq, w);
670 			break;
671 		}
672 	}
673 }
674 
acc_queue_full(struct acc_queue * acc_queue)675 static bool acc_queue_full(struct acc_queue *acc_queue)
676 {
677 	lockdep_assert_held(&acc_queue->lock);
678 
679 	return CIRC_SPACE(acc_queue->head, acc_queue->tail, ACC_QUEUE_NUM_DW) <=
680 		ACC_MSG_LEN_DW;
681 }
682 
xe_guc_access_counter_notify_handler(struct xe_guc * guc,u32 * msg,u32 len)683 int xe_guc_access_counter_notify_handler(struct xe_guc *guc, u32 *msg, u32 len)
684 {
685 	struct xe_gt *gt = guc_to_gt(guc);
686 	struct acc_queue *acc_queue;
687 	u32 asid;
688 	bool full;
689 
690 	/*
691 	 * The below logic doesn't work unless ACC_QUEUE_NUM_DW % ACC_MSG_LEN_DW == 0
692 	 */
693 	BUILD_BUG_ON(ACC_QUEUE_NUM_DW % ACC_MSG_LEN_DW);
694 
695 	if (unlikely(len != ACC_MSG_LEN_DW))
696 		return -EPROTO;
697 
698 	asid = FIELD_GET(ACC_ASID, msg[1]);
699 	acc_queue = &gt->usm.acc_queue[asid % NUM_ACC_QUEUE];
700 
701 	spin_lock(&acc_queue->lock);
702 	full = acc_queue_full(acc_queue);
703 	if (!full) {
704 		memcpy(acc_queue->data + acc_queue->head, msg,
705 		       len * sizeof(u32));
706 		acc_queue->head = (acc_queue->head + len) % ACC_QUEUE_NUM_DW;
707 		queue_work(gt->usm.acc_wq, &acc_queue->worker);
708 	} else {
709 		drm_warn(&gt_to_xe(gt)->drm, "ACC Queue full, dropping ACC");
710 	}
711 	spin_unlock(&acc_queue->lock);
712 
713 	return full ? -ENOSPC : 0;
714 }
715