xref: /linux/drivers/gpu/drm/xe/xe_gt_pagefault.c (revision d53b8e36925256097a08d7cb749198d85cbf9b2b)
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2022 Intel Corporation
4  */
5 
6 #include "xe_gt_pagefault.h"
7 
8 #include <linux/bitfield.h>
9 #include <linux/circ_buf.h>
10 
11 #include <drm/drm_exec.h>
12 #include <drm/drm_managed.h>
13 #include <drm/ttm/ttm_execbuf_util.h>
14 
15 #include "abi/guc_actions_abi.h"
16 #include "xe_bo.h"
17 #include "xe_gt.h"
18 #include "xe_gt_tlb_invalidation.h"
19 #include "xe_guc.h"
20 #include "xe_guc_ct.h"
21 #include "xe_migrate.h"
22 #include "xe_trace_bo.h"
23 #include "xe_vm.h"
24 
25 struct pagefault {
26 	u64 page_addr;
27 	u32 asid;
28 	u16 pdata;
29 	u8 vfid;
30 	u8 access_type;
31 	u8 fault_type;
32 	u8 fault_level;
33 	u8 engine_class;
34 	u8 engine_instance;
35 	u8 fault_unsuccessful;
36 	bool trva_fault;
37 };
38 
39 enum access_type {
40 	ACCESS_TYPE_READ = 0,
41 	ACCESS_TYPE_WRITE = 1,
42 	ACCESS_TYPE_ATOMIC = 2,
43 	ACCESS_TYPE_RESERVED = 3,
44 };
45 
46 enum fault_type {
47 	NOT_PRESENT = 0,
48 	WRITE_ACCESS_VIOLATION = 1,
49 	ATOMIC_ACCESS_VIOLATION = 2,
50 };
51 
52 struct acc {
53 	u64 va_range_base;
54 	u32 asid;
55 	u32 sub_granularity;
56 	u8 granularity;
57 	u8 vfid;
58 	u8 access_type;
59 	u8 engine_class;
60 	u8 engine_instance;
61 };
62 
63 static bool access_is_atomic(enum access_type access_type)
64 {
65 	return access_type == ACCESS_TYPE_ATOMIC;
66 }
67 
68 static bool vma_is_valid(struct xe_tile *tile, struct xe_vma *vma)
69 {
70 	return BIT(tile->id) & vma->tile_present &&
71 		!(BIT(tile->id) & vma->tile_invalidated);
72 }
73 
74 static bool vma_matches(struct xe_vma *vma, u64 page_addr)
75 {
76 	if (page_addr > xe_vma_end(vma) - 1 ||
77 	    page_addr + SZ_4K - 1 < xe_vma_start(vma))
78 		return false;
79 
80 	return true;
81 }
82 
83 static struct xe_vma *lookup_vma(struct xe_vm *vm, u64 page_addr)
84 {
85 	struct xe_vma *vma = NULL;
86 
87 	if (vm->usm.last_fault_vma) {   /* Fast lookup */
88 		if (vma_matches(vm->usm.last_fault_vma, page_addr))
89 			vma = vm->usm.last_fault_vma;
90 	}
91 	if (!vma)
92 		vma = xe_vm_find_overlapping_vma(vm, page_addr, SZ_4K);
93 
94 	return vma;
95 }
96 
97 static int xe_pf_begin(struct drm_exec *exec, struct xe_vma *vma,
98 		       bool atomic, unsigned int id)
99 {
100 	struct xe_bo *bo = xe_vma_bo(vma);
101 	struct xe_vm *vm = xe_vma_vm(vma);
102 	int err;
103 
104 	err = xe_vm_lock_vma(exec, vma);
105 	if (err)
106 		return err;
107 
108 	if (atomic && IS_DGFX(vm->xe)) {
109 		if (xe_vma_is_userptr(vma)) {
110 			err = -EACCES;
111 			return err;
112 		}
113 
114 		/* Migrate to VRAM, move should invalidate the VMA first */
115 		err = xe_bo_migrate(bo, XE_PL_VRAM0 + id);
116 		if (err)
117 			return err;
118 	} else if (bo) {
119 		/* Create backing store if needed */
120 		err = xe_bo_validate(bo, vm, true);
121 		if (err)
122 			return err;
123 	}
124 
125 	return 0;
126 }
127 
128 static int handle_vma_pagefault(struct xe_tile *tile, struct pagefault *pf,
129 				struct xe_vma *vma)
130 {
131 	struct xe_vm *vm = xe_vma_vm(vma);
132 	struct drm_exec exec;
133 	struct dma_fence *fence;
134 	ktime_t end = 0;
135 	int err;
136 	bool atomic;
137 
138 	trace_xe_vma_pagefault(vma);
139 	atomic = access_is_atomic(pf->access_type);
140 
141 	/* Check if VMA is valid */
142 	if (vma_is_valid(tile, vma) && !atomic)
143 		return 0;
144 
145 retry_userptr:
146 	if (xe_vma_is_userptr(vma) &&
147 	    xe_vma_userptr_check_repin(to_userptr_vma(vma))) {
148 		struct xe_userptr_vma *uvma = to_userptr_vma(vma);
149 
150 		err = xe_vma_userptr_pin_pages(uvma);
151 		if (err)
152 			return err;
153 	}
154 
155 	/* Lock VM and BOs dma-resv */
156 	drm_exec_init(&exec, 0, 0);
157 	drm_exec_until_all_locked(&exec) {
158 		err = xe_pf_begin(&exec, vma, atomic, tile->id);
159 		drm_exec_retry_on_contention(&exec);
160 		if (xe_vm_validate_should_retry(&exec, err, &end))
161 			err = -EAGAIN;
162 		if (err)
163 			goto unlock_dma_resv;
164 
165 		/* Bind VMA only to the GT that has faulted */
166 		trace_xe_vma_pf_bind(vma);
167 		fence = xe_vma_rebind(vm, vma, BIT(tile->id));
168 		if (IS_ERR(fence)) {
169 			err = PTR_ERR(fence);
170 			if (xe_vm_validate_should_retry(&exec, err, &end))
171 				err = -EAGAIN;
172 			goto unlock_dma_resv;
173 		}
174 	}
175 
176 	dma_fence_wait(fence, false);
177 	dma_fence_put(fence);
178 	vma->tile_invalidated &= ~BIT(tile->id);
179 
180 unlock_dma_resv:
181 	drm_exec_fini(&exec);
182 	if (err == -EAGAIN)
183 		goto retry_userptr;
184 
185 	return err;
186 }
187 
188 static int handle_pagefault(struct xe_gt *gt, struct pagefault *pf)
189 {
190 	struct xe_device *xe = gt_to_xe(gt);
191 	struct xe_tile *tile = gt_to_tile(gt);
192 	struct xe_vm *vm;
193 	struct xe_vma *vma = NULL;
194 	int err;
195 
196 	/* SW isn't expected to handle TRTT faults */
197 	if (pf->trva_fault)
198 		return -EFAULT;
199 
200 	/* ASID to VM */
201 	mutex_lock(&xe->usm.lock);
202 	vm = xa_load(&xe->usm.asid_to_vm, pf->asid);
203 	if (vm && xe_vm_in_fault_mode(vm))
204 		xe_vm_get(vm);
205 	else
206 		vm = NULL;
207 	mutex_unlock(&xe->usm.lock);
208 	if (!vm)
209 		return -EINVAL;
210 
211 	/*
212 	 * TODO: Change to read lock? Using write lock for simplicity.
213 	 */
214 	down_write(&vm->lock);
215 	vma = lookup_vma(vm, pf->page_addr);
216 	if (!vma) {
217 		err = -EINVAL;
218 		goto unlock_vm;
219 	}
220 
221 	err = handle_vma_pagefault(tile, pf, vma);
222 
223 unlock_vm:
224 	if (!err)
225 		vm->usm.last_fault_vma = vma;
226 	up_write(&vm->lock);
227 	xe_vm_put(vm);
228 
229 	return err;
230 }
231 
232 static int send_pagefault_reply(struct xe_guc *guc,
233 				struct xe_guc_pagefault_reply *reply)
234 {
235 	u32 action[] = {
236 		XE_GUC_ACTION_PAGE_FAULT_RES_DESC,
237 		reply->dw0,
238 		reply->dw1,
239 	};
240 
241 	return xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 0, 0);
242 }
243 
244 static void print_pagefault(struct xe_device *xe, struct pagefault *pf)
245 {
246 	drm_dbg(&xe->drm, "\n\tASID: %d\n"
247 		 "\tVFID: %d\n"
248 		 "\tPDATA: 0x%04x\n"
249 		 "\tFaulted Address: 0x%08x%08x\n"
250 		 "\tFaultType: %d\n"
251 		 "\tAccessType: %d\n"
252 		 "\tFaultLevel: %d\n"
253 		 "\tEngineClass: %d\n"
254 		 "\tEngineInstance: %d\n",
255 		 pf->asid, pf->vfid, pf->pdata, upper_32_bits(pf->page_addr),
256 		 lower_32_bits(pf->page_addr),
257 		 pf->fault_type, pf->access_type, pf->fault_level,
258 		 pf->engine_class, pf->engine_instance);
259 }
260 
261 #define PF_MSG_LEN_DW	4
262 
263 static bool get_pagefault(struct pf_queue *pf_queue, struct pagefault *pf)
264 {
265 	const struct xe_guc_pagefault_desc *desc;
266 	bool ret = false;
267 
268 	spin_lock_irq(&pf_queue->lock);
269 	if (pf_queue->tail != pf_queue->head) {
270 		desc = (const struct xe_guc_pagefault_desc *)
271 			(pf_queue->data + pf_queue->tail);
272 
273 		pf->fault_level = FIELD_GET(PFD_FAULT_LEVEL, desc->dw0);
274 		pf->trva_fault = FIELD_GET(XE2_PFD_TRVA_FAULT, desc->dw0);
275 		pf->engine_class = FIELD_GET(PFD_ENG_CLASS, desc->dw0);
276 		pf->engine_instance = FIELD_GET(PFD_ENG_INSTANCE, desc->dw0);
277 		pf->pdata = FIELD_GET(PFD_PDATA_HI, desc->dw1) <<
278 			PFD_PDATA_HI_SHIFT;
279 		pf->pdata |= FIELD_GET(PFD_PDATA_LO, desc->dw0);
280 		pf->asid = FIELD_GET(PFD_ASID, desc->dw1);
281 		pf->vfid = FIELD_GET(PFD_VFID, desc->dw2);
282 		pf->access_type = FIELD_GET(PFD_ACCESS_TYPE, desc->dw2);
283 		pf->fault_type = FIELD_GET(PFD_FAULT_TYPE, desc->dw2);
284 		pf->page_addr = (u64)(FIELD_GET(PFD_VIRTUAL_ADDR_HI, desc->dw3)) <<
285 			PFD_VIRTUAL_ADDR_HI_SHIFT;
286 		pf->page_addr |= FIELD_GET(PFD_VIRTUAL_ADDR_LO, desc->dw2) <<
287 			PFD_VIRTUAL_ADDR_LO_SHIFT;
288 
289 		pf_queue->tail = (pf_queue->tail + PF_MSG_LEN_DW) %
290 			pf_queue->num_dw;
291 		ret = true;
292 	}
293 	spin_unlock_irq(&pf_queue->lock);
294 
295 	return ret;
296 }
297 
298 static bool pf_queue_full(struct pf_queue *pf_queue)
299 {
300 	lockdep_assert_held(&pf_queue->lock);
301 
302 	return CIRC_SPACE(pf_queue->head, pf_queue->tail,
303 			  pf_queue->num_dw) <=
304 		PF_MSG_LEN_DW;
305 }
306 
307 int xe_guc_pagefault_handler(struct xe_guc *guc, u32 *msg, u32 len)
308 {
309 	struct xe_gt *gt = guc_to_gt(guc);
310 	struct xe_device *xe = gt_to_xe(gt);
311 	struct pf_queue *pf_queue;
312 	unsigned long flags;
313 	u32 asid;
314 	bool full;
315 
316 	if (unlikely(len != PF_MSG_LEN_DW))
317 		return -EPROTO;
318 
319 	asid = FIELD_GET(PFD_ASID, msg[1]);
320 	pf_queue = gt->usm.pf_queue + (asid % NUM_PF_QUEUE);
321 
322 	/*
323 	 * The below logic doesn't work unless PF_QUEUE_NUM_DW % PF_MSG_LEN_DW == 0
324 	 */
325 	xe_gt_assert(gt, !(pf_queue->num_dw % PF_MSG_LEN_DW));
326 
327 	spin_lock_irqsave(&pf_queue->lock, flags);
328 	full = pf_queue_full(pf_queue);
329 	if (!full) {
330 		memcpy(pf_queue->data + pf_queue->head, msg, len * sizeof(u32));
331 		pf_queue->head = (pf_queue->head + len) %
332 			pf_queue->num_dw;
333 		queue_work(gt->usm.pf_wq, &pf_queue->worker);
334 	} else {
335 		drm_warn(&xe->drm, "PF Queue full, shouldn't be possible");
336 	}
337 	spin_unlock_irqrestore(&pf_queue->lock, flags);
338 
339 	return full ? -ENOSPC : 0;
340 }
341 
342 #define USM_QUEUE_MAX_RUNTIME_MS	20
343 
344 static void pf_queue_work_func(struct work_struct *w)
345 {
346 	struct pf_queue *pf_queue = container_of(w, struct pf_queue, worker);
347 	struct xe_gt *gt = pf_queue->gt;
348 	struct xe_device *xe = gt_to_xe(gt);
349 	struct xe_guc_pagefault_reply reply = {};
350 	struct pagefault pf = {};
351 	unsigned long threshold;
352 	int ret;
353 
354 	threshold = jiffies + msecs_to_jiffies(USM_QUEUE_MAX_RUNTIME_MS);
355 
356 	while (get_pagefault(pf_queue, &pf)) {
357 		ret = handle_pagefault(gt, &pf);
358 		if (unlikely(ret)) {
359 			print_pagefault(xe, &pf);
360 			pf.fault_unsuccessful = 1;
361 			drm_dbg(&xe->drm, "Fault response: Unsuccessful %d\n", ret);
362 		}
363 
364 		reply.dw0 = FIELD_PREP(PFR_VALID, 1) |
365 			FIELD_PREP(PFR_SUCCESS, pf.fault_unsuccessful) |
366 			FIELD_PREP(PFR_REPLY, PFR_ACCESS) |
367 			FIELD_PREP(PFR_DESC_TYPE, FAULT_RESPONSE_DESC) |
368 			FIELD_PREP(PFR_ASID, pf.asid);
369 
370 		reply.dw1 = FIELD_PREP(PFR_VFID, pf.vfid) |
371 			FIELD_PREP(PFR_ENG_INSTANCE, pf.engine_instance) |
372 			FIELD_PREP(PFR_ENG_CLASS, pf.engine_class) |
373 			FIELD_PREP(PFR_PDATA, pf.pdata);
374 
375 		send_pagefault_reply(&gt->uc.guc, &reply);
376 
377 		if (time_after(jiffies, threshold) &&
378 		    pf_queue->tail != pf_queue->head) {
379 			queue_work(gt->usm.pf_wq, w);
380 			break;
381 		}
382 	}
383 }
384 
385 static void acc_queue_work_func(struct work_struct *w);
386 
387 static void pagefault_fini(void *arg)
388 {
389 	struct xe_gt *gt = arg;
390 	struct xe_device *xe = gt_to_xe(gt);
391 	int i;
392 
393 	if (!xe->info.has_usm)
394 		return;
395 
396 	destroy_workqueue(gt->usm.acc_wq);
397 	destroy_workqueue(gt->usm.pf_wq);
398 
399 	for (i = 0; i < NUM_PF_QUEUE; ++i)
400 		kfree(gt->usm.pf_queue[i].data);
401 }
402 
403 static int xe_alloc_pf_queue(struct xe_gt *gt, struct pf_queue *pf_queue)
404 {
405 	xe_dss_mask_t all_dss;
406 	int num_dss, num_eus;
407 
408 	bitmap_or(all_dss, gt->fuse_topo.g_dss_mask, gt->fuse_topo.c_dss_mask,
409 		  XE_MAX_DSS_FUSE_BITS);
410 
411 	num_dss = bitmap_weight(all_dss, XE_MAX_DSS_FUSE_BITS);
412 	num_eus = bitmap_weight(gt->fuse_topo.eu_mask_per_dss,
413 				XE_MAX_EU_FUSE_BITS) * num_dss;
414 
415 	/* user can issue separate page faults per EU and per CS */
416 	pf_queue->num_dw =
417 		(num_eus + XE_NUM_HW_ENGINES) * PF_MSG_LEN_DW;
418 
419 	pf_queue->gt = gt;
420 	pf_queue->data = kcalloc(pf_queue->num_dw, sizeof(u32), GFP_KERNEL);
421 	if (!pf_queue->data)
422 		return -ENOMEM;
423 
424 	spin_lock_init(&pf_queue->lock);
425 	INIT_WORK(&pf_queue->worker, pf_queue_work_func);
426 
427 	return 0;
428 }
429 
430 int xe_gt_pagefault_init(struct xe_gt *gt)
431 {
432 	struct xe_device *xe = gt_to_xe(gt);
433 	int i, ret = 0;
434 
435 	if (!xe->info.has_usm)
436 		return 0;
437 
438 	for (i = 0; i < NUM_PF_QUEUE; ++i) {
439 		ret = xe_alloc_pf_queue(gt, &gt->usm.pf_queue[i]);
440 		if (ret)
441 			return ret;
442 	}
443 	for (i = 0; i < NUM_ACC_QUEUE; ++i) {
444 		gt->usm.acc_queue[i].gt = gt;
445 		spin_lock_init(&gt->usm.acc_queue[i].lock);
446 		INIT_WORK(&gt->usm.acc_queue[i].worker, acc_queue_work_func);
447 	}
448 
449 	gt->usm.pf_wq = alloc_workqueue("xe_gt_page_fault_work_queue",
450 					WQ_UNBOUND | WQ_HIGHPRI, NUM_PF_QUEUE);
451 	if (!gt->usm.pf_wq)
452 		return -ENOMEM;
453 
454 	gt->usm.acc_wq = alloc_workqueue("xe_gt_access_counter_work_queue",
455 					 WQ_UNBOUND | WQ_HIGHPRI,
456 					 NUM_ACC_QUEUE);
457 	if (!gt->usm.acc_wq) {
458 		destroy_workqueue(gt->usm.pf_wq);
459 		return -ENOMEM;
460 	}
461 
462 	return devm_add_action_or_reset(xe->drm.dev, pagefault_fini, gt);
463 }
464 
465 void xe_gt_pagefault_reset(struct xe_gt *gt)
466 {
467 	struct xe_device *xe = gt_to_xe(gt);
468 	int i;
469 
470 	if (!xe->info.has_usm)
471 		return;
472 
473 	for (i = 0; i < NUM_PF_QUEUE; ++i) {
474 		spin_lock_irq(&gt->usm.pf_queue[i].lock);
475 		gt->usm.pf_queue[i].head = 0;
476 		gt->usm.pf_queue[i].tail = 0;
477 		spin_unlock_irq(&gt->usm.pf_queue[i].lock);
478 	}
479 
480 	for (i = 0; i < NUM_ACC_QUEUE; ++i) {
481 		spin_lock(&gt->usm.acc_queue[i].lock);
482 		gt->usm.acc_queue[i].head = 0;
483 		gt->usm.acc_queue[i].tail = 0;
484 		spin_unlock(&gt->usm.acc_queue[i].lock);
485 	}
486 }
487 
488 static int granularity_in_byte(int val)
489 {
490 	switch (val) {
491 	case 0:
492 		return SZ_128K;
493 	case 1:
494 		return SZ_2M;
495 	case 2:
496 		return SZ_16M;
497 	case 3:
498 		return SZ_64M;
499 	default:
500 		return 0;
501 	}
502 }
503 
504 static int sub_granularity_in_byte(int val)
505 {
506 	return (granularity_in_byte(val) / 32);
507 }
508 
509 static void print_acc(struct xe_device *xe, struct acc *acc)
510 {
511 	drm_warn(&xe->drm, "Access counter request:\n"
512 		 "\tType: %s\n"
513 		 "\tASID: %d\n"
514 		 "\tVFID: %d\n"
515 		 "\tEngine: %d:%d\n"
516 		 "\tGranularity: 0x%x KB Region/ %d KB sub-granularity\n"
517 		 "\tSub_Granularity Vector: 0x%08x\n"
518 		 "\tVA Range base: 0x%016llx\n",
519 		 acc->access_type ? "AC_NTFY_VAL" : "AC_TRIG_VAL",
520 		 acc->asid, acc->vfid, acc->engine_class, acc->engine_instance,
521 		 granularity_in_byte(acc->granularity) / SZ_1K,
522 		 sub_granularity_in_byte(acc->granularity) / SZ_1K,
523 		 acc->sub_granularity, acc->va_range_base);
524 }
525 
526 static struct xe_vma *get_acc_vma(struct xe_vm *vm, struct acc *acc)
527 {
528 	u64 page_va = acc->va_range_base + (ffs(acc->sub_granularity) - 1) *
529 		sub_granularity_in_byte(acc->granularity);
530 
531 	return xe_vm_find_overlapping_vma(vm, page_va, SZ_4K);
532 }
533 
534 static int handle_acc(struct xe_gt *gt, struct acc *acc)
535 {
536 	struct xe_device *xe = gt_to_xe(gt);
537 	struct xe_tile *tile = gt_to_tile(gt);
538 	struct drm_exec exec;
539 	struct xe_vm *vm;
540 	struct xe_vma *vma;
541 	int ret = 0;
542 
543 	/* We only support ACC_TRIGGER at the moment */
544 	if (acc->access_type != ACC_TRIGGER)
545 		return -EINVAL;
546 
547 	/* ASID to VM */
548 	mutex_lock(&xe->usm.lock);
549 	vm = xa_load(&xe->usm.asid_to_vm, acc->asid);
550 	if (vm)
551 		xe_vm_get(vm);
552 	mutex_unlock(&xe->usm.lock);
553 	if (!vm || !xe_vm_in_fault_mode(vm))
554 		return -EINVAL;
555 
556 	down_read(&vm->lock);
557 
558 	/* Lookup VMA */
559 	vma = get_acc_vma(vm, acc);
560 	if (!vma) {
561 		ret = -EINVAL;
562 		goto unlock_vm;
563 	}
564 
565 	trace_xe_vma_acc(vma);
566 
567 	/* Userptr or null can't be migrated, nothing to do */
568 	if (xe_vma_has_no_bo(vma))
569 		goto unlock_vm;
570 
571 	/* Lock VM and BOs dma-resv */
572 	drm_exec_init(&exec, 0, 0);
573 	drm_exec_until_all_locked(&exec) {
574 		ret = xe_pf_begin(&exec, vma, true, tile->id);
575 		drm_exec_retry_on_contention(&exec);
576 		if (ret)
577 			break;
578 	}
579 
580 	drm_exec_fini(&exec);
581 unlock_vm:
582 	up_read(&vm->lock);
583 	xe_vm_put(vm);
584 
585 	return ret;
586 }
587 
588 #define make_u64(hi__, low__)  ((u64)(hi__) << 32 | (u64)(low__))
589 
590 #define ACC_MSG_LEN_DW        4
591 
592 static bool get_acc(struct acc_queue *acc_queue, struct acc *acc)
593 {
594 	const struct xe_guc_acc_desc *desc;
595 	bool ret = false;
596 
597 	spin_lock(&acc_queue->lock);
598 	if (acc_queue->tail != acc_queue->head) {
599 		desc = (const struct xe_guc_acc_desc *)
600 			(acc_queue->data + acc_queue->tail);
601 
602 		acc->granularity = FIELD_GET(ACC_GRANULARITY, desc->dw2);
603 		acc->sub_granularity = FIELD_GET(ACC_SUBG_HI, desc->dw1) << 31 |
604 			FIELD_GET(ACC_SUBG_LO, desc->dw0);
605 		acc->engine_class = FIELD_GET(ACC_ENG_CLASS, desc->dw1);
606 		acc->engine_instance = FIELD_GET(ACC_ENG_INSTANCE, desc->dw1);
607 		acc->asid =  FIELD_GET(ACC_ASID, desc->dw1);
608 		acc->vfid =  FIELD_GET(ACC_VFID, desc->dw2);
609 		acc->access_type = FIELD_GET(ACC_TYPE, desc->dw0);
610 		acc->va_range_base = make_u64(desc->dw3 & ACC_VIRTUAL_ADDR_RANGE_HI,
611 					      desc->dw2 & ACC_VIRTUAL_ADDR_RANGE_LO);
612 
613 		acc_queue->tail = (acc_queue->tail + ACC_MSG_LEN_DW) %
614 				  ACC_QUEUE_NUM_DW;
615 		ret = true;
616 	}
617 	spin_unlock(&acc_queue->lock);
618 
619 	return ret;
620 }
621 
622 static void acc_queue_work_func(struct work_struct *w)
623 {
624 	struct acc_queue *acc_queue = container_of(w, struct acc_queue, worker);
625 	struct xe_gt *gt = acc_queue->gt;
626 	struct xe_device *xe = gt_to_xe(gt);
627 	struct acc acc = {};
628 	unsigned long threshold;
629 	int ret;
630 
631 	threshold = jiffies + msecs_to_jiffies(USM_QUEUE_MAX_RUNTIME_MS);
632 
633 	while (get_acc(acc_queue, &acc)) {
634 		ret = handle_acc(gt, &acc);
635 		if (unlikely(ret)) {
636 			print_acc(xe, &acc);
637 			drm_warn(&xe->drm, "ACC: Unsuccessful %d\n", ret);
638 		}
639 
640 		if (time_after(jiffies, threshold) &&
641 		    acc_queue->tail != acc_queue->head) {
642 			queue_work(gt->usm.acc_wq, w);
643 			break;
644 		}
645 	}
646 }
647 
648 static bool acc_queue_full(struct acc_queue *acc_queue)
649 {
650 	lockdep_assert_held(&acc_queue->lock);
651 
652 	return CIRC_SPACE(acc_queue->head, acc_queue->tail, ACC_QUEUE_NUM_DW) <=
653 		ACC_MSG_LEN_DW;
654 }
655 
656 int xe_guc_access_counter_notify_handler(struct xe_guc *guc, u32 *msg, u32 len)
657 {
658 	struct xe_gt *gt = guc_to_gt(guc);
659 	struct acc_queue *acc_queue;
660 	u32 asid;
661 	bool full;
662 
663 	/*
664 	 * The below logic doesn't work unless ACC_QUEUE_NUM_DW % ACC_MSG_LEN_DW == 0
665 	 */
666 	BUILD_BUG_ON(ACC_QUEUE_NUM_DW % ACC_MSG_LEN_DW);
667 
668 	if (unlikely(len != ACC_MSG_LEN_DW))
669 		return -EPROTO;
670 
671 	asid = FIELD_GET(ACC_ASID, msg[1]);
672 	acc_queue = &gt->usm.acc_queue[asid % NUM_ACC_QUEUE];
673 
674 	spin_lock(&acc_queue->lock);
675 	full = acc_queue_full(acc_queue);
676 	if (!full) {
677 		memcpy(acc_queue->data + acc_queue->head, msg,
678 		       len * sizeof(u32));
679 		acc_queue->head = (acc_queue->head + len) % ACC_QUEUE_NUM_DW;
680 		queue_work(gt->usm.acc_wq, &acc_queue->worker);
681 	} else {
682 		drm_warn(&gt_to_xe(gt)->drm, "ACC Queue full, dropping ACC");
683 	}
684 	spin_unlock(&acc_queue->lock);
685 
686 	return full ? -ENOSPC : 0;
687 }
688