266 static bool get_pagefault(struct pf_queue *pf_queue, struct pagefault *pf)  in get_pagefault()  argument
271 	spin_lock_irq(&pf_queue->lock);  in get_pagefault()
272 	if (pf_queue->tail != pf_queue->head) {  in get_pagefault()
274 			(pf_queue->data + pf_queue->tail);  in get_pagefault()
292 		pf_queue->tail = (pf_queue->tail + PF_MSG_LEN_DW) %  in get_pagefault()
293 			pf_queue->num_dw;  in get_pagefault()
296 	spin_unlock_irq(&pf_queue->lock);  in get_pagefault()
301 static bool pf_queue_full(struct pf_queue *pf_queue)  in pf_queue_full()  argument
303 	lockdep_assert_held(&pf_queue->lock);  in pf_queue_full()
305 	return CIRC_SPACE(pf_queue->head, pf_queue->tail,  in pf_queue_full()
306 			  pf_queue->num_dw) <=  in pf_queue_full()
313 	struct pf_queue *pf_queue;  in xe_guc_pagefault_handler()  local
322 	pf_queue = gt->usm.pf_queue + (asid % NUM_PF_QUEUE);  in xe_guc_pagefault_handler()
327 	xe_gt_assert(gt, !(pf_queue->num_dw % PF_MSG_LEN_DW));  in xe_guc_pagefault_handler()
329 	spin_lock_irqsave(&pf_queue->lock, flags);  in xe_guc_pagefault_handler()
330 	full = pf_queue_full(pf_queue);  in xe_guc_pagefault_handler()
332 		memcpy(pf_queue->data + pf_queue->head, msg, len * sizeof(u32));  in xe_guc_pagefault_handler()
333 		pf_queue->head = (pf_queue->head + len) %  in xe_guc_pagefault_handler()
334 			pf_queue->num_dw;  in xe_guc_pagefault_handler()
335 		queue_work(gt->usm.pf_wq, &pf_queue->worker);  in xe_guc_pagefault_handler()
339 	spin_unlock_irqrestore(&pf_queue->lock, flags);  in xe_guc_pagefault_handler()
348 	struct pf_queue *pf_queue = container_of(w, struct pf_queue, worker);  in pf_queue_work_func()  local
349 	struct xe_gt *gt = pf_queue->gt;  in pf_queue_work_func()
357 	while (get_pagefault(pf_queue, &pf)) {  in pf_queue_work_func()
379 		    pf_queue->tail != pf_queue->head) {  in pf_queue_work_func()
400 static int xe_alloc_pf_queue(struct xe_gt *gt, struct pf_queue *pf_queue)  in xe_alloc_pf_queue()  argument
420 	pf_queue->num_dw =  in xe_alloc_pf_queue()
422 	pf_queue->num_dw = roundup_pow_of_two(pf_queue->num_dw);  in xe_alloc_pf_queue()
425 	pf_queue->gt = gt;  in xe_alloc_pf_queue()
426 	pf_queue->data = devm_kcalloc(xe->drm.dev, pf_queue->num_dw,  in xe_alloc_pf_queue()
428 	if (!pf_queue->data)  in xe_alloc_pf_queue()
431 	spin_lock_init(&pf_queue->lock);  in xe_alloc_pf_queue()
432 	INIT_WORK(&pf_queue->worker, pf_queue_work_func);  in xe_alloc_pf_queue()
446 		ret = xe_alloc_pf_queue(gt, &gt->usm.pf_queue[i]);  in xe_gt_pagefault_init()
481 		spin_lock_irq(&gt->usm.pf_queue[i].lock);  in xe_gt_pagefault_reset()
482 		gt->usm.pf_queue[i].head = 0;  in xe_gt_pagefault_reset()
483 		gt->usm.pf_queue[i].tail = 0;  in xe_gt_pagefault_reset()
484 		spin_unlock_irq(&gt->usm.pf_queue[i].lock);  in xe_gt_pagefault_reset()