207 static bool xe_pagefault_queue_pop(struct xe_pagefault_queue *pf_queue,  in xe_pagefault_queue_pop()  argument
212 	spin_lock_irq(&pf_queue->lock);  in xe_pagefault_queue_pop()
213 	if (pf_queue->tail != pf_queue->head) {  in xe_pagefault_queue_pop()
214 		memcpy(pf, pf_queue->data + pf_queue->tail, sizeof(*pf));  in xe_pagefault_queue_pop()
215 		pf_queue->tail = (pf_queue->tail + xe_pagefault_entry_size()) %  in xe_pagefault_queue_pop()
216 			pf_queue->size;  in xe_pagefault_queue_pop()
219 	spin_unlock_irq(&pf_queue->lock);  in xe_pagefault_queue_pop()
246 	struct xe_pagefault_queue *pf_queue =  in xe_pagefault_queue_work()  local
247 		container_of(w, typeof(*pf_queue), worker);  in xe_pagefault_queue_work()
254 	while (xe_pagefault_queue_pop(pf_queue, &pf)) {  in xe_pagefault_queue_work()
278 				   struct xe_pagefault_queue *pf_queue)  in xe_pagefault_queue_init()  argument
307 	pf_queue->size = (total_num_eus + XE_NUM_HW_ENGINES) *  in xe_pagefault_queue_init()
309 	pf_queue->size = roundup_pow_of_two(pf_queue->size);  in xe_pagefault_queue_init()
313 		xe_pagefault_entry_size(), total_num_eus, pf_queue->size);  in xe_pagefault_queue_init()
315 	spin_lock_init(&pf_queue->lock);  in xe_pagefault_queue_init()
316 	INIT_WORK(&pf_queue->worker, xe_pagefault_queue_work);  in xe_pagefault_queue_init()
318 	pf_queue->data = drmm_kzalloc(&xe->drm, pf_queue->size, GFP_KERNEL);  in xe_pagefault_queue_init()
319 	if (!pf_queue->data)  in xe_pagefault_queue_init()
354 		err = xe_pagefault_queue_init(xe, xe->usm.pf_queue + i);  in xe_pagefault_init()
367 				     struct xe_pagefault_queue *pf_queue)  in xe_pagefault_queue_reset()  argument
372 	if (!pf_queue->data)  in xe_pagefault_queue_reset()
377 	spin_lock_irq(&pf_queue->lock);  in xe_pagefault_queue_reset()
378 	for (i = pf_queue->tail; i != pf_queue->head;  in xe_pagefault_queue_reset()
379 	     i = (i + xe_pagefault_entry_size()) % pf_queue->size) {  in xe_pagefault_queue_reset()
380 		struct xe_pagefault *pf = pf_queue->data + i;  in xe_pagefault_queue_reset()
385 	spin_unlock_irq(&pf_queue->lock);  in xe_pagefault_queue_reset()
401 		xe_pagefault_queue_reset(xe, gt, xe->usm.pf_queue + i);  in xe_pagefault_reset()
404 static bool xe_pagefault_queue_full(struct xe_pagefault_queue *pf_queue)  in xe_pagefault_queue_full()  argument
406 	lockdep_assert_held(&pf_queue->lock);  in xe_pagefault_queue_full()
408 	return CIRC_SPACE(pf_queue->head, pf_queue->tail, pf_queue->size) <=  in xe_pagefault_queue_full()
424 	struct xe_pagefault_queue *pf_queue = xe->usm.pf_queue +  in xe_pagefault_handler()  local
429 	spin_lock_irqsave(&pf_queue->lock, flags);  in xe_pagefault_handler()
430 	full = xe_pagefault_queue_full(pf_queue);  in xe_pagefault_handler()
432 		memcpy(pf_queue->data + pf_queue->head, pf, sizeof(*pf));  in xe_pagefault_handler()
433 		pf_queue->head = (pf_queue->head + xe_pagefault_entry_size()) %  in xe_pagefault_handler()
434 			pf_queue->size;  in xe_pagefault_handler()
435 		queue_work(xe->usm.pf_wq, &pf_queue->worker);  in xe_pagefault_handler()
441 	spin_unlock_irqrestore(&pf_queue->lock, flags);  in xe_pagefault_handler()