1 // SPDX-License-Identifier: MIT
32  * a multi-threaded worker to process them. Multiple producers are supported,
62 	return need_vram_move ? xe_bo_migrate(bo, vram->placement, NULL, exec) :  in xe_pagefault_begin()
71 	struct xe_validation_ctx ctx;  in xe_pagefault_handle_vma()  local
76 	lockdep_assert_held_write(&vm->lock);  in xe_pagefault_handle_vma()
78 	needs_vram = xe_vma_need_vram_for_atomic(vm->xe, vma, atomic);  in xe_pagefault_handle_vma()
80 		return needs_vram < 0 ? needs_vram : -EACCES;  in xe_pagefault_handle_vma()
89 	if (xe_vm_has_valid_gpu_mapping(tile, vma->tile_present,  in xe_pagefault_handle_vma()
90 					vma->tile_invalidated) && !atomic)  in xe_pagefault_handle_vma()
103 	/* Lock VM and BOs dma-resv */  in xe_pagefault_handle_vma()
104 	xe_validation_ctx_init(&ctx, &vm->xe->val, &exec, (struct xe_val_flags) {});  in xe_pagefault_handle_vma()
106 		err = xe_pagefault_begin(&exec, vma, tile->mem.vram,  in xe_pagefault_handle_vma()
109 		xe_validation_retry_on_oom(&ctx, &err);  in xe_pagefault_handle_vma()
116 		fence = xe_vma_rebind(vm, vma, BIT(tile->id));  in xe_pagefault_handle_vma()
120 			xe_validation_retry_on_oom(&ctx, &err);  in xe_pagefault_handle_vma()
129 	xe_validation_ctx_fini(&ctx);  in xe_pagefault_handle_vma()
130 	if (err == -EAGAIN)  in xe_pagefault_handle_vma()
142 static struct xe_vm *xe_pagefault_asid_to_vm(struct xe_device *xe, u32 asid)  in xe_pagefault_asid_to_vm()  argument
146 	down_read(&xe->usm.lock);  in xe_pagefault_asid_to_vm()
147 	vm = xa_load(&xe->usm.asid_to_vm, asid);  in xe_pagefault_asid_to_vm()
151 		vm = ERR_PTR(-EINVAL);  in xe_pagefault_asid_to_vm()
152 	up_read(&xe->usm.lock);  in xe_pagefault_asid_to_vm()
159 	struct xe_gt *gt = pf->gt;  in xe_pagefault_service()
167 	if (pf->consumer.fault_level == XE_PAGEFAULT_LEVEL_NACK)  in xe_pagefault_service()
168 		return -EFAULT;  in xe_pagefault_service()
170 	vm = xe_pagefault_asid_to_vm(xe, pf->consumer.asid);  in xe_pagefault_service()
177 	down_write(&vm->lock);  in xe_pagefault_service()
180 		err = -ENOENT;  in xe_pagefault_service()
184 	vma = xe_vm_find_vma_by_addr(vm, pf->consumer.page_addr);  in xe_pagefault_service()
186 		err = -EINVAL;  in xe_pagefault_service()
190 	atomic = xe_pagefault_access_is_atomic(pf->consumer.access_type);  in xe_pagefault_service()
194 					      pf->consumer.page_addr, atomic);  in xe_pagefault_service()
200 		vm->usm.last_fault_vma = vma;  in xe_pagefault_service()
201 	up_write(&vm->lock);  in xe_pagefault_service()
212 	spin_lock_irq(&pf_queue->lock);  in xe_pagefault_queue_pop()
213 	if (pf_queue->tail != pf_queue->head) {  in xe_pagefault_queue_pop()
214 		memcpy(pf, pf_queue->data + pf_queue->tail, sizeof(*pf));  in xe_pagefault_queue_pop()
215 		pf_queue->tail = (pf_queue->tail + xe_pagefault_entry_size()) %  in xe_pagefault_queue_pop()
216 			pf_queue->size;  in xe_pagefault_queue_pop()
219 	spin_unlock_irq(&pf_queue->lock);  in xe_pagefault_queue_pop()
226 	xe_gt_dbg(pf->gt, "\n\tASID: %d\n"  in xe_pagefault_print()
233 		  pf->consumer.asid,  in xe_pagefault_print()
234 		  upper_32_bits(pf->consumer.page_addr),  in xe_pagefault_print()
235 		  lower_32_bits(pf->consumer.page_addr),  in xe_pagefault_print()
236 		  pf->consumer.fault_type,  in xe_pagefault_print()
237 		  pf->consumer.access_type,  in xe_pagefault_print()
238 		  pf->consumer.fault_level,  in xe_pagefault_print()
239 		  pf->consumer.engine_class,  in xe_pagefault_print()
240 		  xe_hw_engine_class_to_str(pf->consumer.engine_class),  in xe_pagefault_print()
241 		  pf->consumer.engine_instance);  in xe_pagefault_print()
267 		pf.producer.ops->ack_fault(&pf, err);  in xe_pagefault_queue_work()
270 			queue_work(gt_to_xe(pf.gt)->usm.pf_wq, w);  in xe_pagefault_queue_work()
288 		bitmap_or(all_dss, gt->fuse_topo.g_dss_mask,  in xe_pagefault_queue_init()
289 			  gt->fuse_topo.c_dss_mask, XE_MAX_DSS_FUSE_BITS);  in xe_pagefault_queue_init()
292 		num_eus = bitmap_weight(gt->fuse_topo.eu_mask_per_dss,  in xe_pagefault_queue_init()
307 	pf_queue->size = (total_num_eus + XE_NUM_HW_ENGINES) *  in xe_pagefault_queue_init()
309 	pf_queue->size = roundup_pow_of_two(pf_queue->size);  in xe_pagefault_queue_init()
312 	drm_dbg(&xe->drm, "xe_pagefault_entry_size=%d, total_num_eus=%d, pf_queue->size=%u",  in xe_pagefault_queue_init()
313 		xe_pagefault_entry_size(), total_num_eus, pf_queue->size);  in xe_pagefault_queue_init()
315 	spin_lock_init(&pf_queue->lock);  in xe_pagefault_queue_init()
316 	INIT_WORK(&pf_queue->worker, xe_pagefault_queue_work);  in xe_pagefault_queue_init()
318 	pf_queue->data = drmm_kzalloc(&xe->drm, pf_queue->size, GFP_KERNEL);  in xe_pagefault_queue_init()
319 	if (!pf_queue->data)  in xe_pagefault_queue_init()
320 		return -ENOMEM;  in xe_pagefault_queue_init()
329 	destroy_workqueue(xe->usm.pf_wq);  in xe_pagefault_fini()
333  * xe_pagefault_init() - Page fault init
344 	if (!xe->info.has_usm)  in xe_pagefault_init()
347 	xe->usm.pf_wq = alloc_workqueue("xe_page_fault_work_queue",  in xe_pagefault_init()
350 	if (!xe->usm.pf_wq)  in xe_pagefault_init()
351 		return -ENOMEM;  in xe_pagefault_init()
354 		err = xe_pagefault_queue_init(xe, xe->usm.pf_queue + i);  in xe_pagefault_init()
359 	return devm_add_action_or_reset(xe->drm.dev, xe_pagefault_fini, xe);  in xe_pagefault_init()
362 	destroy_workqueue(xe->usm.pf_wq);  in xe_pagefault_init()
372 	if (!pf_queue->data)  in xe_pagefault_queue_reset()
377 	spin_lock_irq(&pf_queue->lock);  in xe_pagefault_queue_reset()
378 	for (i = pf_queue->tail; i != pf_queue->head;  in xe_pagefault_queue_reset()
379 	     i = (i + xe_pagefault_entry_size()) % pf_queue->size) {  in xe_pagefault_queue_reset()
380 		struct xe_pagefault *pf = pf_queue->data + i;  in xe_pagefault_queue_reset()
382 		if (pf->gt == gt)  in xe_pagefault_queue_reset()
383 			pf->gt = NULL;  in xe_pagefault_queue_reset()
385 	spin_unlock_irq(&pf_queue->lock);  in xe_pagefault_queue_reset()
389  * xe_pagefault_reset() - Page fault reset for a GT
401 		xe_pagefault_queue_reset(xe, gt, xe->usm.pf_queue + i);  in xe_pagefault_reset()
406 	lockdep_assert_held(&pf_queue->lock);  in xe_pagefault_queue_full()
408 	return CIRC_SPACE(pf_queue->head, pf_queue->tail, pf_queue->size) <=  in xe_pagefault_queue_full()
413  * xe_pagefault_handler() - Page fault handler
424 	struct xe_pagefault_queue *pf_queue = xe->usm.pf_queue +  in xe_pagefault_handler()
425 		(pf->consumer.asid % XE_PAGEFAULT_QUEUE_COUNT);  in xe_pagefault_handler()
429 	spin_lock_irqsave(&pf_queue->lock, flags);  in xe_pagefault_handler()
432 		memcpy(pf_queue->data + pf_queue->head, pf, sizeof(*pf));  in xe_pagefault_handler()
433 		pf_queue->head = (pf_queue->head + xe_pagefault_entry_size()) %  in xe_pagefault_handler()
434 			pf_queue->size;  in xe_pagefault_handler()
435 		queue_work(xe->usm.pf_wq, &pf_queue->worker);  in xe_pagefault_handler()
437 		drm_warn(&xe->drm,  in xe_pagefault_handler()
439 			 pf->consumer.asid % XE_PAGEFAULT_QUEUE_COUNT);  in xe_pagefault_handler()
441 	spin_unlock_irqrestore(&pf_queue->lock, flags);  in xe_pagefault_handler()
443 	return full ? -ENOSPC : 0;  in xe_pagefault_handler()