87 	if (!hwpt->fault)  in iommufd_fault_domain_attach_dev()
104 	struct iommufd_fault *fault = hwpt->fault;  in iommufd_auto_response_faults()  local
109 	if (!fault)  in iommufd_auto_response_faults()
113 	mutex_lock(&fault->mutex);  in iommufd_auto_response_faults()
114 	spin_lock(&fault->lock);  in iommufd_auto_response_faults()
115 	list_for_each_entry_safe(group, next, &fault->deliver, node) {  in iommufd_auto_response_faults()
120 	spin_unlock(&fault->lock);  in iommufd_auto_response_faults()
128 	xa_for_each(&fault->response, index, group) {  in iommufd_auto_response_faults()
131 		xa_erase(&fault->response, index);  in iommufd_auto_response_faults()
135 	mutex_unlock(&fault->mutex);  in iommufd_auto_response_faults()
169 	if (old->fault)  in __fault_domain_replace_dev()
172 	if (hwpt->fault) {  in __fault_domain_replace_dev()
197 	bool iopf_off = !hwpt->fault && old->fault;  in iommufd_fault_domain_replace_dev()
198 	bool iopf_on = hwpt->fault && !old->fault;  in iommufd_fault_domain_replace_dev()
222 	struct iommufd_fault *fault = container_of(obj, struct iommufd_fault, obj);  in iommufd_fault_destroy()  local
232 	list_for_each_entry_safe(group, next, &fault->deliver, node) {  in iommufd_fault_destroy()
237 	xa_for_each(&fault->response, index, group) {  in iommufd_fault_destroy()
238 		xa_erase(&fault->response, index);  in iommufd_fault_destroy()
242 	xa_destroy(&fault->response);  in iommufd_fault_destroy()
243 	mutex_destroy(&fault->mutex);  in iommufd_fault_destroy()
246 static void iommufd_compose_fault_message(struct iommu_fault *fault,  in iommufd_compose_fault_message()  argument
251 	hwpt_fault->flags = fault->prm.flags;  in iommufd_compose_fault_message()
253 	hwpt_fault->pasid = fault->prm.pasid;  in iommufd_compose_fault_message()
254 	hwpt_fault->grpid = fault->prm.grpid;  in iommufd_compose_fault_message()
255 	hwpt_fault->perm = fault->prm.perm;  in iommufd_compose_fault_message()
256 	hwpt_fault->addr = fault->prm.addr;  in iommufd_compose_fault_message()
265 	struct iommufd_fault *fault = filep->private_data;  in iommufd_fault_fops_read()  local
276 	mutex_lock(&fault->mutex);  in iommufd_fault_fops_read()
277 	while ((group = iommufd_fault_deliver_fetch(fault))) {  in iommufd_fault_fops_read()
280 			iommufd_fault_deliver_restore(fault, group);  in iommufd_fault_fops_read()
284 		rc = xa_alloc(&fault->response, &group->cookie, group,  in iommufd_fault_fops_read()
287 			iommufd_fault_deliver_restore(fault, group);  in iommufd_fault_fops_read()
293 			iommufd_compose_fault_message(&iopf->fault,  in iommufd_fault_fops_read()
297 				xa_erase(&fault->response, group->cookie);  in iommufd_fault_fops_read()
298 				iommufd_fault_deliver_restore(fault, group);  in iommufd_fault_fops_read()
305 	mutex_unlock(&fault->mutex);  in iommufd_fault_fops_read()
314 	struct iommufd_fault *fault = filep->private_data;  in iommufd_fault_fops_write()  local
323 	mutex_lock(&fault->mutex);  in iommufd_fault_fops_write()
339 		group = xa_erase(&fault->response, response.cookie);  in iommufd_fault_fops_write()
349 	mutex_unlock(&fault->mutex);  in iommufd_fault_fops_write()
357 	struct iommufd_fault *fault = filep->private_data;  in iommufd_fault_fops_poll()  local
360 	poll_wait(filep, &fault->wait_queue, wait);  in iommufd_fault_fops_poll()
361 	spin_lock(&fault->lock);  in iommufd_fault_fops_poll()
362 	if (!list_empty(&fault->deliver))  in iommufd_fault_fops_poll()
364 	spin_unlock(&fault->lock);  in iommufd_fault_fops_poll()
371 	struct iommufd_fault *fault = filep->private_data;  in iommufd_fault_fops_release()  local
373 	refcount_dec(&fault->obj.users);  in iommufd_fault_fops_release()
374 	iommufd_ctx_put(fault->ictx);  in iommufd_fault_fops_release()
390 	struct iommufd_fault *fault;  in iommufd_fault_alloc()  local
398 	fault = iommufd_object_alloc(ucmd->ictx, fault, IOMMUFD_OBJ_FAULT);  in iommufd_fault_alloc()
399 	if (IS_ERR(fault))  in iommufd_fault_alloc()
400 		return PTR_ERR(fault);  in iommufd_fault_alloc()
402 	fault->ictx = ucmd->ictx;  in iommufd_fault_alloc()
403 	INIT_LIST_HEAD(&fault->deliver);  in iommufd_fault_alloc()
404 	xa_init_flags(&fault->response, XA_FLAGS_ALLOC1);  in iommufd_fault_alloc()
405 	mutex_init(&fault->mutex);  in iommufd_fault_alloc()
406 	spin_lock_init(&fault->lock);  in iommufd_fault_alloc()
407 	init_waitqueue_head(&fault->wait_queue);  in iommufd_fault_alloc()
410 				   fault, O_RDWR);  in iommufd_fault_alloc()
416 	refcount_inc(&fault->obj.users);  in iommufd_fault_alloc()
417 	iommufd_ctx_get(fault->ictx);  in iommufd_fault_alloc()
418 	fault->filep = filep;  in iommufd_fault_alloc()
426 	cmd->out_fault_id = fault->obj.id;  in iommufd_fault_alloc()
432 	iommufd_object_finalize(ucmd->ictx, &fault->obj);  in iommufd_fault_alloc()
434 	fd_install(fdno, fault->filep);  in iommufd_fault_alloc()
442 	iommufd_object_abort_and_destroy(ucmd->ictx, &fault->obj);  in iommufd_fault_alloc()
450 	struct iommufd_fault *fault;  in iommufd_fault_iopf_handler()  local
453 	fault = hwpt->fault;  in iommufd_fault_iopf_handler()
455 	spin_lock(&fault->lock);  in iommufd_fault_iopf_handler()
456 	list_add_tail(&group->node, &fault->deliver);  in iommufd_fault_iopf_handler()
457 	spin_unlock(&fault->lock);  in iommufd_fault_iopf_handler()
459 	wake_up_interruptible(&fault->wait_queue);  in iommufd_fault_iopf_handler()