22 	struct iommufd_fault *fault = hwpt->fault;  in iommufd_auto_response_faults()  local
27 	if (!fault || !handle)  in iommufd_auto_response_faults()
31 	mutex_lock(&fault->mutex);  in iommufd_auto_response_faults()
32 	spin_lock(&fault->common.lock);  in iommufd_auto_response_faults()
33 	list_for_each_entry_safe(group, next, &fault->common.deliver, node) {  in iommufd_auto_response_faults()
38 	spin_unlock(&fault->common.lock);  in iommufd_auto_response_faults()
46 	xa_for_each(&fault->response, index, group) {  in iommufd_auto_response_faults()
49 		xa_erase(&fault->response, index);  in iommufd_auto_response_faults()
53 	mutex_unlock(&fault->mutex);  in iommufd_auto_response_faults()
60 	struct iommufd_fault *fault = eventq_to_fault(eventq);  in iommufd_fault_destroy()  local
70 	list_for_each_entry_safe(group, next, &fault->common.deliver, node) {  in iommufd_fault_destroy()
75 	xa_for_each(&fault->response, index, group) {  in iommufd_fault_destroy()
76 		xa_erase(&fault->response, index);  in iommufd_fault_destroy()
80 	xa_destroy(&fault->response);  in iommufd_fault_destroy()
81 	mutex_destroy(&fault->mutex);  in iommufd_fault_destroy()
84 static void iommufd_compose_fault_message(struct iommu_fault *fault,  in iommufd_compose_fault_message()  argument
89 	hwpt_fault->flags = fault->prm.flags;  in iommufd_compose_fault_message()
91 	hwpt_fault->pasid = fault->prm.pasid;  in iommufd_compose_fault_message()
92 	hwpt_fault->grpid = fault->prm.grpid;  in iommufd_compose_fault_message()
93 	hwpt_fault->perm = fault->prm.perm;  in iommufd_compose_fault_message()
94 	hwpt_fault->addr = fault->prm.addr;  in iommufd_compose_fault_message()
99 /* Fetch the first node out of the fault->deliver list */
101 iommufd_fault_deliver_fetch(struct iommufd_fault *fault)  in iommufd_fault_deliver_fetch()  argument
103 	struct list_head *list = &fault->common.deliver;  in iommufd_fault_deliver_fetch()
106 	spin_lock(&fault->common.lock);  in iommufd_fault_deliver_fetch()
111 	spin_unlock(&fault->common.lock);  in iommufd_fault_deliver_fetch()
115 /* Restore a node back to the head of the fault->deliver list */
116 static void iommufd_fault_deliver_restore(struct iommufd_fault *fault,  in iommufd_fault_deliver_restore()  argument
119 	spin_lock(&fault->common.lock);  in iommufd_fault_deliver_restore()
120 	list_add(&group->node, &fault->common.deliver);  in iommufd_fault_deliver_restore()
121 	spin_unlock(&fault->common.lock);  in iommufd_fault_deliver_restore()
129 	struct iommufd_fault *fault = eventq_to_fault(eventq);  in iommufd_fault_fops_read()  local
140 	mutex_lock(&fault->mutex);  in iommufd_fault_fops_read()
141 	while ((group = iommufd_fault_deliver_fetch(fault))) {  in iommufd_fault_fops_read()
144 			iommufd_fault_deliver_restore(fault, group);  in iommufd_fault_fops_read()
148 		rc = xa_alloc(&fault->response, &group->cookie, group,  in iommufd_fault_fops_read()
151 			iommufd_fault_deliver_restore(fault, group);  in iommufd_fault_fops_read()
157 			iommufd_compose_fault_message(&iopf->fault,  in iommufd_fault_fops_read()
161 				xa_erase(&fault->response, group->cookie);  in iommufd_fault_fops_read()
162 				iommufd_fault_deliver_restore(fault, group);  in iommufd_fault_fops_read()
169 	mutex_unlock(&fault->mutex);  in iommufd_fault_fops_read()
179 	struct iommufd_fault *fault = eventq_to_fault(eventq);  in iommufd_fault_fops_write()  local
188 	mutex_lock(&fault->mutex);  in iommufd_fault_fops_write()
204 		group = xa_erase(&fault->response, response.cookie);  in iommufd_fault_fops_write()
214 	mutex_unlock(&fault->mutex);  in iommufd_fault_fops_write()
423 	struct iommufd_fault *fault;  in iommufd_fault_alloc()  local
430 	fault = __iommufd_object_alloc(ucmd->ictx, fault, IOMMUFD_OBJ_FAULT,  in iommufd_fault_alloc()
432 	if (IS_ERR(fault))  in iommufd_fault_alloc()
433 		return PTR_ERR(fault);  in iommufd_fault_alloc()
435 	xa_init_flags(&fault->response, XA_FLAGS_ALLOC1);  in iommufd_fault_alloc()
436 	mutex_init(&fault->mutex);  in iommufd_fault_alloc()
438 	fdno = iommufd_eventq_init(&fault->common, "[iommufd-pgfault]",  in iommufd_fault_alloc()
445 	cmd->out_fault_id = fault->common.obj.id;  in iommufd_fault_alloc()
451 	iommufd_object_finalize(ucmd->ictx, &fault->common.obj);  in iommufd_fault_alloc()
453 	fd_install(fdno, fault->common.filep);  in iommufd_fault_alloc()
458 	fput(fault->common.filep);  in iommufd_fault_alloc()
460 	iommufd_object_abort_and_destroy(ucmd->ictx, &fault->common.obj);  in iommufd_fault_alloc()
468 	struct iommufd_fault *fault;  in iommufd_fault_iopf_handler()  local
471 	fault = hwpt->fault;  in iommufd_fault_iopf_handler()
473 	spin_lock(&fault->common.lock);  in iommufd_fault_iopf_handler()
474 	list_add_tail(&group->node, &fault->common.deliver);  in iommufd_fault_iopf_handler()
475 	spin_unlock(&fault->common.lock);  in iommufd_fault_iopf_handler()
477 	wake_up_interruptible(&fault->common.wait_queue);  in iommufd_fault_iopf_handler()