xref: /linux/drivers/iommu/iommufd/eventq.c (revision 440ca8a4b69f40088065b9e05ef11373292a254e) 
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (C) 2024 Intel Corporation
3  */
4 #define pr_fmt(fmt) "iommufd: " fmt
5 
6 #include <linux/anon_inodes.h>
7 #include <linux/file.h>
8 #include <linux/fs.h>
9 #include <linux/iommufd.h>
10 #include <linux/module.h>
11 #include <linux/mutex.h>
12 #include <linux/poll.h>
13 #include <uapi/linux/iommufd.h>
14 
15 #include "../iommu-priv.h"
16 #include "iommufd_private.h"
17 
18 /* IOMMUFD_OBJ_FAULT Functions */
19 void iommufd_auto_response_faults(struct iommufd_hw_pagetable *hwpt,
20 				  struct iommufd_attach_handle *handle)
21 {
22 	struct iommufd_fault *fault = hwpt->fault;
23 	struct iopf_group *group, *next;
24 	struct list_head free_list;
25 	unsigned long index;
26 
27 	if (!fault || !handle)
28 		return;
29 	INIT_LIST_HEAD(&free_list);
30 
31 	mutex_lock(&fault->mutex);
32 	spin_lock(&fault->common.lock);
33 	list_for_each_entry_safe(group, next, &fault->common.deliver, node) {
34 		if (group->attach_handle != &handle->handle)
35 			continue;
36 		list_move(&group->node, &free_list);
37 	}
38 	spin_unlock(&fault->common.lock);
39 
40 	list_for_each_entry_safe(group, next, &free_list, node) {
41 		list_del(&group->node);
42 		iopf_group_response(group, IOMMU_PAGE_RESP_INVALID);
43 		iopf_free_group(group);
44 	}
45 
46 	xa_for_each(&fault->response, index, group) {
47 		if (group->attach_handle != &handle->handle)
48 			continue;
49 		xa_erase(&fault->response, index);
50 		iopf_group_response(group, IOMMU_PAGE_RESP_INVALID);
51 		iopf_free_group(group);
52 	}
53 	mutex_unlock(&fault->mutex);
54 }
55 
56 void iommufd_fault_destroy(struct iommufd_object *obj)
57 {
58 	struct iommufd_eventq *eventq =
59 		container_of(obj, struct iommufd_eventq, obj);
60 	struct iommufd_fault *fault = eventq_to_fault(eventq);
61 	struct iopf_group *group, *next;
62 	unsigned long index;
63 
64 	/*
65 	 * The iommufd object's reference count is zero at this point.
66 	 * We can be confident that no other threads are currently
67 	 * accessing this pointer. Therefore, acquiring the mutex here
68 	 * is unnecessary.
69 	 */
70 	list_for_each_entry_safe(group, next, &fault->common.deliver, node) {
71 		list_del(&group->node);
72 		iopf_group_response(group, IOMMU_PAGE_RESP_INVALID);
73 		iopf_free_group(group);
74 	}
75 	xa_for_each(&fault->response, index, group) {
76 		xa_erase(&fault->response, index);
77 		iopf_group_response(group, IOMMU_PAGE_RESP_INVALID);
78 		iopf_free_group(group);
79 	}
80 	xa_destroy(&fault->response);
81 	mutex_destroy(&fault->mutex);
82 }
83 
84 static void iommufd_compose_fault_message(struct iommu_fault *fault,
85 					  struct iommu_hwpt_pgfault *hwpt_fault,
86 					  struct iommufd_device *idev,
87 					  u32 cookie)
88 {
89 	hwpt_fault->flags = fault->prm.flags;
90 	hwpt_fault->dev_id = idev->obj.id;
91 	hwpt_fault->pasid = fault->prm.pasid;
92 	hwpt_fault->grpid = fault->prm.grpid;
93 	hwpt_fault->perm = fault->prm.perm;
94 	hwpt_fault->addr = fault->prm.addr;
95 	hwpt_fault->length = 0;
96 	hwpt_fault->cookie = cookie;
97 }
98 
99 /* Fetch the first node out of the fault->deliver list */
100 static struct iopf_group *
101 iommufd_fault_deliver_fetch(struct iommufd_fault *fault)
102 {
103 	struct list_head *list = &fault->common.deliver;
104 	struct iopf_group *group = NULL;
105 
106 	spin_lock(&fault->common.lock);
107 	if (!list_empty(list)) {
108 		group = list_first_entry(list, struct iopf_group, node);
109 		list_del(&group->node);
110 	}
111 	spin_unlock(&fault->common.lock);
112 	return group;
113 }
114 
115 /* Restore a node back to the head of the fault->deliver list */
116 static void iommufd_fault_deliver_restore(struct iommufd_fault *fault,
117 					  struct iopf_group *group)
118 {
119 	spin_lock(&fault->common.lock);
120 	list_add(&group->node, &fault->common.deliver);
121 	spin_unlock(&fault->common.lock);
122 }
123 
124 static ssize_t iommufd_fault_fops_read(struct file *filep, char __user *buf,
125 				       size_t count, loff_t *ppos)
126 {
127 	size_t fault_size = sizeof(struct iommu_hwpt_pgfault);
128 	struct iommufd_eventq *eventq = filep->private_data;
129 	struct iommufd_fault *fault = eventq_to_fault(eventq);
130 	struct iommu_hwpt_pgfault data = {};
131 	struct iommufd_device *idev;
132 	struct iopf_group *group;
133 	struct iopf_fault *iopf;
134 	size_t done = 0;
135 	int rc = 0;
136 
137 	if (*ppos || count % fault_size)
138 		return -ESPIPE;
139 
140 	mutex_lock(&fault->mutex);
141 	while ((group = iommufd_fault_deliver_fetch(fault))) {
142 		size_t group_done = done;
143 
144 		if (done >= count ||
145 		    group->fault_count * fault_size > count - done) {
146 			iommufd_fault_deliver_restore(fault, group);
147 			/* Read count doesn't fit the first fault group */
148 			if (done == 0)
149 				rc = -EINVAL;
150 			break;
151 		}
152 
153 		rc = xa_alloc(&fault->response, &group->cookie, group,
154 			      xa_limit_32b, GFP_KERNEL);
155 		if (rc) {
156 			iommufd_fault_deliver_restore(fault, group);
157 			break;
158 		}
159 
160 		idev = to_iommufd_handle(group->attach_handle)->idev;
161 		list_for_each_entry(iopf, &group->faults, list) {
162 			iommufd_compose_fault_message(&iopf->fault,
163 						      &data, idev,
164 						      group->cookie);
165 			if (copy_to_user(buf + group_done, &data, fault_size)) {
166 				xa_erase(&fault->response, group->cookie);
167 				iommufd_fault_deliver_restore(fault, group);
168 				rc = -EFAULT;
169 				break;
170 			}
171 			group_done += fault_size;
172 		}
173 		if (rc)
174 			break;
175 		done = group_done;
176 	}
177 	mutex_unlock(&fault->mutex);
178 
179 	return done == 0 ? rc : done;
180 }
181 
182 static ssize_t iommufd_fault_fops_write(struct file *filep, const char __user *buf,
183 					size_t count, loff_t *ppos)
184 {
185 	size_t response_size = sizeof(struct iommu_hwpt_page_response);
186 	struct iommufd_eventq *eventq = filep->private_data;
187 	struct iommufd_fault *fault = eventq_to_fault(eventq);
188 	struct iommu_hwpt_page_response response;
189 	struct iopf_group *group;
190 	size_t done = 0;
191 	int rc = 0;
192 
193 	if (*ppos || count % response_size)
194 		return -ESPIPE;
195 
196 	mutex_lock(&fault->mutex);
197 	while (count > done) {
198 		if (copy_from_user(&response, buf + done, response_size)) {
199 			rc = -EFAULT;
200 			break;
201 		}
202 
203 		static_assert((int)IOMMUFD_PAGE_RESP_SUCCESS ==
204 			      (int)IOMMU_PAGE_RESP_SUCCESS);
205 		static_assert((int)IOMMUFD_PAGE_RESP_INVALID ==
206 			      (int)IOMMU_PAGE_RESP_INVALID);
207 		if (response.code != IOMMUFD_PAGE_RESP_SUCCESS &&
208 		    response.code != IOMMUFD_PAGE_RESP_INVALID) {
209 			rc = -EINVAL;
210 			break;
211 		}
212 
213 		group = xa_erase(&fault->response, response.cookie);
214 		if (!group) {
215 			rc = -EINVAL;
216 			break;
217 		}
218 
219 		iopf_group_response(group, response.code);
220 		iopf_free_group(group);
221 		done += response_size;
222 	}
223 	mutex_unlock(&fault->mutex);
224 
225 	return done == 0 ? rc : done;
226 }
227 
228 /* IOMMUFD_OBJ_VEVENTQ Functions */
229 
230 void iommufd_veventq_abort(struct iommufd_object *obj)
231 {
232 	struct iommufd_eventq *eventq =
233 		container_of(obj, struct iommufd_eventq, obj);
234 	struct iommufd_veventq *veventq = eventq_to_veventq(eventq);
235 	struct iommufd_viommu *viommu = veventq->viommu;
236 	struct iommufd_vevent *cur, *next;
237 
238 	lockdep_assert_held_write(&viommu->veventqs_rwsem);
239 
240 	list_for_each_entry_safe(cur, next, &eventq->deliver, node) {
241 		list_del(&cur->node);
242 		if (cur != &veventq->lost_events_header)
243 			kfree(cur);
244 	}
245 
246 	refcount_dec(&viommu->obj.users);
247 	list_del(&veventq->node);
248 }
249 
250 void iommufd_veventq_destroy(struct iommufd_object *obj)
251 {
252 	struct iommufd_veventq *veventq = eventq_to_veventq(
253 		container_of(obj, struct iommufd_eventq, obj));
254 
255 	down_write(&veventq->viommu->veventqs_rwsem);
256 	iommufd_veventq_abort(obj);
257 	up_write(&veventq->viommu->veventqs_rwsem);
258 }
259 
260 static struct iommufd_vevent *
261 iommufd_veventq_deliver_fetch(struct iommufd_veventq *veventq)
262 {
263 	struct iommufd_eventq *eventq = &veventq->common;
264 	struct list_head *list = &eventq->deliver;
265 	struct iommufd_vevent *vevent = NULL;
266 
267 	spin_lock(&eventq->lock);
268 	if (!list_empty(list)) {
269 		struct iommufd_vevent *next;
270 
271 		next = list_first_entry(list, struct iommufd_vevent, node);
272 		/* Make a copy of the lost_events_header for copy_to_user */
273 		if (next == &veventq->lost_events_header) {
274 			vevent = kzalloc_obj(*vevent, GFP_ATOMIC);
275 			if (!vevent) {
276 				vevent = ERR_PTR(-ENOMEM);
277 				goto out_unlock;
278 			}
279 		}
280 		list_del(&next->node);
281 		if (vevent)
282 			memcpy(vevent, next, sizeof(*vevent));
283 		else
284 			vevent = next;
285 	}
286 out_unlock:
287 	spin_unlock(&eventq->lock);
288 	return vevent;
289 }
290 
291 static void iommufd_veventq_deliver_restore(struct iommufd_veventq *veventq,
292 					    struct iommufd_vevent *vevent)
293 {
294 	struct iommufd_eventq *eventq = &veventq->common;
295 	struct list_head *list = &eventq->deliver;
296 
297 	spin_lock(&eventq->lock);
298 	if (vevent_for_lost_events_header(vevent)) {
299 		/* Remove the copy of the lost_events_header */
300 		kfree(vevent);
301 		vevent = NULL;
302 		/* An empty list needs the lost_events_header back */
303 		if (list_empty(list))
304 			vevent = &veventq->lost_events_header;
305 	}
306 	if (vevent)
307 		list_add(&vevent->node, list);
308 	spin_unlock(&eventq->lock);
309 }
310 
311 static ssize_t iommufd_veventq_fops_read(struct file *filep, char __user *buf,
312 					 size_t count, loff_t *ppos)
313 {
314 	struct iommufd_eventq *eventq = filep->private_data;
315 	struct iommufd_veventq *veventq = eventq_to_veventq(eventq);
316 	struct iommufd_vevent_header *hdr;
317 	struct iommufd_vevent *cur;
318 	size_t done = 0;
319 	int rc = 0;
320 
321 	if (*ppos)
322 		return -ESPIPE;
323 	/* Minimum read count is a vEVENT header */
324 	if (count < sizeof(*hdr))
325 		return -EINVAL;
326 
327 	while ((cur = iommufd_veventq_deliver_fetch(veventq))) {
328 		if (IS_ERR(cur)) {
329 			if (done == 0)
330 				rc = PTR_ERR(cur);
331 			break;
332 		}
333 
334 		/* Validate the remaining bytes against the header size */
335 		if (done >= count || sizeof(*hdr) > count - done) {
336 			iommufd_veventq_deliver_restore(veventq, cur);
337 			break;
338 		}
339 		hdr = &cur->header;
340 
341 		/* If being a normal vEVENT, validate against the full size */
342 		if (!vevent_for_lost_events_header(cur) &&
343 		    sizeof(*hdr) + cur->data_len > count - done) {
344 			iommufd_veventq_deliver_restore(veventq, cur);
345 			/* Read count doesn't fit a single normal vEVENT */
346 			if (done == 0)
347 				rc = -EINVAL;
348 			break;
349 		}
350 
351 		if (copy_to_user(buf + done, hdr, sizeof(*hdr))) {
352 			iommufd_veventq_deliver_restore(veventq, cur);
353 			rc = -EFAULT;
354 			break;
355 		}
356 		done += sizeof(*hdr);
357 
358 		if (cur->data_len &&
359 		    copy_to_user(buf + done, cur->event_data, cur->data_len)) {
360 			iommufd_veventq_deliver_restore(veventq, cur);
361 			done -= sizeof(*hdr);
362 			rc = -EFAULT;
363 			break;
364 		}
365 		spin_lock(&eventq->lock);
366 		if (!vevent_for_lost_events_header(cur))
367 			veventq->num_events--;
368 		spin_unlock(&eventq->lock);
369 		done += cur->data_len;
370 		kfree(cur);
371 	}
372 
373 	return done == 0 ? rc : done;
374 }
375 
376 /* Common Event Queue Functions */
377 
378 static __poll_t iommufd_eventq_fops_poll(struct file *filep,
379 					 struct poll_table_struct *wait)
380 {
381 	struct iommufd_eventq *eventq = filep->private_data;
382 	__poll_t pollflags = 0;
383 
384 	if (eventq->obj.type == IOMMUFD_OBJ_FAULT)
385 		pollflags |= EPOLLOUT;
386 
387 	poll_wait(filep, &eventq->wait_queue, wait);
388 	spin_lock(&eventq->lock);
389 	if (!list_empty(&eventq->deliver))
390 		pollflags |= EPOLLIN | EPOLLRDNORM;
391 	spin_unlock(&eventq->lock);
392 
393 	return pollflags;
394 }
395 
396 static int iommufd_eventq_fops_release(struct inode *inode, struct file *filep)
397 {
398 	struct iommufd_eventq *eventq = filep->private_data;
399 	/* eventq may be freed once the users reference is dropped. */
400 	struct iommufd_ctx *ictx = eventq->ictx;
401 
402 	refcount_dec(&eventq->obj.users);
403 	iommufd_ctx_put(ictx);
404 	return 0;
405 }
406 
407 #define INIT_EVENTQ_FOPS(read_op, write_op)                                    \
408 	((const struct file_operations){                                       \
409 		.owner = THIS_MODULE,                                          \
410 		.open = nonseekable_open,                                      \
411 		.read = read_op,                                               \
412 		.write = write_op,                                             \
413 		.poll = iommufd_eventq_fops_poll,                              \
414 		.release = iommufd_eventq_fops_release,                        \
415 	})
416 
417 static int iommufd_eventq_init(struct iommufd_eventq *eventq, char *name,
418 			       struct iommufd_ctx *ictx,
419 			       const struct file_operations *fops)
420 {
421 	struct file *filep;
422 
423 	spin_lock_init(&eventq->lock);
424 	INIT_LIST_HEAD(&eventq->deliver);
425 	init_waitqueue_head(&eventq->wait_queue);
426 
427 	/* The filep is fput() by the core code during failure */
428 	filep = anon_inode_getfile(name, fops, eventq, O_RDWR);
429 	if (IS_ERR(filep))
430 		return PTR_ERR(filep);
431 
432 	eventq->ictx = ictx;
433 	iommufd_ctx_get(eventq->ictx);
434 	eventq->filep = filep;
435 	refcount_inc(&eventq->obj.users);
436 
437 	return get_unused_fd_flags(O_CLOEXEC);
438 }
439 
440 static const struct file_operations iommufd_fault_fops =
441 	INIT_EVENTQ_FOPS(iommufd_fault_fops_read, iommufd_fault_fops_write);
442 
443 int iommufd_fault_alloc(struct iommufd_ucmd *ucmd)
444 {
445 	struct iommu_fault_alloc *cmd = ucmd->cmd;
446 	struct iommufd_fault *fault;
447 	int fdno;
448 	int rc;
449 
450 	if (cmd->flags)
451 		return -EOPNOTSUPP;
452 
453 	fault = __iommufd_object_alloc_ucmd(ucmd, fault, IOMMUFD_OBJ_FAULT,
454 					    common.obj);
455 	if (IS_ERR(fault))
456 		return PTR_ERR(fault);
457 
458 	xa_init_flags(&fault->response, XA_FLAGS_ALLOC1);
459 	mutex_init(&fault->mutex);
460 
461 	fdno = iommufd_eventq_init(&fault->common, "[iommufd-pgfault]",
462 				   ucmd->ictx, &iommufd_fault_fops);
463 	if (fdno < 0)
464 		return fdno;
465 
466 	cmd->out_fault_id = fault->common.obj.id;
467 	cmd->out_fault_fd = fdno;
468 
469 	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
470 	if (rc)
471 		goto out_put_fdno;
472 
473 	fd_install(fdno, fault->common.filep);
474 
475 	return 0;
476 out_put_fdno:
477 	put_unused_fd(fdno);
478 	return rc;
479 }
480 
481 int iommufd_fault_iopf_handler(struct iopf_group *group)
482 {
483 	struct iommufd_hw_pagetable *hwpt;
484 	struct iommufd_fault *fault;
485 
486 	hwpt = group->attach_handle->domain->iommufd_hwpt;
487 	fault = hwpt->fault;
488 
489 	iopf_group_dequeue(group);
490 
491 	spin_lock(&fault->common.lock);
492 	list_add_tail(&group->node, &fault->common.deliver);
493 	spin_unlock(&fault->common.lock);
494 
495 	wake_up_interruptible(&fault->common.wait_queue);
496 
497 	return 0;
498 }
499 
500 static const struct file_operations iommufd_veventq_fops =
501 	INIT_EVENTQ_FOPS(iommufd_veventq_fops_read, NULL);
502 
503 /* An arbitrary upper bound for veventq_depth that fits all existing HWs */
504 #define VEVENTQ_MAX_DEPTH (1U << 19)
505 
506 int iommufd_veventq_alloc(struct iommufd_ucmd *ucmd)
507 {
508 	struct iommu_veventq_alloc *cmd = ucmd->cmd;
509 	struct iommufd_veventq *veventq;
510 	struct iommufd_viommu *viommu;
511 	int fdno;
512 	int rc;
513 
514 	if (cmd->flags || cmd->__reserved ||
515 	    cmd->type == IOMMU_VEVENTQ_TYPE_DEFAULT)
516 		return -EOPNOTSUPP;
517 	if (!cmd->veventq_depth || cmd->veventq_depth > VEVENTQ_MAX_DEPTH)
518 		return -EINVAL;
519 
520 	viommu = iommufd_get_viommu(ucmd, cmd->viommu_id);
521 	if (IS_ERR(viommu))
522 		return PTR_ERR(viommu);
523 
524 	down_write(&viommu->veventqs_rwsem);
525 
526 	if (iommufd_viommu_find_veventq(viommu, cmd->type)) {
527 		rc = -EEXIST;
528 		goto out_unlock_veventqs;
529 	}
530 
531 	veventq = __iommufd_object_alloc(ucmd->ictx, veventq,
532 					 IOMMUFD_OBJ_VEVENTQ, common.obj);
533 	if (IS_ERR(veventq)) {
534 		rc = PTR_ERR(veventq);
535 		goto out_unlock_veventqs;
536 	}
537 
538 	veventq->type = cmd->type;
539 	veventq->viommu = viommu;
540 	refcount_inc(&viommu->obj.users);
541 	veventq->depth = cmd->veventq_depth;
542 	list_add_tail(&veventq->node, &viommu->veventqs);
543 	veventq->lost_events_header.header.flags =
544 		IOMMU_VEVENTQ_FLAG_LOST_EVENTS;
545 
546 	fdno = iommufd_eventq_init(&veventq->common, "[iommufd-viommu-event]",
547 				   ucmd->ictx, &iommufd_veventq_fops);
548 	if (fdno < 0) {
549 		rc = fdno;
550 		goto out_abort;
551 	}
552 
553 	cmd->out_veventq_id = veventq->common.obj.id;
554 	cmd->out_veventq_fd = fdno;
555 
556 	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
557 	if (rc)
558 		goto out_put_fdno;
559 
560 	iommufd_object_finalize(ucmd->ictx, &veventq->common.obj);
561 	fd_install(fdno, veventq->common.filep);
562 	goto out_unlock_veventqs;
563 
564 out_put_fdno:
565 	put_unused_fd(fdno);
566 out_abort:
567 	iommufd_object_abort_and_destroy(ucmd->ictx, &veventq->common.obj);
568 out_unlock_veventqs:
569 	up_write(&viommu->veventqs_rwsem);
570 	iommufd_put_object(ucmd->ictx, &viommu->obj);
571 	return rc;
572 }
573