1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (C) 2024 Intel Corporation
3 */
4 #define pr_fmt(fmt) "iommufd: " fmt
5
6 #include <linux/anon_inodes.h>
7 #include <linux/file.h>
8 #include <linux/fs.h>
9 #include <linux/iommufd.h>
10 #include <linux/module.h>
11 #include <linux/mutex.h>
12 #include <linux/poll.h>
13 #include <uapi/linux/iommufd.h>
14
15 #include "../iommu-priv.h"
16 #include "iommufd_private.h"
17
18 /* IOMMUFD_OBJ_FAULT Functions */
iommufd_auto_response_faults(struct iommufd_hw_pagetable * hwpt,struct iommufd_attach_handle * handle)19 void iommufd_auto_response_faults(struct iommufd_hw_pagetable *hwpt,
20 struct iommufd_attach_handle *handle)
21 {
22 struct iommufd_fault *fault = hwpt->fault;
23 struct iopf_group *group, *next;
24 struct list_head free_list;
25 unsigned long index;
26
27 if (!fault || !handle)
28 return;
29 INIT_LIST_HEAD(&free_list);
30
31 mutex_lock(&fault->mutex);
32 spin_lock(&fault->common.lock);
33 list_for_each_entry_safe(group, next, &fault->common.deliver, node) {
34 if (group->attach_handle != &handle->handle)
35 continue;
36 list_move(&group->node, &free_list);
37 }
38 spin_unlock(&fault->common.lock);
39
40 list_for_each_entry_safe(group, next, &free_list, node) {
41 list_del(&group->node);
42 iopf_group_response(group, IOMMU_PAGE_RESP_INVALID);
43 iopf_free_group(group);
44 }
45
46 xa_for_each(&fault->response, index, group) {
47 if (group->attach_handle != &handle->handle)
48 continue;
49 xa_erase(&fault->response, index);
50 iopf_group_response(group, IOMMU_PAGE_RESP_INVALID);
51 iopf_free_group(group);
52 }
53 mutex_unlock(&fault->mutex);
54 }
55
iommufd_fault_destroy(struct iommufd_object * obj)56 void iommufd_fault_destroy(struct iommufd_object *obj)
57 {
58 struct iommufd_eventq *eventq =
59 container_of(obj, struct iommufd_eventq, obj);
60 struct iommufd_fault *fault = eventq_to_fault(eventq);
61 struct iopf_group *group, *next;
62 unsigned long index;
63
64 /*
65 * The iommufd object's reference count is zero at this point.
66 * We can be confident that no other threads are currently
67 * accessing this pointer. Therefore, acquiring the mutex here
68 * is unnecessary.
69 */
70 list_for_each_entry_safe(group, next, &fault->common.deliver, node) {
71 list_del(&group->node);
72 iopf_group_response(group, IOMMU_PAGE_RESP_INVALID);
73 iopf_free_group(group);
74 }
75 xa_for_each(&fault->response, index, group) {
76 xa_erase(&fault->response, index);
77 iopf_group_response(group, IOMMU_PAGE_RESP_INVALID);
78 iopf_free_group(group);
79 }
80 xa_destroy(&fault->response);
81 mutex_destroy(&fault->mutex);
82 }
83
iommufd_compose_fault_message(struct iommu_fault * fault,struct iommu_hwpt_pgfault * hwpt_fault,struct iommufd_device * idev,u32 cookie)84 static void iommufd_compose_fault_message(struct iommu_fault *fault,
85 struct iommu_hwpt_pgfault *hwpt_fault,
86 struct iommufd_device *idev,
87 u32 cookie)
88 {
89 hwpt_fault->flags = fault->prm.flags;
90 hwpt_fault->dev_id = idev->obj.id;
91 hwpt_fault->pasid = fault->prm.pasid;
92 hwpt_fault->grpid = fault->prm.grpid;
93 hwpt_fault->perm = fault->prm.perm;
94 hwpt_fault->addr = fault->prm.addr;
95 hwpt_fault->length = 0;
96 hwpt_fault->cookie = cookie;
97 }
98
99 /* Fetch the first node out of the fault->deliver list */
100 static struct iopf_group *
iommufd_fault_deliver_fetch(struct iommufd_fault * fault)101 iommufd_fault_deliver_fetch(struct iommufd_fault *fault)
102 {
103 struct list_head *list = &fault->common.deliver;
104 struct iopf_group *group = NULL;
105
106 spin_lock(&fault->common.lock);
107 if (!list_empty(list)) {
108 group = list_first_entry(list, struct iopf_group, node);
109 list_del(&group->node);
110 }
111 spin_unlock(&fault->common.lock);
112 return group;
113 }
114
115 /* Restore a node back to the head of the fault->deliver list */
iommufd_fault_deliver_restore(struct iommufd_fault * fault,struct iopf_group * group)116 static void iommufd_fault_deliver_restore(struct iommufd_fault *fault,
117 struct iopf_group *group)
118 {
119 spin_lock(&fault->common.lock);
120 list_add(&group->node, &fault->common.deliver);
121 spin_unlock(&fault->common.lock);
122 }
123
iommufd_fault_fops_read(struct file * filep,char __user * buf,size_t count,loff_t * ppos)124 static ssize_t iommufd_fault_fops_read(struct file *filep, char __user *buf,
125 size_t count, loff_t *ppos)
126 {
127 size_t fault_size = sizeof(struct iommu_hwpt_pgfault);
128 struct iommufd_eventq *eventq = filep->private_data;
129 struct iommufd_fault *fault = eventq_to_fault(eventq);
130 struct iommu_hwpt_pgfault data = {};
131 struct iommufd_device *idev;
132 struct iopf_group *group;
133 struct iopf_fault *iopf;
134 size_t done = 0;
135 int rc = 0;
136
137 if (*ppos || count % fault_size)
138 return -ESPIPE;
139
140 mutex_lock(&fault->mutex);
141 while ((group = iommufd_fault_deliver_fetch(fault))) {
142 size_t group_done = done;
143
144 if (done >= count ||
145 group->fault_count * fault_size > count - done) {
146 iommufd_fault_deliver_restore(fault, group);
147 /* Read count doesn't fit the first fault group */
148 if (done == 0)
149 rc = -EINVAL;
150 break;
151 }
152
153 rc = xa_alloc(&fault->response, &group->cookie, group,
154 xa_limit_32b, GFP_KERNEL);
155 if (rc) {
156 iommufd_fault_deliver_restore(fault, group);
157 break;
158 }
159
160 idev = to_iommufd_handle(group->attach_handle)->idev;
161 list_for_each_entry(iopf, &group->faults, list) {
162 iommufd_compose_fault_message(&iopf->fault,
163 &data, idev,
164 group->cookie);
165 if (copy_to_user(buf + group_done, &data, fault_size)) {
166 xa_erase(&fault->response, group->cookie);
167 iommufd_fault_deliver_restore(fault, group);
168 rc = -EFAULT;
169 break;
170 }
171 group_done += fault_size;
172 }
173 if (rc)
174 break;
175 done = group_done;
176 }
177 mutex_unlock(&fault->mutex);
178
179 return done == 0 ? rc : done;
180 }
181
iommufd_fault_fops_write(struct file * filep,const char __user * buf,size_t count,loff_t * ppos)182 static ssize_t iommufd_fault_fops_write(struct file *filep, const char __user *buf,
183 size_t count, loff_t *ppos)
184 {
185 size_t response_size = sizeof(struct iommu_hwpt_page_response);
186 struct iommufd_eventq *eventq = filep->private_data;
187 struct iommufd_fault *fault = eventq_to_fault(eventq);
188 struct iommu_hwpt_page_response response;
189 struct iopf_group *group;
190 size_t done = 0;
191 int rc = 0;
192
193 if (*ppos || count % response_size)
194 return -ESPIPE;
195
196 mutex_lock(&fault->mutex);
197 while (count > done) {
198 if (copy_from_user(&response, buf + done, response_size)) {
199 rc = -EFAULT;
200 break;
201 }
202
203 static_assert((int)IOMMUFD_PAGE_RESP_SUCCESS ==
204 (int)IOMMU_PAGE_RESP_SUCCESS);
205 static_assert((int)IOMMUFD_PAGE_RESP_INVALID ==
206 (int)IOMMU_PAGE_RESP_INVALID);
207 if (response.code != IOMMUFD_PAGE_RESP_SUCCESS &&
208 response.code != IOMMUFD_PAGE_RESP_INVALID) {
209 rc = -EINVAL;
210 break;
211 }
212
213 group = xa_erase(&fault->response, response.cookie);
214 if (!group) {
215 rc = -EINVAL;
216 break;
217 }
218
219 iopf_group_response(group, response.code);
220 iopf_free_group(group);
221 done += response_size;
222 }
223 mutex_unlock(&fault->mutex);
224
225 return done == 0 ? rc : done;
226 }
227
228 /* IOMMUFD_OBJ_VEVENTQ Functions */
229
iommufd_veventq_abort(struct iommufd_object * obj)230 void iommufd_veventq_abort(struct iommufd_object *obj)
231 {
232 struct iommufd_eventq *eventq =
233 container_of(obj, struct iommufd_eventq, obj);
234 struct iommufd_veventq *veventq = eventq_to_veventq(eventq);
235 struct iommufd_viommu *viommu = veventq->viommu;
236 struct iommufd_vevent *cur, *next;
237
238 lockdep_assert_held_write(&viommu->veventqs_rwsem);
239
240 list_for_each_entry_safe(cur, next, &eventq->deliver, node) {
241 list_del(&cur->node);
242 if (cur != &veventq->lost_events_header)
243 kfree(cur);
244 }
245
246 refcount_dec(&viommu->obj.users);
247 list_del(&veventq->node);
248 }
249
iommufd_veventq_destroy(struct iommufd_object * obj)250 void iommufd_veventq_destroy(struct iommufd_object *obj)
251 {
252 struct iommufd_veventq *veventq = eventq_to_veventq(
253 container_of(obj, struct iommufd_eventq, obj));
254
255 down_write(&veventq->viommu->veventqs_rwsem);
256 iommufd_veventq_abort(obj);
257 up_write(&veventq->viommu->veventqs_rwsem);
258 }
259
260 static struct iommufd_vevent *
iommufd_veventq_deliver_fetch(struct iommufd_veventq * veventq)261 iommufd_veventq_deliver_fetch(struct iommufd_veventq *veventq)
262 {
263 struct iommufd_eventq *eventq = &veventq->common;
264 struct list_head *list = &eventq->deliver;
265 struct iommufd_vevent *vevent = NULL;
266
267 spin_lock(&eventq->lock);
268 if (!list_empty(list)) {
269 struct iommufd_vevent *next;
270
271 next = list_first_entry(list, struct iommufd_vevent, node);
272 /* Make a copy of the lost_events_header for copy_to_user */
273 if (next == &veventq->lost_events_header) {
274 vevent = kzalloc_obj(*vevent, GFP_ATOMIC);
275 if (!vevent) {
276 vevent = ERR_PTR(-ENOMEM);
277 goto out_unlock;
278 }
279 }
280 list_del(&next->node);
281 if (vevent)
282 memcpy(vevent, next, sizeof(*vevent));
283 else
284 vevent = next;
285 }
286 out_unlock:
287 spin_unlock(&eventq->lock);
288 return vevent;
289 }
290
iommufd_veventq_deliver_restore(struct iommufd_veventq * veventq,struct iommufd_vevent * vevent)291 static void iommufd_veventq_deliver_restore(struct iommufd_veventq *veventq,
292 struct iommufd_vevent *vevent)
293 {
294 struct iommufd_eventq *eventq = &veventq->common;
295 struct list_head *list = &eventq->deliver;
296
297 spin_lock(&eventq->lock);
298 if (vevent_for_lost_events_header(vevent)) {
299 /* Remove the copy of the lost_events_header */
300 kfree(vevent);
301 vevent = NULL;
302 /* An empty list needs the lost_events_header back */
303 if (list_empty(list))
304 vevent = &veventq->lost_events_header;
305 }
306 if (vevent)
307 list_add(&vevent->node, list);
308 spin_unlock(&eventq->lock);
309 }
310
iommufd_veventq_fops_read(struct file * filep,char __user * buf,size_t count,loff_t * ppos)311 static ssize_t iommufd_veventq_fops_read(struct file *filep, char __user *buf,
312 size_t count, loff_t *ppos)
313 {
314 struct iommufd_eventq *eventq = filep->private_data;
315 struct iommufd_veventq *veventq = eventq_to_veventq(eventq);
316 struct iommufd_vevent_header *hdr;
317 struct iommufd_vevent *cur;
318 size_t done = 0;
319 int rc = 0;
320
321 if (*ppos)
322 return -ESPIPE;
323 /* Minimum read count is a vEVENT header */
324 if (count < sizeof(*hdr))
325 return -EINVAL;
326
327 while ((cur = iommufd_veventq_deliver_fetch(veventq))) {
328 if (IS_ERR(cur)) {
329 if (done == 0)
330 rc = PTR_ERR(cur);
331 break;
332 }
333
334 /* Validate the remaining bytes against the header size */
335 if (done >= count || sizeof(*hdr) > count - done) {
336 iommufd_veventq_deliver_restore(veventq, cur);
337 break;
338 }
339 hdr = &cur->header;
340
341 /* If being a normal vEVENT, validate against the full size */
342 if (!vevent_for_lost_events_header(cur) &&
343 sizeof(*hdr) + cur->data_len > count - done) {
344 iommufd_veventq_deliver_restore(veventq, cur);
345 /* Read count doesn't fit a single normal vEVENT */
346 if (done == 0)
347 rc = -EINVAL;
348 break;
349 }
350
351 if (copy_to_user(buf + done, hdr, sizeof(*hdr))) {
352 iommufd_veventq_deliver_restore(veventq, cur);
353 rc = -EFAULT;
354 break;
355 }
356 done += sizeof(*hdr);
357
358 if (cur->data_len &&
359 copy_to_user(buf + done, cur->event_data, cur->data_len)) {
360 iommufd_veventq_deliver_restore(veventq, cur);
361 done -= sizeof(*hdr);
362 rc = -EFAULT;
363 break;
364 }
365 spin_lock(&eventq->lock);
366 if (!vevent_for_lost_events_header(cur))
367 veventq->num_events--;
368 spin_unlock(&eventq->lock);
369 done += cur->data_len;
370 kfree(cur);
371 }
372
373 return done == 0 ? rc : done;
374 }
375
376 /* Common Event Queue Functions */
377
iommufd_eventq_fops_poll(struct file * filep,struct poll_table_struct * wait)378 static __poll_t iommufd_eventq_fops_poll(struct file *filep,
379 struct poll_table_struct *wait)
380 {
381 struct iommufd_eventq *eventq = filep->private_data;
382 __poll_t pollflags = 0;
383
384 if (eventq->obj.type == IOMMUFD_OBJ_FAULT)
385 pollflags |= EPOLLOUT;
386
387 poll_wait(filep, &eventq->wait_queue, wait);
388 spin_lock(&eventq->lock);
389 if (!list_empty(&eventq->deliver))
390 pollflags |= EPOLLIN | EPOLLRDNORM;
391 spin_unlock(&eventq->lock);
392
393 return pollflags;
394 }
395
iommufd_eventq_fops_release(struct inode * inode,struct file * filep)396 static int iommufd_eventq_fops_release(struct inode *inode, struct file *filep)
397 {
398 struct iommufd_eventq *eventq = filep->private_data;
399
400 refcount_dec(&eventq->obj.users);
401 iommufd_ctx_put(eventq->ictx);
402 return 0;
403 }
404
405 #define INIT_EVENTQ_FOPS(read_op, write_op) \
406 ((const struct file_operations){ \
407 .owner = THIS_MODULE, \
408 .open = nonseekable_open, \
409 .read = read_op, \
410 .write = write_op, \
411 .poll = iommufd_eventq_fops_poll, \
412 .release = iommufd_eventq_fops_release, \
413 })
414
iommufd_eventq_init(struct iommufd_eventq * eventq,char * name,struct iommufd_ctx * ictx,const struct file_operations * fops)415 static int iommufd_eventq_init(struct iommufd_eventq *eventq, char *name,
416 struct iommufd_ctx *ictx,
417 const struct file_operations *fops)
418 {
419 struct file *filep;
420
421 spin_lock_init(&eventq->lock);
422 INIT_LIST_HEAD(&eventq->deliver);
423 init_waitqueue_head(&eventq->wait_queue);
424
425 /* The filep is fput() by the core code during failure */
426 filep = anon_inode_getfile(name, fops, eventq, O_RDWR);
427 if (IS_ERR(filep))
428 return PTR_ERR(filep);
429
430 eventq->ictx = ictx;
431 iommufd_ctx_get(eventq->ictx);
432 eventq->filep = filep;
433 refcount_inc(&eventq->obj.users);
434
435 return get_unused_fd_flags(O_CLOEXEC);
436 }
437
438 static const struct file_operations iommufd_fault_fops =
439 INIT_EVENTQ_FOPS(iommufd_fault_fops_read, iommufd_fault_fops_write);
440
iommufd_fault_alloc(struct iommufd_ucmd * ucmd)441 int iommufd_fault_alloc(struct iommufd_ucmd *ucmd)
442 {
443 struct iommu_fault_alloc *cmd = ucmd->cmd;
444 struct iommufd_fault *fault;
445 int fdno;
446 int rc;
447
448 if (cmd->flags)
449 return -EOPNOTSUPP;
450
451 fault = __iommufd_object_alloc_ucmd(ucmd, fault, IOMMUFD_OBJ_FAULT,
452 common.obj);
453 if (IS_ERR(fault))
454 return PTR_ERR(fault);
455
456 xa_init_flags(&fault->response, XA_FLAGS_ALLOC1);
457 mutex_init(&fault->mutex);
458
459 fdno = iommufd_eventq_init(&fault->common, "[iommufd-pgfault]",
460 ucmd->ictx, &iommufd_fault_fops);
461 if (fdno < 0)
462 return fdno;
463
464 cmd->out_fault_id = fault->common.obj.id;
465 cmd->out_fault_fd = fdno;
466
467 rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
468 if (rc)
469 goto out_put_fdno;
470
471 fd_install(fdno, fault->common.filep);
472
473 return 0;
474 out_put_fdno:
475 put_unused_fd(fdno);
476 return rc;
477 }
478
iommufd_fault_iopf_handler(struct iopf_group * group)479 int iommufd_fault_iopf_handler(struct iopf_group *group)
480 {
481 struct iommufd_hw_pagetable *hwpt;
482 struct iommufd_fault *fault;
483
484 hwpt = group->attach_handle->domain->iommufd_hwpt;
485 fault = hwpt->fault;
486
487 iopf_group_dequeue(group);
488
489 spin_lock(&fault->common.lock);
490 list_add_tail(&group->node, &fault->common.deliver);
491 spin_unlock(&fault->common.lock);
492
493 wake_up_interruptible(&fault->common.wait_queue);
494
495 return 0;
496 }
497
498 static const struct file_operations iommufd_veventq_fops =
499 INIT_EVENTQ_FOPS(iommufd_veventq_fops_read, NULL);
500
501 /* An arbitrary upper bound for veventq_depth that fits all existing HWs */
502 #define VEVENTQ_MAX_DEPTH (1U << 19)
503
iommufd_veventq_alloc(struct iommufd_ucmd * ucmd)504 int iommufd_veventq_alloc(struct iommufd_ucmd *ucmd)
505 {
506 struct iommu_veventq_alloc *cmd = ucmd->cmd;
507 struct iommufd_veventq *veventq;
508 struct iommufd_viommu *viommu;
509 int fdno;
510 int rc;
511
512 if (cmd->flags || cmd->__reserved ||
513 cmd->type == IOMMU_VEVENTQ_TYPE_DEFAULT)
514 return -EOPNOTSUPP;
515 if (!cmd->veventq_depth || cmd->veventq_depth > VEVENTQ_MAX_DEPTH)
516 return -EINVAL;
517
518 viommu = iommufd_get_viommu(ucmd, cmd->viommu_id);
519 if (IS_ERR(viommu))
520 return PTR_ERR(viommu);
521
522 down_write(&viommu->veventqs_rwsem);
523
524 if (iommufd_viommu_find_veventq(viommu, cmd->type)) {
525 rc = -EEXIST;
526 goto out_unlock_veventqs;
527 }
528
529 veventq = __iommufd_object_alloc(ucmd->ictx, veventq,
530 IOMMUFD_OBJ_VEVENTQ, common.obj);
531 if (IS_ERR(veventq)) {
532 rc = PTR_ERR(veventq);
533 goto out_unlock_veventqs;
534 }
535
536 veventq->type = cmd->type;
537 veventq->viommu = viommu;
538 refcount_inc(&viommu->obj.users);
539 veventq->depth = cmd->veventq_depth;
540 list_add_tail(&veventq->node, &viommu->veventqs);
541 veventq->lost_events_header.header.flags =
542 IOMMU_VEVENTQ_FLAG_LOST_EVENTS;
543
544 fdno = iommufd_eventq_init(&veventq->common, "[iommufd-viommu-event]",
545 ucmd->ictx, &iommufd_veventq_fops);
546 if (fdno < 0) {
547 rc = fdno;
548 goto out_abort;
549 }
550
551 cmd->out_veventq_id = veventq->common.obj.id;
552 cmd->out_veventq_fd = fdno;
553
554 rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
555 if (rc)
556 goto out_put_fdno;
557
558 iommufd_object_finalize(ucmd->ictx, &veventq->common.obj);
559 fd_install(fdno, veventq->common.filep);
560 goto out_unlock_veventqs;
561
562 out_put_fdno:
563 put_unused_fd(fdno);
564 out_abort:
565 iommufd_object_abort_and_destroy(ucmd->ictx, &veventq->common.obj);
566 out_unlock_veventqs:
567 up_write(&viommu->veventqs_rwsem);
568 iommufd_put_object(ucmd->ictx, &viommu->obj);
569 return rc;
570 }
571