1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (C) 2024 Intel Corporation
3 */
4 #define pr_fmt(fmt) "iommufd: " fmt
5
6 #include <linux/anon_inodes.h>
7 #include <linux/file.h>
8 #include <linux/fs.h>
9 #include <linux/iommufd.h>
10 #include <linux/module.h>
11 #include <linux/mutex.h>
12 #include <linux/pci.h>
13 #include <linux/poll.h>
14 #include <uapi/linux/iommufd.h>
15
16 #include "../iommu-priv.h"
17 #include "iommufd_private.h"
18
iommufd_fault_iopf_enable(struct iommufd_device * idev)19 static int iommufd_fault_iopf_enable(struct iommufd_device *idev)
20 {
21 struct device *dev = idev->dev;
22 int ret;
23
24 /*
25 * Once we turn on PCI/PRI support for VF, the response failure code
26 * should not be forwarded to the hardware due to PRI being a shared
27 * resource between PF and VFs. There is no coordination for this
28 * shared capability. This waits for a vPRI reset to recover.
29 */
30 if (dev_is_pci(dev) && to_pci_dev(dev)->is_virtfn)
31 return -EINVAL;
32
33 mutex_lock(&idev->iopf_lock);
34 /* Device iopf has already been on. */
35 if (++idev->iopf_enabled > 1) {
36 mutex_unlock(&idev->iopf_lock);
37 return 0;
38 }
39
40 ret = iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_IOPF);
41 if (ret)
42 --idev->iopf_enabled;
43 mutex_unlock(&idev->iopf_lock);
44
45 return ret;
46 }
47
iommufd_fault_iopf_disable(struct iommufd_device * idev)48 static void iommufd_fault_iopf_disable(struct iommufd_device *idev)
49 {
50 mutex_lock(&idev->iopf_lock);
51 if (!WARN_ON(idev->iopf_enabled == 0)) {
52 if (--idev->iopf_enabled == 0)
53 iommu_dev_disable_feature(idev->dev, IOMMU_DEV_FEAT_IOPF);
54 }
55 mutex_unlock(&idev->iopf_lock);
56 }
57
__fault_domain_attach_dev(struct iommufd_hw_pagetable * hwpt,struct iommufd_device * idev)58 static int __fault_domain_attach_dev(struct iommufd_hw_pagetable *hwpt,
59 struct iommufd_device *idev)
60 {
61 struct iommufd_attach_handle *handle;
62 int ret;
63
64 handle = kzalloc(sizeof(*handle), GFP_KERNEL);
65 if (!handle)
66 return -ENOMEM;
67
68 handle->idev = idev;
69 ret = iommu_attach_group_handle(hwpt->domain, idev->igroup->group,
70 &handle->handle);
71 if (ret)
72 kfree(handle);
73
74 return ret;
75 }
76
iommufd_fault_domain_attach_dev(struct iommufd_hw_pagetable * hwpt,struct iommufd_device * idev)77 int iommufd_fault_domain_attach_dev(struct iommufd_hw_pagetable *hwpt,
78 struct iommufd_device *idev)
79 {
80 int ret;
81
82 if (!hwpt->fault)
83 return -EINVAL;
84
85 ret = iommufd_fault_iopf_enable(idev);
86 if (ret)
87 return ret;
88
89 ret = __fault_domain_attach_dev(hwpt, idev);
90 if (ret)
91 iommufd_fault_iopf_disable(idev);
92
93 return ret;
94 }
95
iommufd_auto_response_faults(struct iommufd_hw_pagetable * hwpt,struct iommufd_attach_handle * handle)96 static void iommufd_auto_response_faults(struct iommufd_hw_pagetable *hwpt,
97 struct iommufd_attach_handle *handle)
98 {
99 struct iommufd_fault *fault = hwpt->fault;
100 struct iopf_group *group, *next;
101 unsigned long index;
102
103 if (!fault)
104 return;
105
106 mutex_lock(&fault->mutex);
107 list_for_each_entry_safe(group, next, &fault->deliver, node) {
108 if (group->attach_handle != &handle->handle)
109 continue;
110 list_del(&group->node);
111 iopf_group_response(group, IOMMU_PAGE_RESP_INVALID);
112 iopf_free_group(group);
113 }
114
115 xa_for_each(&fault->response, index, group) {
116 if (group->attach_handle != &handle->handle)
117 continue;
118 xa_erase(&fault->response, index);
119 iopf_group_response(group, IOMMU_PAGE_RESP_INVALID);
120 iopf_free_group(group);
121 }
122 mutex_unlock(&fault->mutex);
123 }
124
125 static struct iommufd_attach_handle *
iommufd_device_get_attach_handle(struct iommufd_device * idev)126 iommufd_device_get_attach_handle(struct iommufd_device *idev)
127 {
128 struct iommu_attach_handle *handle;
129
130 handle = iommu_attach_handle_get(idev->igroup->group, IOMMU_NO_PASID, 0);
131 if (IS_ERR(handle))
132 return NULL;
133
134 return to_iommufd_handle(handle);
135 }
136
iommufd_fault_domain_detach_dev(struct iommufd_hw_pagetable * hwpt,struct iommufd_device * idev)137 void iommufd_fault_domain_detach_dev(struct iommufd_hw_pagetable *hwpt,
138 struct iommufd_device *idev)
139 {
140 struct iommufd_attach_handle *handle;
141
142 handle = iommufd_device_get_attach_handle(idev);
143 iommu_detach_group_handle(hwpt->domain, idev->igroup->group);
144 iommufd_auto_response_faults(hwpt, handle);
145 iommufd_fault_iopf_disable(idev);
146 kfree(handle);
147 }
148
__fault_domain_replace_dev(struct iommufd_device * idev,struct iommufd_hw_pagetable * hwpt,struct iommufd_hw_pagetable * old)149 static int __fault_domain_replace_dev(struct iommufd_device *idev,
150 struct iommufd_hw_pagetable *hwpt,
151 struct iommufd_hw_pagetable *old)
152 {
153 struct iommufd_attach_handle *handle, *curr = NULL;
154 int ret;
155
156 if (old->fault)
157 curr = iommufd_device_get_attach_handle(idev);
158
159 if (hwpt->fault) {
160 handle = kzalloc(sizeof(*handle), GFP_KERNEL);
161 if (!handle)
162 return -ENOMEM;
163
164 handle->idev = idev;
165 ret = iommu_replace_group_handle(idev->igroup->group,
166 hwpt->domain, &handle->handle);
167 } else {
168 ret = iommu_replace_group_handle(idev->igroup->group,
169 hwpt->domain, NULL);
170 }
171
172 if (!ret && curr) {
173 iommufd_auto_response_faults(old, curr);
174 kfree(curr);
175 }
176
177 return ret;
178 }
179
iommufd_fault_domain_replace_dev(struct iommufd_device * idev,struct iommufd_hw_pagetable * hwpt,struct iommufd_hw_pagetable * old)180 int iommufd_fault_domain_replace_dev(struct iommufd_device *idev,
181 struct iommufd_hw_pagetable *hwpt,
182 struct iommufd_hw_pagetable *old)
183 {
184 bool iopf_off = !hwpt->fault && old->fault;
185 bool iopf_on = hwpt->fault && !old->fault;
186 int ret;
187
188 if (iopf_on) {
189 ret = iommufd_fault_iopf_enable(idev);
190 if (ret)
191 return ret;
192 }
193
194 ret = __fault_domain_replace_dev(idev, hwpt, old);
195 if (ret) {
196 if (iopf_on)
197 iommufd_fault_iopf_disable(idev);
198 return ret;
199 }
200
201 if (iopf_off)
202 iommufd_fault_iopf_disable(idev);
203
204 return 0;
205 }
206
iommufd_fault_destroy(struct iommufd_object * obj)207 void iommufd_fault_destroy(struct iommufd_object *obj)
208 {
209 struct iommufd_fault *fault = container_of(obj, struct iommufd_fault, obj);
210 struct iopf_group *group, *next;
211
212 /*
213 * The iommufd object's reference count is zero at this point.
214 * We can be confident that no other threads are currently
215 * accessing this pointer. Therefore, acquiring the mutex here
216 * is unnecessary.
217 */
218 list_for_each_entry_safe(group, next, &fault->deliver, node) {
219 list_del(&group->node);
220 iopf_group_response(group, IOMMU_PAGE_RESP_INVALID);
221 iopf_free_group(group);
222 }
223 }
224
iommufd_compose_fault_message(struct iommu_fault * fault,struct iommu_hwpt_pgfault * hwpt_fault,struct iommufd_device * idev,u32 cookie)225 static void iommufd_compose_fault_message(struct iommu_fault *fault,
226 struct iommu_hwpt_pgfault *hwpt_fault,
227 struct iommufd_device *idev,
228 u32 cookie)
229 {
230 hwpt_fault->flags = fault->prm.flags;
231 hwpt_fault->dev_id = idev->obj.id;
232 hwpt_fault->pasid = fault->prm.pasid;
233 hwpt_fault->grpid = fault->prm.grpid;
234 hwpt_fault->perm = fault->prm.perm;
235 hwpt_fault->addr = fault->prm.addr;
236 hwpt_fault->length = 0;
237 hwpt_fault->cookie = cookie;
238 }
239
iommufd_fault_fops_read(struct file * filep,char __user * buf,size_t count,loff_t * ppos)240 static ssize_t iommufd_fault_fops_read(struct file *filep, char __user *buf,
241 size_t count, loff_t *ppos)
242 {
243 size_t fault_size = sizeof(struct iommu_hwpt_pgfault);
244 struct iommufd_fault *fault = filep->private_data;
245 struct iommu_hwpt_pgfault data;
246 struct iommufd_device *idev;
247 struct iopf_group *group;
248 struct iopf_fault *iopf;
249 size_t done = 0;
250 int rc = 0;
251
252 if (*ppos || count % fault_size)
253 return -ESPIPE;
254
255 mutex_lock(&fault->mutex);
256 while (!list_empty(&fault->deliver) && count > done) {
257 group = list_first_entry(&fault->deliver,
258 struct iopf_group, node);
259
260 if (group->fault_count * fault_size > count - done)
261 break;
262
263 rc = xa_alloc(&fault->response, &group->cookie, group,
264 xa_limit_32b, GFP_KERNEL);
265 if (rc)
266 break;
267
268 idev = to_iommufd_handle(group->attach_handle)->idev;
269 list_for_each_entry(iopf, &group->faults, list) {
270 iommufd_compose_fault_message(&iopf->fault,
271 &data, idev,
272 group->cookie);
273 if (copy_to_user(buf + done, &data, fault_size)) {
274 xa_erase(&fault->response, group->cookie);
275 rc = -EFAULT;
276 break;
277 }
278 done += fault_size;
279 }
280
281 list_del(&group->node);
282 }
283 mutex_unlock(&fault->mutex);
284
285 return done == 0 ? rc : done;
286 }
287
iommufd_fault_fops_write(struct file * filep,const char __user * buf,size_t count,loff_t * ppos)288 static ssize_t iommufd_fault_fops_write(struct file *filep, const char __user *buf,
289 size_t count, loff_t *ppos)
290 {
291 size_t response_size = sizeof(struct iommu_hwpt_page_response);
292 struct iommufd_fault *fault = filep->private_data;
293 struct iommu_hwpt_page_response response;
294 struct iopf_group *group;
295 size_t done = 0;
296 int rc = 0;
297
298 if (*ppos || count % response_size)
299 return -ESPIPE;
300
301 mutex_lock(&fault->mutex);
302 while (count > done) {
303 rc = copy_from_user(&response, buf + done, response_size);
304 if (rc)
305 break;
306
307 static_assert((int)IOMMUFD_PAGE_RESP_SUCCESS ==
308 (int)IOMMU_PAGE_RESP_SUCCESS);
309 static_assert((int)IOMMUFD_PAGE_RESP_INVALID ==
310 (int)IOMMU_PAGE_RESP_INVALID);
311 if (response.code != IOMMUFD_PAGE_RESP_SUCCESS &&
312 response.code != IOMMUFD_PAGE_RESP_INVALID) {
313 rc = -EINVAL;
314 break;
315 }
316
317 group = xa_erase(&fault->response, response.cookie);
318 if (!group) {
319 rc = -EINVAL;
320 break;
321 }
322
323 iopf_group_response(group, response.code);
324 iopf_free_group(group);
325 done += response_size;
326 }
327 mutex_unlock(&fault->mutex);
328
329 return done == 0 ? rc : done;
330 }
331
iommufd_fault_fops_poll(struct file * filep,struct poll_table_struct * wait)332 static __poll_t iommufd_fault_fops_poll(struct file *filep,
333 struct poll_table_struct *wait)
334 {
335 struct iommufd_fault *fault = filep->private_data;
336 __poll_t pollflags = EPOLLOUT;
337
338 poll_wait(filep, &fault->wait_queue, wait);
339 mutex_lock(&fault->mutex);
340 if (!list_empty(&fault->deliver))
341 pollflags |= EPOLLIN | EPOLLRDNORM;
342 mutex_unlock(&fault->mutex);
343
344 return pollflags;
345 }
346
iommufd_fault_fops_release(struct inode * inode,struct file * filep)347 static int iommufd_fault_fops_release(struct inode *inode, struct file *filep)
348 {
349 struct iommufd_fault *fault = filep->private_data;
350
351 refcount_dec(&fault->obj.users);
352 iommufd_ctx_put(fault->ictx);
353 return 0;
354 }
355
356 static const struct file_operations iommufd_fault_fops = {
357 .owner = THIS_MODULE,
358 .open = nonseekable_open,
359 .read = iommufd_fault_fops_read,
360 .write = iommufd_fault_fops_write,
361 .poll = iommufd_fault_fops_poll,
362 .release = iommufd_fault_fops_release,
363 };
364
iommufd_fault_alloc(struct iommufd_ucmd * ucmd)365 int iommufd_fault_alloc(struct iommufd_ucmd *ucmd)
366 {
367 struct iommu_fault_alloc *cmd = ucmd->cmd;
368 struct iommufd_fault *fault;
369 struct file *filep;
370 int fdno;
371 int rc;
372
373 if (cmd->flags)
374 return -EOPNOTSUPP;
375
376 fault = iommufd_object_alloc(ucmd->ictx, fault, IOMMUFD_OBJ_FAULT);
377 if (IS_ERR(fault))
378 return PTR_ERR(fault);
379
380 fault->ictx = ucmd->ictx;
381 INIT_LIST_HEAD(&fault->deliver);
382 xa_init_flags(&fault->response, XA_FLAGS_ALLOC1);
383 mutex_init(&fault->mutex);
384 init_waitqueue_head(&fault->wait_queue);
385
386 filep = anon_inode_getfile("[iommufd-pgfault]", &iommufd_fault_fops,
387 fault, O_RDWR);
388 if (IS_ERR(filep)) {
389 rc = PTR_ERR(filep);
390 goto out_abort;
391 }
392
393 refcount_inc(&fault->obj.users);
394 iommufd_ctx_get(fault->ictx);
395 fault->filep = filep;
396
397 fdno = get_unused_fd_flags(O_CLOEXEC);
398 if (fdno < 0) {
399 rc = fdno;
400 goto out_fput;
401 }
402
403 cmd->out_fault_id = fault->obj.id;
404 cmd->out_fault_fd = fdno;
405
406 rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
407 if (rc)
408 goto out_put_fdno;
409 iommufd_object_finalize(ucmd->ictx, &fault->obj);
410
411 fd_install(fdno, fault->filep);
412
413 return 0;
414 out_put_fdno:
415 put_unused_fd(fdno);
416 out_fput:
417 fput(filep);
418 refcount_dec(&fault->obj.users);
419 iommufd_ctx_put(fault->ictx);
420 out_abort:
421 iommufd_object_abort_and_destroy(ucmd->ictx, &fault->obj);
422
423 return rc;
424 }
425
iommufd_fault_iopf_handler(struct iopf_group * group)426 int iommufd_fault_iopf_handler(struct iopf_group *group)
427 {
428 struct iommufd_hw_pagetable *hwpt;
429 struct iommufd_fault *fault;
430
431 hwpt = group->attach_handle->domain->fault_data;
432 fault = hwpt->fault;
433
434 mutex_lock(&fault->mutex);
435 list_add_tail(&group->node, &fault->deliver);
436 mutex_unlock(&fault->mutex);
437
438 wake_up_interruptible(&fault->wait_queue);
439
440 return 0;
441 }
442