1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES
3 */
4 #include "iommufd_private.h"
5
iommufd_viommu_destroy(struct iommufd_object * obj)6 void iommufd_viommu_destroy(struct iommufd_object *obj)
7 {
8 struct iommufd_viommu *viommu =
9 container_of(obj, struct iommufd_viommu, obj);
10
11 if (viommu->ops && viommu->ops->destroy)
12 viommu->ops->destroy(viommu);
13 refcount_dec(&viommu->hwpt->common.obj.users);
14 xa_destroy(&viommu->vdevs);
15 }
16
iommufd_viommu_alloc_ioctl(struct iommufd_ucmd * ucmd)17 int iommufd_viommu_alloc_ioctl(struct iommufd_ucmd *ucmd)
18 {
19 struct iommu_viommu_alloc *cmd = ucmd->cmd;
20 const struct iommu_user_data user_data = {
21 .type = cmd->type,
22 .uptr = u64_to_user_ptr(cmd->data_uptr),
23 .len = cmd->data_len,
24 };
25 struct iommufd_hwpt_paging *hwpt_paging;
26 struct iommufd_viommu *viommu;
27 struct iommufd_device *idev;
28 struct iommu_device *iommu_dev;
29 const struct iommu_ops *ops;
30 size_t viommu_size;
31 int rc;
32
33 if (cmd->flags || cmd->type == IOMMU_VIOMMU_TYPE_DEFAULT)
34 return -EOPNOTSUPP;
35
36 idev = iommufd_get_device(ucmd, cmd->dev_id);
37 if (IS_ERR(idev))
38 return PTR_ERR(idev);
39
40 iommu_dev = iommufd_device_get_iommu_dev(idev);
41 if (!iommu_dev) {
42 rc = -EOPNOTSUPP;
43 goto out_put_idev;
44 }
45 ops = iommu_dev->ops;
46 if (!ops->get_viommu_size || !ops->viommu_init) {
47 rc = -EOPNOTSUPP;
48 goto out_put_idev;
49 }
50
51 viommu_size = ops->get_viommu_size(idev->dev, cmd->type);
52 if (!viommu_size) {
53 rc = -EOPNOTSUPP;
54 goto out_put_idev;
55 }
56
57 /*
58 * It is a driver bug for providing a viommu_size smaller than the core
59 * vIOMMU structure size
60 */
61 if (WARN_ON_ONCE(viommu_size < sizeof(*viommu))) {
62 rc = -EOPNOTSUPP;
63 goto out_put_idev;
64 }
65
66 hwpt_paging = iommufd_get_hwpt_paging(ucmd, cmd->hwpt_id);
67 if (IS_ERR(hwpt_paging)) {
68 rc = PTR_ERR(hwpt_paging);
69 goto out_put_idev;
70 }
71
72 if (!hwpt_paging->nest_parent) {
73 rc = -EINVAL;
74 goto out_put_hwpt;
75 }
76
77 viommu = (struct iommufd_viommu *)_iommufd_object_alloc_ucmd(
78 ucmd, viommu_size, IOMMUFD_OBJ_VIOMMU);
79 if (IS_ERR(viommu)) {
80 rc = PTR_ERR(viommu);
81 goto out_put_hwpt;
82 }
83
84 xa_init(&viommu->vdevs);
85 viommu->type = cmd->type;
86 viommu->ictx = ucmd->ictx;
87 viommu->hwpt = hwpt_paging;
88 refcount_inc(&viommu->hwpt->common.obj.users);
89 INIT_LIST_HEAD(&viommu->veventqs);
90 init_rwsem(&viommu->veventqs_rwsem);
91 /*
92 * It is the most likely case that a physical IOMMU is unpluggable. A
93 * pluggable IOMMU instance (if exists) is responsible for refcounting
94 * on its own.
95 */
96 viommu->iommu_dev = iommu_dev;
97
98 rc = ops->viommu_init(viommu, hwpt_paging->common.domain,
99 user_data.len ? &user_data : NULL);
100 if (rc)
101 goto out_put_hwpt;
102
103 /* It is a driver bug that viommu->ops isn't filled */
104 if (WARN_ON_ONCE(!viommu->ops)) {
105 rc = -EOPNOTSUPP;
106 goto out_put_hwpt;
107 }
108
109 cmd->out_viommu_id = viommu->obj.id;
110 rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
111
112 out_put_hwpt:
113 iommufd_put_object(ucmd->ictx, &hwpt_paging->common.obj);
114 out_put_idev:
115 iommufd_put_object(ucmd->ictx, &idev->obj);
116 return rc;
117 }
118
iommufd_vdevice_abort(struct iommufd_object * obj)119 void iommufd_vdevice_abort(struct iommufd_object *obj)
120 {
121 struct iommufd_vdevice *vdev =
122 container_of(obj, struct iommufd_vdevice, obj);
123 struct iommufd_viommu *viommu = vdev->viommu;
124 struct iommufd_device *idev = vdev->idev;
125
126 lockdep_assert_held(&idev->igroup->lock);
127
128 if (vdev->destroy)
129 vdev->destroy(vdev);
130 /* xa_cmpxchg is okay to fail if alloc failed xa_cmpxchg previously */
131 xa_cmpxchg(&viommu->vdevs, vdev->virt_id, vdev, NULL, GFP_KERNEL);
132 refcount_dec(&viommu->obj.users);
133 idev->vdev = NULL;
134 }
135
iommufd_vdevice_destroy(struct iommufd_object * obj)136 void iommufd_vdevice_destroy(struct iommufd_object *obj)
137 {
138 struct iommufd_vdevice *vdev =
139 container_of(obj, struct iommufd_vdevice, obj);
140 struct iommufd_device *idev = vdev->idev;
141 struct iommufd_ctx *ictx = idev->ictx;
142
143 mutex_lock(&idev->igroup->lock);
144 iommufd_vdevice_abort(obj);
145 mutex_unlock(&idev->igroup->lock);
146 iommufd_put_object(ictx, &idev->obj);
147 }
148
iommufd_vdevice_alloc_ioctl(struct iommufd_ucmd * ucmd)149 int iommufd_vdevice_alloc_ioctl(struct iommufd_ucmd *ucmd)
150 {
151 struct iommu_vdevice_alloc *cmd = ucmd->cmd;
152 struct iommufd_vdevice *vdev;
153 size_t vdev_size = sizeof(*vdev);
154 struct iommufd_viommu *viommu;
155 struct iommu_device *iommu_dev;
156 struct iommufd_device *idev;
157 u64 virt_id = cmd->virt_id;
158 int rc = 0;
159
160 /* virt_id indexes an xarray */
161 if (virt_id > ULONG_MAX)
162 return -EINVAL;
163
164 viommu = iommufd_get_viommu(ucmd, cmd->viommu_id);
165 if (IS_ERR(viommu))
166 return PTR_ERR(viommu);
167
168 idev = iommufd_get_device(ucmd, cmd->dev_id);
169 if (IS_ERR(idev)) {
170 rc = PTR_ERR(idev);
171 goto out_put_viommu;
172 }
173
174 iommu_dev = iommufd_device_get_iommu_dev(idev);
175 if (!iommu_dev || viommu->iommu_dev != iommu_dev) {
176 rc = -EINVAL;
177 goto out_put_idev;
178 }
179
180 mutex_lock(&idev->igroup->lock);
181 if (idev->destroying) {
182 rc = -ENOENT;
183 goto out_unlock_igroup;
184 }
185
186 if (idev->vdev) {
187 rc = -EEXIST;
188 goto out_unlock_igroup;
189 }
190
191 if (viommu->ops && viommu->ops->vdevice_size) {
192 /*
193 * It is a driver bug for:
194 * - ops->vdevice_size smaller than the core structure size
195 * - not implementing a pairing ops->vdevice_init op
196 */
197 if (WARN_ON_ONCE(viommu->ops->vdevice_size < vdev_size ||
198 !viommu->ops->vdevice_init)) {
199 rc = -EOPNOTSUPP;
200 goto out_unlock_igroup;
201 }
202 vdev_size = viommu->ops->vdevice_size;
203 }
204
205 vdev = (struct iommufd_vdevice *)_iommufd_object_alloc(
206 ucmd->ictx, vdev_size, IOMMUFD_OBJ_VDEVICE);
207 if (IS_ERR(vdev)) {
208 rc = PTR_ERR(vdev);
209 goto out_unlock_igroup;
210 }
211
212 vdev->virt_id = virt_id;
213 vdev->viommu = viommu;
214 refcount_inc(&viommu->obj.users);
215 /*
216 * A wait_cnt reference is held on the idev so long as we have the
217 * pointer. iommufd_device_pre_destroy() will revoke it before the
218 * idev real destruction.
219 */
220 vdev->idev = idev;
221
222 /*
223 * iommufd_device_destroy() delays until idev->vdev is NULL before
224 * freeing the idev, which only happens once the vdev is finished
225 * destruction.
226 */
227 idev->vdev = vdev;
228
229 /*
230 * Reserve the slot with a zero entry (reads back as NULL) until the
231 * vdevice_init() op accepts the vDEVICE. Only the xa_* helpers hide a
232 * reserved entry, so never use a raw xas_* iterator on this xarray.
233 */
234 rc = xa_insert(&viommu->vdevs, virt_id, NULL, GFP_KERNEL);
235 if (rc) {
236 if (rc == -EBUSY)
237 rc = -EEXIST;
238 goto out_abort;
239 }
240
241 if (viommu->ops && viommu->ops->vdevice_init) {
242 rc = viommu->ops->vdevice_init(vdev);
243 if (rc) {
244 xa_release(&viommu->vdevs, virt_id);
245 goto out_abort;
246 }
247 }
248
249 xa_store(&viommu->vdevs, virt_id, vdev, GFP_KERNEL);
250
251 cmd->out_vdevice_id = vdev->obj.id;
252 rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
253 if (rc)
254 goto out_abort;
255 iommufd_object_finalize(ucmd->ictx, &vdev->obj);
256 goto out_unlock_igroup;
257
258 out_abort:
259 iommufd_object_abort_and_destroy(ucmd->ictx, &vdev->obj);
260 out_unlock_igroup:
261 mutex_unlock(&idev->igroup->lock);
262 out_put_idev:
263 if (rc)
264 iommufd_put_object(ucmd->ictx, &idev->obj);
265 out_put_viommu:
266 iommufd_put_object(ucmd->ictx, &viommu->obj);
267 return rc;
268 }
269
iommufd_hw_queue_destroy_access(struct iommufd_ctx * ictx,struct iommufd_access * access,u64 base_iova,size_t length)270 static void iommufd_hw_queue_destroy_access(struct iommufd_ctx *ictx,
271 struct iommufd_access *access,
272 u64 base_iova, size_t length)
273 {
274 u64 aligned_iova = PAGE_ALIGN_DOWN(base_iova);
275 u64 offset = base_iova - aligned_iova;
276
277 iommufd_access_unpin_pages(access, aligned_iova,
278 PAGE_ALIGN(length + offset));
279 iommufd_access_detach_internal(access);
280 iommufd_access_destroy_internal(ictx, access);
281 }
282
iommufd_hw_queue_destroy(struct iommufd_object * obj)283 void iommufd_hw_queue_destroy(struct iommufd_object *obj)
284 {
285 struct iommufd_hw_queue *hw_queue =
286 container_of(obj, struct iommufd_hw_queue, obj);
287
288 if (hw_queue->destroy)
289 hw_queue->destroy(hw_queue);
290 if (hw_queue->access)
291 iommufd_hw_queue_destroy_access(hw_queue->viommu->ictx,
292 hw_queue->access,
293 hw_queue->base_addr,
294 hw_queue->length);
295 if (hw_queue->viommu)
296 refcount_dec(&hw_queue->viommu->obj.users);
297 }
298
299 /*
300 * When the HW accesses the guest queue via physical addresses, the underlying
301 * physical pages of the guest queue must be contiguous. Also, for the security
302 * concern that IOMMUFD_CMD_IOAS_UNMAP could potentially remove the mappings of
303 * the guest queue from the nesting parent iopt while the HW is still accessing
304 * the guest queue memory physically, such a HW queue must require an access to
305 * pin the underlying pages and prevent that from happening.
306 */
307 static struct iommufd_access *
iommufd_hw_queue_alloc_phys(struct iommu_hw_queue_alloc * cmd,struct iommufd_viommu * viommu,phys_addr_t * base_pa)308 iommufd_hw_queue_alloc_phys(struct iommu_hw_queue_alloc *cmd,
309 struct iommufd_viommu *viommu, phys_addr_t *base_pa)
310 {
311 u64 aligned_iova = PAGE_ALIGN_DOWN(cmd->nesting_parent_iova);
312 u64 offset = cmd->nesting_parent_iova - aligned_iova;
313 struct iommufd_access *access;
314 struct page **pages;
315 size_t max_npages;
316 size_t length;
317 size_t i;
318 int rc;
319
320 /* max_npages = DIV_ROUND_UP(offset + cmd->length, PAGE_SIZE) */
321 if (check_add_overflow(offset, cmd->length, &length))
322 return ERR_PTR(-ERANGE);
323 if (check_add_overflow(length, PAGE_SIZE - 1, &length))
324 return ERR_PTR(-ERANGE);
325 max_npages = length / PAGE_SIZE;
326 /* length needs to be page aligned too */
327 length = max_npages * PAGE_SIZE;
328
329 /*
330 * Use kvcalloc() to avoid memory fragmentation for a large page array.
331 * Set __GFP_NOWARN to avoid syzkaller blowups
332 */
333 pages = kvzalloc_objs(*pages, max_npages, GFP_KERNEL | __GFP_NOWARN);
334 if (!pages)
335 return ERR_PTR(-ENOMEM);
336
337 access = iommufd_access_create_internal(viommu->ictx);
338 if (IS_ERR(access)) {
339 rc = PTR_ERR(access);
340 goto out_free;
341 }
342
343 rc = iommufd_access_attach_internal(access, viommu->hwpt->ioas);
344 if (rc)
345 goto out_destroy;
346
347 rc = iommufd_access_pin_pages(access, aligned_iova, length, pages, 0);
348 if (rc)
349 goto out_detach;
350
351 /* Validate if the underlying physical pages are contiguous */
352 for (i = 1; i < max_npages; i++) {
353 if (page_to_pfn(pages[i]) == page_to_pfn(pages[i - 1]) + 1)
354 continue;
355 rc = -EFAULT;
356 goto out_unpin;
357 }
358
359 *base_pa = (page_to_pfn(pages[0]) << PAGE_SHIFT) + offset;
360 kvfree(pages);
361 return access;
362
363 out_unpin:
364 iommufd_access_unpin_pages(access, aligned_iova, length);
365 out_detach:
366 iommufd_access_detach_internal(access);
367 out_destroy:
368 iommufd_access_destroy_internal(viommu->ictx, access);
369 out_free:
370 kvfree(pages);
371 return ERR_PTR(rc);
372 }
373
iommufd_hw_queue_alloc_ioctl(struct iommufd_ucmd * ucmd)374 int iommufd_hw_queue_alloc_ioctl(struct iommufd_ucmd *ucmd)
375 {
376 struct iommu_hw_queue_alloc *cmd = ucmd->cmd;
377 struct iommufd_hw_queue *hw_queue;
378 struct iommufd_viommu *viommu;
379 struct iommufd_access *access;
380 size_t hw_queue_size;
381 phys_addr_t base_pa;
382 u64 last;
383 int rc;
384
385 if (cmd->flags || cmd->type == IOMMU_HW_QUEUE_TYPE_DEFAULT)
386 return -EOPNOTSUPP;
387 if (!cmd->length)
388 return -EINVAL;
389 if (check_add_overflow(cmd->nesting_parent_iova, cmd->length - 1,
390 &last))
391 return -EOVERFLOW;
392
393 viommu = iommufd_get_viommu(ucmd, cmd->viommu_id);
394 if (IS_ERR(viommu))
395 return PTR_ERR(viommu);
396
397 if (!viommu->ops || !viommu->ops->get_hw_queue_size ||
398 !viommu->ops->hw_queue_init_phys) {
399 rc = -EOPNOTSUPP;
400 goto out_put_viommu;
401 }
402
403 hw_queue_size = viommu->ops->get_hw_queue_size(viommu, cmd->type);
404 if (!hw_queue_size) {
405 rc = -EOPNOTSUPP;
406 goto out_put_viommu;
407 }
408
409 /*
410 * It is a driver bug for providing a hw_queue_size smaller than the
411 * core HW queue structure size
412 */
413 if (WARN_ON_ONCE(hw_queue_size < sizeof(*hw_queue))) {
414 rc = -EOPNOTSUPP;
415 goto out_put_viommu;
416 }
417
418 hw_queue = (struct iommufd_hw_queue *)_iommufd_object_alloc_ucmd(
419 ucmd, hw_queue_size, IOMMUFD_OBJ_HW_QUEUE);
420 if (IS_ERR(hw_queue)) {
421 rc = PTR_ERR(hw_queue);
422 goto out_put_viommu;
423 }
424
425 access = iommufd_hw_queue_alloc_phys(cmd, viommu, &base_pa);
426 if (IS_ERR(access)) {
427 rc = PTR_ERR(access);
428 goto out_put_viommu;
429 }
430
431 hw_queue->viommu = viommu;
432 refcount_inc(&viommu->obj.users);
433 hw_queue->access = access;
434 hw_queue->type = cmd->type;
435 hw_queue->length = cmd->length;
436 hw_queue->base_addr = cmd->nesting_parent_iova;
437
438 rc = viommu->ops->hw_queue_init_phys(hw_queue, cmd->index, base_pa);
439 if (rc)
440 goto out_put_viommu;
441
442 cmd->out_hw_queue_id = hw_queue->obj.id;
443 rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
444
445 out_put_viommu:
446 iommufd_put_object(ucmd->ictx, &viommu->obj);
447 return rc;
448 }
449