xref: /linux/drivers/iommu/iommufd/hw_pagetable.c (revision fe7fad476ec8153a8b8767a08114e3e4a58a837e) 
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES
4  */
5 #include <linux/iommu.h>
6 #include <uapi/linux/iommufd.h>
7 
8 #include "../iommu-priv.h"
9 #include "iommufd_private.h"
10 
11 static void __iommufd_hwpt_destroy(struct iommufd_hw_pagetable *hwpt)
12 {
13 	if (hwpt->domain)
14 		iommu_domain_free(hwpt->domain);
15 
16 	if (hwpt->fault)
17 		refcount_dec(&hwpt->fault->obj.users);
18 }
19 
20 void iommufd_hwpt_paging_destroy(struct iommufd_object *obj)
21 {
22 	struct iommufd_hwpt_paging *hwpt_paging =
23 		container_of(obj, struct iommufd_hwpt_paging, common.obj);
24 
25 	if (!list_empty(&hwpt_paging->hwpt_item)) {
26 		mutex_lock(&hwpt_paging->ioas->mutex);
27 		list_del(&hwpt_paging->hwpt_item);
28 		mutex_unlock(&hwpt_paging->ioas->mutex);
29 
30 		iopt_table_remove_domain(&hwpt_paging->ioas->iopt,
31 					 hwpt_paging->common.domain);
32 	}
33 
34 	__iommufd_hwpt_destroy(&hwpt_paging->common);
35 	refcount_dec(&hwpt_paging->ioas->obj.users);
36 }
37 
38 void iommufd_hwpt_paging_abort(struct iommufd_object *obj)
39 {
40 	struct iommufd_hwpt_paging *hwpt_paging =
41 		container_of(obj, struct iommufd_hwpt_paging, common.obj);
42 
43 	/* The ioas->mutex must be held until finalize is called. */
44 	lockdep_assert_held(&hwpt_paging->ioas->mutex);
45 
46 	if (!list_empty(&hwpt_paging->hwpt_item)) {
47 		list_del_init(&hwpt_paging->hwpt_item);
48 		iopt_table_remove_domain(&hwpt_paging->ioas->iopt,
49 					 hwpt_paging->common.domain);
50 	}
51 	iommufd_hwpt_paging_destroy(obj);
52 }
53 
54 void iommufd_hwpt_nested_destroy(struct iommufd_object *obj)
55 {
56 	struct iommufd_hwpt_nested *hwpt_nested =
57 		container_of(obj, struct iommufd_hwpt_nested, common.obj);
58 
59 	__iommufd_hwpt_destroy(&hwpt_nested->common);
60 	if (hwpt_nested->viommu)
61 		refcount_dec(&hwpt_nested->viommu->obj.users);
62 	else
63 		refcount_dec(&hwpt_nested->parent->common.obj.users);
64 }
65 
66 void iommufd_hwpt_nested_abort(struct iommufd_object *obj)
67 {
68 	iommufd_hwpt_nested_destroy(obj);
69 }
70 
71 static int
72 iommufd_hwpt_paging_enforce_cc(struct iommufd_hwpt_paging *hwpt_paging)
73 {
74 	struct iommu_domain *paging_domain = hwpt_paging->common.domain;
75 
76 	if (hwpt_paging->enforce_cache_coherency)
77 		return 0;
78 
79 	if (paging_domain->ops->enforce_cache_coherency)
80 		hwpt_paging->enforce_cache_coherency =
81 			paging_domain->ops->enforce_cache_coherency(
82 				paging_domain);
83 	if (!hwpt_paging->enforce_cache_coherency)
84 		return -EINVAL;
85 	return 0;
86 }
87 
88 /**
89  * iommufd_hwpt_paging_alloc() - Get a PAGING iommu_domain for a device
90  * @ictx: iommufd context
91  * @ioas: IOAS to associate the domain with
92  * @idev: Device to get an iommu_domain for
93  * @flags: Flags from userspace
94  * @immediate_attach: True if idev should be attached to the hwpt
95  * @user_data: The user provided driver specific data describing the domain to
96  *             create
97  *
98  * Allocate a new iommu_domain and return it as a hw_pagetable. The HWPT
99  * will be linked to the given ioas and upon return the underlying iommu_domain
100  * is fully popoulated.
101  *
102  * The caller must hold the ioas->mutex until after
103  * iommufd_object_abort_and_destroy() or iommufd_object_finalize() is called on
104  * the returned hwpt.
105  */
106 struct iommufd_hwpt_paging *
107 iommufd_hwpt_paging_alloc(struct iommufd_ctx *ictx, struct iommufd_ioas *ioas,
108 			  struct iommufd_device *idev, u32 flags,
109 			  bool immediate_attach,
110 			  const struct iommu_user_data *user_data)
111 {
112 	const u32 valid_flags = IOMMU_HWPT_ALLOC_NEST_PARENT |
113 				IOMMU_HWPT_ALLOC_DIRTY_TRACKING |
114 				IOMMU_HWPT_FAULT_ID_VALID;
115 	const struct iommu_ops *ops = dev_iommu_ops(idev->dev);
116 	struct iommufd_hwpt_paging *hwpt_paging;
117 	struct iommufd_hw_pagetable *hwpt;
118 	int rc;
119 
120 	lockdep_assert_held(&ioas->mutex);
121 
122 	if ((flags || user_data) && !ops->domain_alloc_paging_flags)
123 		return ERR_PTR(-EOPNOTSUPP);
124 	if (flags & ~valid_flags)
125 		return ERR_PTR(-EOPNOTSUPP);
126 	if ((flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING) &&
127 	    !device_iommu_capable(idev->dev, IOMMU_CAP_DIRTY_TRACKING))
128 		return ERR_PTR(-EOPNOTSUPP);
129 
130 	hwpt_paging = __iommufd_object_alloc(
131 		ictx, hwpt_paging, IOMMUFD_OBJ_HWPT_PAGING, common.obj);
132 	if (IS_ERR(hwpt_paging))
133 		return ERR_CAST(hwpt_paging);
134 	hwpt = &hwpt_paging->common;
135 
136 	INIT_LIST_HEAD(&hwpt_paging->hwpt_item);
137 	/* Pairs with iommufd_hw_pagetable_destroy() */
138 	refcount_inc(&ioas->obj.users);
139 	hwpt_paging->ioas = ioas;
140 	hwpt_paging->nest_parent = flags & IOMMU_HWPT_ALLOC_NEST_PARENT;
141 
142 	if (ops->domain_alloc_paging_flags) {
143 		hwpt->domain = ops->domain_alloc_paging_flags(idev->dev,
144 				flags & ~IOMMU_HWPT_FAULT_ID_VALID, user_data);
145 		if (IS_ERR(hwpt->domain)) {
146 			rc = PTR_ERR(hwpt->domain);
147 			hwpt->domain = NULL;
148 			goto out_abort;
149 		}
150 		hwpt->domain->owner = ops;
151 	} else {
152 		hwpt->domain = iommu_paging_domain_alloc(idev->dev);
153 		if (IS_ERR(hwpt->domain)) {
154 			rc = PTR_ERR(hwpt->domain);
155 			hwpt->domain = NULL;
156 			goto out_abort;
157 		}
158 	}
159 
160 	/*
161 	 * Set the coherency mode before we do iopt_table_add_domain() as some
162 	 * iommus have a per-PTE bit that controls it and need to decide before
163 	 * doing any maps. It is an iommu driver bug to report
164 	 * IOMMU_CAP_ENFORCE_CACHE_COHERENCY but fail enforce_cache_coherency on
165 	 * a new domain.
166 	 *
167 	 * The cache coherency mode must be configured here and unchanged later.
168 	 * Note that a HWPT (non-CC) created for a device (non-CC) can be later
169 	 * reused by another device (either non-CC or CC). However, A HWPT (CC)
170 	 * created for a device (CC) cannot be reused by another device (non-CC)
171 	 * but only devices (CC). Instead user space in this case would need to
172 	 * allocate a separate HWPT (non-CC).
173 	 */
174 	if (idev->enforce_cache_coherency) {
175 		rc = iommufd_hwpt_paging_enforce_cc(hwpt_paging);
176 		if (WARN_ON(rc))
177 			goto out_abort;
178 	}
179 
180 	/*
181 	 * immediate_attach exists only to accommodate iommu drivers that cannot
182 	 * directly allocate a domain. These drivers do not finish creating the
183 	 * domain until attach is completed. Thus we must have this call
184 	 * sequence. Once those drivers are fixed this should be removed.
185 	 */
186 	if (immediate_attach) {
187 		rc = iommufd_hw_pagetable_attach(hwpt, idev);
188 		if (rc)
189 			goto out_abort;
190 	}
191 
192 	rc = iopt_table_add_domain(&ioas->iopt, hwpt->domain);
193 	if (rc)
194 		goto out_detach;
195 	list_add_tail(&hwpt_paging->hwpt_item, &ioas->hwpt_list);
196 	return hwpt_paging;
197 
198 out_detach:
199 	if (immediate_attach)
200 		iommufd_hw_pagetable_detach(idev);
201 out_abort:
202 	iommufd_object_abort_and_destroy(ictx, &hwpt->obj);
203 	return ERR_PTR(rc);
204 }
205 
206 /**
207  * iommufd_hwpt_nested_alloc() - Get a NESTED iommu_domain for a device
208  * @ictx: iommufd context
209  * @parent: Parent PAGING-type hwpt to associate the domain with
210  * @idev: Device to get an iommu_domain for
211  * @flags: Flags from userspace
212  * @user_data: user_data pointer. Must be valid
213  *
214  * Allocate a new iommu_domain (must be IOMMU_DOMAIN_NESTED) and return it as
215  * a NESTED hw_pagetable. The given parent PAGING-type hwpt must be capable of
216  * being a parent.
217  */
218 static struct iommufd_hwpt_nested *
219 iommufd_hwpt_nested_alloc(struct iommufd_ctx *ictx,
220 			  struct iommufd_hwpt_paging *parent,
221 			  struct iommufd_device *idev, u32 flags,
222 			  const struct iommu_user_data *user_data)
223 {
224 	const struct iommu_ops *ops = dev_iommu_ops(idev->dev);
225 	struct iommufd_hwpt_nested *hwpt_nested;
226 	struct iommufd_hw_pagetable *hwpt;
227 	int rc;
228 
229 	if ((flags & ~IOMMU_HWPT_FAULT_ID_VALID) ||
230 	    !user_data->len || !ops->domain_alloc_nested)
231 		return ERR_PTR(-EOPNOTSUPP);
232 	if (parent->auto_domain || !parent->nest_parent ||
233 	    parent->common.domain->owner != ops)
234 		return ERR_PTR(-EINVAL);
235 
236 	hwpt_nested = __iommufd_object_alloc(
237 		ictx, hwpt_nested, IOMMUFD_OBJ_HWPT_NESTED, common.obj);
238 	if (IS_ERR(hwpt_nested))
239 		return ERR_CAST(hwpt_nested);
240 	hwpt = &hwpt_nested->common;
241 
242 	refcount_inc(&parent->common.obj.users);
243 	hwpt_nested->parent = parent;
244 
245 	hwpt->domain = ops->domain_alloc_nested(
246 		idev->dev, parent->common.domain,
247 		flags & ~IOMMU_HWPT_FAULT_ID_VALID, user_data);
248 	if (IS_ERR(hwpt->domain)) {
249 		rc = PTR_ERR(hwpt->domain);
250 		hwpt->domain = NULL;
251 		goto out_abort;
252 	}
253 	hwpt->domain->owner = ops;
254 
255 	if (WARN_ON_ONCE(hwpt->domain->type != IOMMU_DOMAIN_NESTED)) {
256 		rc = -EINVAL;
257 		goto out_abort;
258 	}
259 	return hwpt_nested;
260 
261 out_abort:
262 	iommufd_object_abort_and_destroy(ictx, &hwpt->obj);
263 	return ERR_PTR(rc);
264 }
265 
266 /**
267  * iommufd_viommu_alloc_hwpt_nested() - Get a hwpt_nested for a vIOMMU
268  * @viommu: vIOMMU ojbect to associate the hwpt_nested/domain with
269  * @flags: Flags from userspace
270  * @user_data: user_data pointer. Must be valid
271  *
272  * Allocate a new IOMMU_DOMAIN_NESTED for a vIOMMU and return it as a NESTED
273  * hw_pagetable.
274  */
275 static struct iommufd_hwpt_nested *
276 iommufd_viommu_alloc_hwpt_nested(struct iommufd_viommu *viommu, u32 flags,
277 				 const struct iommu_user_data *user_data)
278 {
279 	struct iommufd_hwpt_nested *hwpt_nested;
280 	struct iommufd_hw_pagetable *hwpt;
281 	int rc;
282 
283 	if (flags & ~IOMMU_HWPT_FAULT_ID_VALID)
284 		return ERR_PTR(-EOPNOTSUPP);
285 	if (!user_data->len)
286 		return ERR_PTR(-EOPNOTSUPP);
287 	if (!viommu->ops || !viommu->ops->alloc_domain_nested)
288 		return ERR_PTR(-EOPNOTSUPP);
289 
290 	hwpt_nested = __iommufd_object_alloc(
291 		viommu->ictx, hwpt_nested, IOMMUFD_OBJ_HWPT_NESTED, common.obj);
292 	if (IS_ERR(hwpt_nested))
293 		return ERR_CAST(hwpt_nested);
294 	hwpt = &hwpt_nested->common;
295 
296 	hwpt_nested->viommu = viommu;
297 	refcount_inc(&viommu->obj.users);
298 	hwpt_nested->parent = viommu->hwpt;
299 
300 	hwpt->domain =
301 		viommu->ops->alloc_domain_nested(viommu,
302 				flags & ~IOMMU_HWPT_FAULT_ID_VALID,
303 				user_data);
304 	if (IS_ERR(hwpt->domain)) {
305 		rc = PTR_ERR(hwpt->domain);
306 		hwpt->domain = NULL;
307 		goto out_abort;
308 	}
309 	hwpt->domain->owner = viommu->iommu_dev->ops;
310 
311 	if (WARN_ON_ONCE(hwpt->domain->type != IOMMU_DOMAIN_NESTED)) {
312 		rc = -EINVAL;
313 		goto out_abort;
314 	}
315 	return hwpt_nested;
316 
317 out_abort:
318 	iommufd_object_abort_and_destroy(viommu->ictx, &hwpt->obj);
319 	return ERR_PTR(rc);
320 }
321 
322 int iommufd_hwpt_alloc(struct iommufd_ucmd *ucmd)
323 {
324 	struct iommu_hwpt_alloc *cmd = ucmd->cmd;
325 	const struct iommu_user_data user_data = {
326 		.type = cmd->data_type,
327 		.uptr = u64_to_user_ptr(cmd->data_uptr),
328 		.len = cmd->data_len,
329 	};
330 	struct iommufd_hw_pagetable *hwpt;
331 	struct iommufd_ioas *ioas = NULL;
332 	struct iommufd_object *pt_obj;
333 	struct iommufd_device *idev;
334 	int rc;
335 
336 	if (cmd->__reserved)
337 		return -EOPNOTSUPP;
338 	if ((cmd->data_type == IOMMU_HWPT_DATA_NONE && cmd->data_len) ||
339 	    (cmd->data_type != IOMMU_HWPT_DATA_NONE && !cmd->data_len))
340 		return -EINVAL;
341 
342 	idev = iommufd_get_device(ucmd, cmd->dev_id);
343 	if (IS_ERR(idev))
344 		return PTR_ERR(idev);
345 
346 	pt_obj = iommufd_get_object(ucmd->ictx, cmd->pt_id, IOMMUFD_OBJ_ANY);
347 	if (IS_ERR(pt_obj)) {
348 		rc = -EINVAL;
349 		goto out_put_idev;
350 	}
351 
352 	if (pt_obj->type == IOMMUFD_OBJ_IOAS) {
353 		struct iommufd_hwpt_paging *hwpt_paging;
354 
355 		ioas = container_of(pt_obj, struct iommufd_ioas, obj);
356 		mutex_lock(&ioas->mutex);
357 		hwpt_paging = iommufd_hwpt_paging_alloc(
358 			ucmd->ictx, ioas, idev, cmd->flags, false,
359 			user_data.len ? &user_data : NULL);
360 		if (IS_ERR(hwpt_paging)) {
361 			rc = PTR_ERR(hwpt_paging);
362 			goto out_unlock;
363 		}
364 		hwpt = &hwpt_paging->common;
365 	} else if (pt_obj->type == IOMMUFD_OBJ_HWPT_PAGING) {
366 		struct iommufd_hwpt_nested *hwpt_nested;
367 
368 		hwpt_nested = iommufd_hwpt_nested_alloc(
369 			ucmd->ictx,
370 			container_of(pt_obj, struct iommufd_hwpt_paging,
371 				     common.obj),
372 			idev, cmd->flags, &user_data);
373 		if (IS_ERR(hwpt_nested)) {
374 			rc = PTR_ERR(hwpt_nested);
375 			goto out_unlock;
376 		}
377 		hwpt = &hwpt_nested->common;
378 	} else if (pt_obj->type == IOMMUFD_OBJ_VIOMMU) {
379 		struct iommufd_hwpt_nested *hwpt_nested;
380 		struct iommufd_viommu *viommu;
381 
382 		viommu = container_of(pt_obj, struct iommufd_viommu, obj);
383 		if (viommu->iommu_dev != __iommu_get_iommu_dev(idev->dev)) {
384 			rc = -EINVAL;
385 			goto out_unlock;
386 		}
387 		hwpt_nested = iommufd_viommu_alloc_hwpt_nested(
388 			viommu, cmd->flags, &user_data);
389 		if (IS_ERR(hwpt_nested)) {
390 			rc = PTR_ERR(hwpt_nested);
391 			goto out_unlock;
392 		}
393 		hwpt = &hwpt_nested->common;
394 	} else {
395 		rc = -EINVAL;
396 		goto out_put_pt;
397 	}
398 
399 	if (cmd->flags & IOMMU_HWPT_FAULT_ID_VALID) {
400 		struct iommufd_fault *fault;
401 
402 		fault = iommufd_get_fault(ucmd, cmd->fault_id);
403 		if (IS_ERR(fault)) {
404 			rc = PTR_ERR(fault);
405 			goto out_hwpt;
406 		}
407 		hwpt->fault = fault;
408 		hwpt->domain->iopf_handler = iommufd_fault_iopf_handler;
409 		hwpt->domain->fault_data = hwpt;
410 		refcount_inc(&fault->obj.users);
411 		iommufd_put_object(ucmd->ictx, &fault->obj);
412 	}
413 
414 	cmd->out_hwpt_id = hwpt->obj.id;
415 	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
416 	if (rc)
417 		goto out_hwpt;
418 	iommufd_object_finalize(ucmd->ictx, &hwpt->obj);
419 	goto out_unlock;
420 
421 out_hwpt:
422 	iommufd_object_abort_and_destroy(ucmd->ictx, &hwpt->obj);
423 out_unlock:
424 	if (ioas)
425 		mutex_unlock(&ioas->mutex);
426 out_put_pt:
427 	iommufd_put_object(ucmd->ictx, pt_obj);
428 out_put_idev:
429 	iommufd_put_object(ucmd->ictx, &idev->obj);
430 	return rc;
431 }
432 
433 int iommufd_hwpt_set_dirty_tracking(struct iommufd_ucmd *ucmd)
434 {
435 	struct iommu_hwpt_set_dirty_tracking *cmd = ucmd->cmd;
436 	struct iommufd_hwpt_paging *hwpt_paging;
437 	struct iommufd_ioas *ioas;
438 	int rc = -EOPNOTSUPP;
439 	bool enable;
440 
441 	if (cmd->flags & ~IOMMU_HWPT_DIRTY_TRACKING_ENABLE)
442 		return rc;
443 
444 	hwpt_paging = iommufd_get_hwpt_paging(ucmd, cmd->hwpt_id);
445 	if (IS_ERR(hwpt_paging))
446 		return PTR_ERR(hwpt_paging);
447 
448 	ioas = hwpt_paging->ioas;
449 	enable = cmd->flags & IOMMU_HWPT_DIRTY_TRACKING_ENABLE;
450 
451 	rc = iopt_set_dirty_tracking(&ioas->iopt, hwpt_paging->common.domain,
452 				     enable);
453 
454 	iommufd_put_object(ucmd->ictx, &hwpt_paging->common.obj);
455 	return rc;
456 }
457 
458 int iommufd_hwpt_get_dirty_bitmap(struct iommufd_ucmd *ucmd)
459 {
460 	struct iommu_hwpt_get_dirty_bitmap *cmd = ucmd->cmd;
461 	struct iommufd_hwpt_paging *hwpt_paging;
462 	struct iommufd_ioas *ioas;
463 	int rc = -EOPNOTSUPP;
464 
465 	if ((cmd->flags & ~(IOMMU_HWPT_GET_DIRTY_BITMAP_NO_CLEAR)) ||
466 	    cmd->__reserved)
467 		return -EOPNOTSUPP;
468 
469 	hwpt_paging = iommufd_get_hwpt_paging(ucmd, cmd->hwpt_id);
470 	if (IS_ERR(hwpt_paging))
471 		return PTR_ERR(hwpt_paging);
472 
473 	ioas = hwpt_paging->ioas;
474 	rc = iopt_read_and_clear_dirty_data(
475 		&ioas->iopt, hwpt_paging->common.domain, cmd->flags, cmd);
476 
477 	iommufd_put_object(ucmd->ictx, &hwpt_paging->common.obj);
478 	return rc;
479 }
480 
481 int iommufd_hwpt_invalidate(struct iommufd_ucmd *ucmd)
482 {
483 	struct iommu_hwpt_invalidate *cmd = ucmd->cmd;
484 	struct iommu_user_data_array data_array = {
485 		.type = cmd->data_type,
486 		.uptr = u64_to_user_ptr(cmd->data_uptr),
487 		.entry_len = cmd->entry_len,
488 		.entry_num = cmd->entry_num,
489 	};
490 	struct iommufd_object *pt_obj;
491 	u32 done_num = 0;
492 	int rc;
493 
494 	if (cmd->__reserved) {
495 		rc = -EOPNOTSUPP;
496 		goto out;
497 	}
498 
499 	if (cmd->entry_num && (!cmd->data_uptr || !cmd->entry_len)) {
500 		rc = -EINVAL;
501 		goto out;
502 	}
503 
504 	pt_obj = iommufd_get_object(ucmd->ictx, cmd->hwpt_id, IOMMUFD_OBJ_ANY);
505 	if (IS_ERR(pt_obj)) {
506 		rc = PTR_ERR(pt_obj);
507 		goto out;
508 	}
509 	if (pt_obj->type == IOMMUFD_OBJ_HWPT_NESTED) {
510 		struct iommufd_hw_pagetable *hwpt =
511 			container_of(pt_obj, struct iommufd_hw_pagetable, obj);
512 
513 		if (!hwpt->domain->ops ||
514 		    !hwpt->domain->ops->cache_invalidate_user) {
515 			rc = -EOPNOTSUPP;
516 			goto out_put_pt;
517 		}
518 		rc = hwpt->domain->ops->cache_invalidate_user(hwpt->domain,
519 							      &data_array);
520 	} else if (pt_obj->type == IOMMUFD_OBJ_VIOMMU) {
521 		struct iommufd_viommu *viommu =
522 			container_of(pt_obj, struct iommufd_viommu, obj);
523 
524 		if (!viommu->ops || !viommu->ops->cache_invalidate) {
525 			rc = -EOPNOTSUPP;
526 			goto out_put_pt;
527 		}
528 		rc = viommu->ops->cache_invalidate(viommu, &data_array);
529 	} else {
530 		rc = -EINVAL;
531 		goto out_put_pt;
532 	}
533 
534 	done_num = data_array.entry_num;
535 
536 out_put_pt:
537 	iommufd_put_object(ucmd->ictx, pt_obj);
538 out:
539 	cmd->entry_num = done_num;
540 	if (iommufd_ucmd_respond(ucmd, sizeof(*cmd)))
541 		return -EFAULT;
542 	return rc;
543 }
544