xref: /linux/drivers/iommu/iommufd/ioas.c (revision 895b4fae931a256f6c7c6c68f77a8e337980b5a1) 
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES
4  */
5 #include <linux/interval_tree.h>
6 #include <linux/iommufd.h>
7 #include <linux/iommu.h>
8 #include <uapi/linux/iommufd.h>
9 
10 #include "io_pagetable.h"
11 
12 void iommufd_ioas_destroy(struct iommufd_object *obj)
13 {
14 	struct iommufd_ioas *ioas = container_of(obj, struct iommufd_ioas, obj);
15 	int rc;
16 
17 	rc = iopt_unmap_all(&ioas->iopt, NULL);
18 	WARN_ON(rc && rc != -ENOENT);
19 	iopt_destroy_table(&ioas->iopt);
20 	mutex_destroy(&ioas->mutex);
21 }
22 
23 struct iommufd_ioas *iommufd_ioas_alloc(struct iommufd_ctx *ictx)
24 {
25 	struct iommufd_ioas *ioas;
26 
27 	ioas = iommufd_object_alloc(ictx, ioas, IOMMUFD_OBJ_IOAS);
28 	if (IS_ERR(ioas))
29 		return ioas;
30 
31 	iopt_init_table(&ioas->iopt);
32 	INIT_LIST_HEAD(&ioas->hwpt_list);
33 	mutex_init(&ioas->mutex);
34 	return ioas;
35 }
36 
37 int iommufd_ioas_alloc_ioctl(struct iommufd_ucmd *ucmd)
38 {
39 	struct iommu_ioas_alloc *cmd = ucmd->cmd;
40 	struct iommufd_ioas *ioas;
41 	int rc;
42 
43 	if (cmd->flags)
44 		return -EOPNOTSUPP;
45 
46 	ioas = iommufd_ioas_alloc(ucmd->ictx);
47 	if (IS_ERR(ioas))
48 		return PTR_ERR(ioas);
49 
50 	cmd->out_ioas_id = ioas->obj.id;
51 	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
52 	if (rc)
53 		goto out_table;
54 	iommufd_object_finalize(ucmd->ictx, &ioas->obj);
55 	return 0;
56 
57 out_table:
58 	iommufd_object_abort_and_destroy(ucmd->ictx, &ioas->obj);
59 	return rc;
60 }
61 
62 int iommufd_ioas_iova_ranges(struct iommufd_ucmd *ucmd)
63 {
64 	struct iommu_iova_range __user *ranges;
65 	struct iommu_ioas_iova_ranges *cmd = ucmd->cmd;
66 	struct iommufd_ioas *ioas;
67 	struct interval_tree_span_iter span;
68 	u32 max_iovas;
69 	int rc;
70 
71 	if (cmd->__reserved)
72 		return -EOPNOTSUPP;
73 
74 	ioas = iommufd_get_ioas(ucmd->ictx, cmd->ioas_id);
75 	if (IS_ERR(ioas))
76 		return PTR_ERR(ioas);
77 
78 	down_read(&ioas->iopt.iova_rwsem);
79 	max_iovas = cmd->num_iovas;
80 	ranges = u64_to_user_ptr(cmd->allowed_iovas);
81 	cmd->num_iovas = 0;
82 	cmd->out_iova_alignment = ioas->iopt.iova_alignment;
83 	interval_tree_for_each_span(&span, &ioas->iopt.reserved_itree, 0,
84 				    ULONG_MAX) {
85 		if (!span.is_hole)
86 			continue;
87 		if (cmd->num_iovas < max_iovas) {
88 			struct iommu_iova_range elm = {
89 				.start = span.start_hole,
90 				.last = span.last_hole,
91 			};
92 
93 			if (copy_to_user(&ranges[cmd->num_iovas], &elm,
94 					 sizeof(elm))) {
95 				rc = -EFAULT;
96 				goto out_put;
97 			}
98 		}
99 		cmd->num_iovas++;
100 	}
101 	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
102 	if (rc)
103 		goto out_put;
104 	if (cmd->num_iovas > max_iovas)
105 		rc = -EMSGSIZE;
106 out_put:
107 	up_read(&ioas->iopt.iova_rwsem);
108 	iommufd_put_object(ucmd->ictx, &ioas->obj);
109 	return rc;
110 }
111 
112 static int iommufd_ioas_load_iovas(struct rb_root_cached *itree,
113 				   struct iommu_iova_range __user *ranges,
114 				   u32 num)
115 {
116 	u32 i;
117 
118 	for (i = 0; i != num; i++) {
119 		struct iommu_iova_range range;
120 		struct iopt_allowed *allowed;
121 
122 		if (copy_from_user(&range, ranges + i, sizeof(range)))
123 			return -EFAULT;
124 
125 		if (range.start >= range.last)
126 			return -EINVAL;
127 
128 		if (interval_tree_iter_first(itree, range.start, range.last))
129 			return -EINVAL;
130 
131 		allowed = kzalloc(sizeof(*allowed), GFP_KERNEL_ACCOUNT);
132 		if (!allowed)
133 			return -ENOMEM;
134 		allowed->node.start = range.start;
135 		allowed->node.last = range.last;
136 
137 		interval_tree_insert(&allowed->node, itree);
138 	}
139 	return 0;
140 }
141 
142 int iommufd_ioas_allow_iovas(struct iommufd_ucmd *ucmd)
143 {
144 	struct iommu_ioas_allow_iovas *cmd = ucmd->cmd;
145 	struct rb_root_cached allowed_iova = RB_ROOT_CACHED;
146 	struct interval_tree_node *node;
147 	struct iommufd_ioas *ioas;
148 	struct io_pagetable *iopt;
149 	int rc = 0;
150 
151 	if (cmd->__reserved)
152 		return -EOPNOTSUPP;
153 
154 	ioas = iommufd_get_ioas(ucmd->ictx, cmd->ioas_id);
155 	if (IS_ERR(ioas))
156 		return PTR_ERR(ioas);
157 	iopt = &ioas->iopt;
158 
159 	rc = iommufd_ioas_load_iovas(&allowed_iova,
160 				     u64_to_user_ptr(cmd->allowed_iovas),
161 				     cmd->num_iovas);
162 	if (rc)
163 		goto out_free;
164 
165 	/*
166 	 * We want the allowed tree update to be atomic, so we have to keep the
167 	 * original nodes around, and keep track of the new nodes as we allocate
168 	 * memory for them. The simplest solution is to have a new/old tree and
169 	 * then swap new for old. On success we free the old tree, on failure we
170 	 * free the new tree.
171 	 */
172 	rc = iopt_set_allow_iova(iopt, &allowed_iova);
173 out_free:
174 	while ((node = interval_tree_iter_first(&allowed_iova, 0, ULONG_MAX))) {
175 		interval_tree_remove(node, &allowed_iova);
176 		kfree(container_of(node, struct iopt_allowed, node));
177 	}
178 	iommufd_put_object(ucmd->ictx, &ioas->obj);
179 	return rc;
180 }
181 
182 static int conv_iommu_prot(u32 map_flags)
183 {
184 	/*
185 	 * We provide no manual cache coherency ioctls to userspace and most
186 	 * architectures make the CPU ops for cache flushing privileged.
187 	 * Therefore we require the underlying IOMMU to support CPU coherent
188 	 * operation. Support for IOMMU_CACHE is enforced by the
189 	 * IOMMU_CAP_CACHE_COHERENCY test during bind.
190 	 */
191 	int iommu_prot = IOMMU_CACHE;
192 
193 	if (map_flags & IOMMU_IOAS_MAP_WRITEABLE)
194 		iommu_prot |= IOMMU_WRITE;
195 	if (map_flags & IOMMU_IOAS_MAP_READABLE)
196 		iommu_prot |= IOMMU_READ;
197 	return iommu_prot;
198 }
199 
200 int iommufd_ioas_map(struct iommufd_ucmd *ucmd)
201 {
202 	struct iommu_ioas_map *cmd = ucmd->cmd;
203 	unsigned long iova = cmd->iova;
204 	struct iommufd_ioas *ioas;
205 	unsigned int flags = 0;
206 	int rc;
207 
208 	if ((cmd->flags &
209 	     ~(IOMMU_IOAS_MAP_FIXED_IOVA | IOMMU_IOAS_MAP_WRITEABLE |
210 	       IOMMU_IOAS_MAP_READABLE)) ||
211 	    cmd->__reserved)
212 		return -EOPNOTSUPP;
213 	if (cmd->iova >= ULONG_MAX || cmd->length >= ULONG_MAX)
214 		return -EOVERFLOW;
215 
216 	if (!(cmd->flags &
217 	      (IOMMU_IOAS_MAP_WRITEABLE | IOMMU_IOAS_MAP_READABLE)))
218 		return -EINVAL;
219 
220 	ioas = iommufd_get_ioas(ucmd->ictx, cmd->ioas_id);
221 	if (IS_ERR(ioas))
222 		return PTR_ERR(ioas);
223 
224 	if (!(cmd->flags & IOMMU_IOAS_MAP_FIXED_IOVA))
225 		flags = IOPT_ALLOC_IOVA;
226 	rc = iopt_map_user_pages(ucmd->ictx, &ioas->iopt, &iova,
227 				 u64_to_user_ptr(cmd->user_va), cmd->length,
228 				 conv_iommu_prot(cmd->flags), flags);
229 	if (rc)
230 		goto out_put;
231 
232 	cmd->iova = iova;
233 	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
234 out_put:
235 	iommufd_put_object(ucmd->ictx, &ioas->obj);
236 	return rc;
237 }
238 
239 int iommufd_ioas_copy(struct iommufd_ucmd *ucmd)
240 {
241 	struct iommu_ioas_copy *cmd = ucmd->cmd;
242 	struct iommufd_ioas *src_ioas;
243 	struct iommufd_ioas *dst_ioas;
244 	unsigned int flags = 0;
245 	LIST_HEAD(pages_list);
246 	unsigned long iova;
247 	int rc;
248 
249 	iommufd_test_syz_conv_iova_id(ucmd, cmd->src_ioas_id, &cmd->src_iova,
250 				      &cmd->flags);
251 
252 	if ((cmd->flags &
253 	     ~(IOMMU_IOAS_MAP_FIXED_IOVA | IOMMU_IOAS_MAP_WRITEABLE |
254 	       IOMMU_IOAS_MAP_READABLE)))
255 		return -EOPNOTSUPP;
256 	if (cmd->length >= ULONG_MAX || cmd->src_iova >= ULONG_MAX ||
257 	    cmd->dst_iova >= ULONG_MAX)
258 		return -EOVERFLOW;
259 
260 	if (!(cmd->flags &
261 	      (IOMMU_IOAS_MAP_WRITEABLE | IOMMU_IOAS_MAP_READABLE)))
262 		return -EINVAL;
263 
264 	src_ioas = iommufd_get_ioas(ucmd->ictx, cmd->src_ioas_id);
265 	if (IS_ERR(src_ioas))
266 		return PTR_ERR(src_ioas);
267 	rc = iopt_get_pages(&src_ioas->iopt, cmd->src_iova, cmd->length,
268 			    &pages_list);
269 	iommufd_put_object(ucmd->ictx, &src_ioas->obj);
270 	if (rc)
271 		return rc;
272 
273 	dst_ioas = iommufd_get_ioas(ucmd->ictx, cmd->dst_ioas_id);
274 	if (IS_ERR(dst_ioas)) {
275 		rc = PTR_ERR(dst_ioas);
276 		goto out_pages;
277 	}
278 
279 	if (!(cmd->flags & IOMMU_IOAS_MAP_FIXED_IOVA))
280 		flags = IOPT_ALLOC_IOVA;
281 	iova = cmd->dst_iova;
282 	rc = iopt_map_pages(&dst_ioas->iopt, &pages_list, cmd->length, &iova,
283 			    conv_iommu_prot(cmd->flags), flags);
284 	if (rc)
285 		goto out_put_dst;
286 
287 	cmd->dst_iova = iova;
288 	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
289 out_put_dst:
290 	iommufd_put_object(ucmd->ictx, &dst_ioas->obj);
291 out_pages:
292 	iopt_free_pages_list(&pages_list);
293 	return rc;
294 }
295 
296 int iommufd_ioas_unmap(struct iommufd_ucmd *ucmd)
297 {
298 	struct iommu_ioas_unmap *cmd = ucmd->cmd;
299 	struct iommufd_ioas *ioas;
300 	unsigned long unmapped = 0;
301 	int rc;
302 
303 	ioas = iommufd_get_ioas(ucmd->ictx, cmd->ioas_id);
304 	if (IS_ERR(ioas))
305 		return PTR_ERR(ioas);
306 
307 	if (cmd->iova == 0 && cmd->length == U64_MAX) {
308 		rc = iopt_unmap_all(&ioas->iopt, &unmapped);
309 		if (rc)
310 			goto out_put;
311 	} else {
312 		if (cmd->iova >= ULONG_MAX || cmd->length >= ULONG_MAX) {
313 			rc = -EOVERFLOW;
314 			goto out_put;
315 		}
316 		rc = iopt_unmap_iova(&ioas->iopt, cmd->iova, cmd->length,
317 				     &unmapped);
318 		if (rc)
319 			goto out_put;
320 	}
321 
322 	cmd->length = unmapped;
323 	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
324 
325 out_put:
326 	iommufd_put_object(ucmd->ictx, &ioas->obj);
327 	return rc;
328 }
329 
330 int iommufd_option_rlimit_mode(struct iommu_option *cmd,
331 			       struct iommufd_ctx *ictx)
332 {
333 	if (cmd->object_id)
334 		return -EOPNOTSUPP;
335 
336 	if (cmd->op == IOMMU_OPTION_OP_GET) {
337 		cmd->val64 = ictx->account_mode == IOPT_PAGES_ACCOUNT_MM;
338 		return 0;
339 	}
340 	if (cmd->op == IOMMU_OPTION_OP_SET) {
341 		int rc = 0;
342 
343 		if (!capable(CAP_SYS_RESOURCE))
344 			return -EPERM;
345 
346 		xa_lock(&ictx->objects);
347 		if (!xa_empty(&ictx->objects)) {
348 			rc = -EBUSY;
349 		} else {
350 			if (cmd->val64 == 0)
351 				ictx->account_mode = IOPT_PAGES_ACCOUNT_USER;
352 			else if (cmd->val64 == 1)
353 				ictx->account_mode = IOPT_PAGES_ACCOUNT_MM;
354 			else
355 				rc = -EINVAL;
356 		}
357 		xa_unlock(&ictx->objects);
358 
359 		return rc;
360 	}
361 	return -EOPNOTSUPP;
362 }
363 
364 static int iommufd_ioas_option_huge_pages(struct iommu_option *cmd,
365 					  struct iommufd_ioas *ioas)
366 {
367 	if (cmd->op == IOMMU_OPTION_OP_GET) {
368 		cmd->val64 = !ioas->iopt.disable_large_pages;
369 		return 0;
370 	}
371 	if (cmd->op == IOMMU_OPTION_OP_SET) {
372 		if (cmd->val64 == 0)
373 			return iopt_disable_large_pages(&ioas->iopt);
374 		if (cmd->val64 == 1) {
375 			iopt_enable_large_pages(&ioas->iopt);
376 			return 0;
377 		}
378 		return -EINVAL;
379 	}
380 	return -EOPNOTSUPP;
381 }
382 
383 int iommufd_ioas_option(struct iommufd_ucmd *ucmd)
384 {
385 	struct iommu_option *cmd = ucmd->cmd;
386 	struct iommufd_ioas *ioas;
387 	int rc = 0;
388 
389 	if (cmd->__reserved)
390 		return -EOPNOTSUPP;
391 
392 	ioas = iommufd_get_ioas(ucmd->ictx, cmd->object_id);
393 	if (IS_ERR(ioas))
394 		return PTR_ERR(ioas);
395 
396 	switch (cmd->option_id) {
397 	case IOMMU_OPTION_HUGE_PAGES:
398 		rc = iommufd_ioas_option_huge_pages(cmd, ioas);
399 		break;
400 	default:
401 		rc = -EOPNOTSUPP;
402 	}
403 
404 	iommufd_put_object(ucmd->ictx, &ioas->obj);
405 	return rc;
406 }
407