Lines Matching +full:elm +full:- +full:id
1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES
18 rc = iopt_unmap_all(&ioas->iopt, NULL); in iommufd_ioas_destroy()
19 WARN_ON(rc && rc != -ENOENT); in iommufd_ioas_destroy()
20 iopt_destroy_table(&ioas->iopt); in iommufd_ioas_destroy()
21 mutex_destroy(&ioas->mutex); in iommufd_ioas_destroy()
32 iopt_init_table(&ioas->iopt); in iommufd_ioas_alloc()
33 INIT_LIST_HEAD(&ioas->hwpt_list); in iommufd_ioas_alloc()
34 mutex_init(&ioas->mutex); in iommufd_ioas_alloc()
40 struct iommu_ioas_alloc *cmd = ucmd->cmd; in iommufd_ioas_alloc_ioctl()
44 if (cmd->flags) in iommufd_ioas_alloc_ioctl()
45 return -EOPNOTSUPP; in iommufd_ioas_alloc_ioctl()
47 ioas = iommufd_ioas_alloc(ucmd->ictx); in iommufd_ioas_alloc_ioctl()
51 cmd->out_ioas_id = ioas->obj.id; in iommufd_ioas_alloc_ioctl()
56 down_read(&ucmd->ictx->ioas_creation_lock); in iommufd_ioas_alloc_ioctl()
57 iommufd_object_finalize(ucmd->ictx, &ioas->obj); in iommufd_ioas_alloc_ioctl()
58 up_read(&ucmd->ictx->ioas_creation_lock); in iommufd_ioas_alloc_ioctl()
62 iommufd_object_abort_and_destroy(ucmd->ictx, &ioas->obj); in iommufd_ioas_alloc_ioctl()
69 struct iommu_ioas_iova_ranges *cmd = ucmd->cmd; in iommufd_ioas_iova_ranges()
75 if (cmd->__reserved) in iommufd_ioas_iova_ranges()
76 return -EOPNOTSUPP; in iommufd_ioas_iova_ranges()
78 ioas = iommufd_get_ioas(ucmd->ictx, cmd->ioas_id); in iommufd_ioas_iova_ranges()
82 down_read(&ioas->iopt.iova_rwsem); in iommufd_ioas_iova_ranges()
83 max_iovas = cmd->num_iovas; in iommufd_ioas_iova_ranges()
84 ranges = u64_to_user_ptr(cmd->allowed_iovas); in iommufd_ioas_iova_ranges()
85 cmd->num_iovas = 0; in iommufd_ioas_iova_ranges()
86 cmd->out_iova_alignment = ioas->iopt.iova_alignment; in iommufd_ioas_iova_ranges()
87 interval_tree_for_each_span(&span, &ioas->iopt.reserved_itree, 0, in iommufd_ioas_iova_ranges()
91 if (cmd->num_iovas < max_iovas) { in iommufd_ioas_iova_ranges()
92 struct iommu_iova_range elm = { in iommufd_ioas_iova_ranges() local
97 if (copy_to_user(&ranges[cmd->num_iovas], &elm, in iommufd_ioas_iova_ranges()
98 sizeof(elm))) { in iommufd_ioas_iova_ranges()
99 rc = -EFAULT; in iommufd_ioas_iova_ranges()
103 cmd->num_iovas++; in iommufd_ioas_iova_ranges()
108 if (cmd->num_iovas > max_iovas) in iommufd_ioas_iova_ranges()
109 rc = -EMSGSIZE; in iommufd_ioas_iova_ranges()
111 up_read(&ioas->iopt.iova_rwsem); in iommufd_ioas_iova_ranges()
112 iommufd_put_object(ucmd->ictx, &ioas->obj); in iommufd_ioas_iova_ranges()
127 return -EFAULT; in iommufd_ioas_load_iovas()
130 return -EINVAL; in iommufd_ioas_load_iovas()
133 return -EINVAL; in iommufd_ioas_load_iovas()
137 return -ENOMEM; in iommufd_ioas_load_iovas()
138 allowed->node.start = range.start; in iommufd_ioas_load_iovas()
139 allowed->node.last = range.last; in iommufd_ioas_load_iovas()
141 interval_tree_insert(&allowed->node, itree); in iommufd_ioas_load_iovas()
148 struct iommu_ioas_allow_iovas *cmd = ucmd->cmd; in iommufd_ioas_allow_iovas()
155 if (cmd->__reserved) in iommufd_ioas_allow_iovas()
156 return -EOPNOTSUPP; in iommufd_ioas_allow_iovas()
158 ioas = iommufd_get_ioas(ucmd->ictx, cmd->ioas_id); in iommufd_ioas_allow_iovas()
161 iopt = &ioas->iopt; in iommufd_ioas_allow_iovas()
164 u64_to_user_ptr(cmd->allowed_iovas), in iommufd_ioas_allow_iovas()
165 cmd->num_iovas); in iommufd_ioas_allow_iovas()
182 iommufd_put_object(ucmd->ictx, &ioas->obj); in iommufd_ioas_allow_iovas()
206 struct iommu_ioas_map_file *cmd = ucmd->cmd; in iommufd_ioas_map_file()
207 unsigned long iova = cmd->iova; in iommufd_ioas_map_file()
212 if (cmd->flags & in iommufd_ioas_map_file()
215 return -EOPNOTSUPP; in iommufd_ioas_map_file()
217 if (cmd->iova >= ULONG_MAX || cmd->length >= ULONG_MAX) in iommufd_ioas_map_file()
218 return -EOVERFLOW; in iommufd_ioas_map_file()
220 if (!(cmd->flags & in iommufd_ioas_map_file()
222 return -EINVAL; in iommufd_ioas_map_file()
224 ioas = iommufd_get_ioas(ucmd->ictx, cmd->ioas_id); in iommufd_ioas_map_file()
228 if (!(cmd->flags & IOMMU_IOAS_MAP_FIXED_IOVA)) in iommufd_ioas_map_file()
231 rc = iopt_map_file_pages(ucmd->ictx, &ioas->iopt, &iova, cmd->fd, in iommufd_ioas_map_file()
232 cmd->start, cmd->length, in iommufd_ioas_map_file()
233 conv_iommu_prot(cmd->flags), flags); in iommufd_ioas_map_file()
237 cmd->iova = iova; in iommufd_ioas_map_file()
240 iommufd_put_object(ucmd->ictx, &ioas->obj); in iommufd_ioas_map_file()
246 struct iommu_ioas_map *cmd = ucmd->cmd; in iommufd_ioas_map()
247 unsigned long iova = cmd->iova; in iommufd_ioas_map()
252 if ((cmd->flags & in iommufd_ioas_map()
255 cmd->__reserved) in iommufd_ioas_map()
256 return -EOPNOTSUPP; in iommufd_ioas_map()
257 if (cmd->iova >= ULONG_MAX || cmd->length >= ULONG_MAX) in iommufd_ioas_map()
258 return -EOVERFLOW; in iommufd_ioas_map()
260 if (!(cmd->flags & in iommufd_ioas_map()
262 return -EINVAL; in iommufd_ioas_map()
264 ioas = iommufd_get_ioas(ucmd->ictx, cmd->ioas_id); in iommufd_ioas_map()
268 if (!(cmd->flags & IOMMU_IOAS_MAP_FIXED_IOVA)) in iommufd_ioas_map()
270 rc = iopt_map_user_pages(ucmd->ictx, &ioas->iopt, &iova, in iommufd_ioas_map()
271 u64_to_user_ptr(cmd->user_va), cmd->length, in iommufd_ioas_map()
272 conv_iommu_prot(cmd->flags), flags); in iommufd_ioas_map()
276 cmd->iova = iova; in iommufd_ioas_map()
279 iommufd_put_object(ucmd->ictx, &ioas->obj); in iommufd_ioas_map()
285 struct iommu_ioas_copy *cmd = ucmd->cmd; in iommufd_ioas_copy()
293 iommufd_test_syz_conv_iova_id(ucmd, cmd->src_ioas_id, &cmd->src_iova, in iommufd_ioas_copy()
294 &cmd->flags); in iommufd_ioas_copy()
296 if ((cmd->flags & in iommufd_ioas_copy()
299 return -EOPNOTSUPP; in iommufd_ioas_copy()
300 if (cmd->length >= ULONG_MAX || cmd->src_iova >= ULONG_MAX || in iommufd_ioas_copy()
301 cmd->dst_iova >= ULONG_MAX) in iommufd_ioas_copy()
302 return -EOVERFLOW; in iommufd_ioas_copy()
304 if (!(cmd->flags & in iommufd_ioas_copy()
306 return -EINVAL; in iommufd_ioas_copy()
308 src_ioas = iommufd_get_ioas(ucmd->ictx, cmd->src_ioas_id); in iommufd_ioas_copy()
311 rc = iopt_get_pages(&src_ioas->iopt, cmd->src_iova, cmd->length, in iommufd_ioas_copy()
313 iommufd_put_object(ucmd->ictx, &src_ioas->obj); in iommufd_ioas_copy()
317 dst_ioas = iommufd_get_ioas(ucmd->ictx, cmd->dst_ioas_id); in iommufd_ioas_copy()
323 if (!(cmd->flags & IOMMU_IOAS_MAP_FIXED_IOVA)) in iommufd_ioas_copy()
325 iova = cmd->dst_iova; in iommufd_ioas_copy()
326 rc = iopt_map_pages(&dst_ioas->iopt, &pages_list, cmd->length, &iova, in iommufd_ioas_copy()
327 conv_iommu_prot(cmd->flags), flags); in iommufd_ioas_copy()
331 cmd->dst_iova = iova; in iommufd_ioas_copy()
334 iommufd_put_object(ucmd->ictx, &dst_ioas->obj); in iommufd_ioas_copy()
342 struct iommu_ioas_unmap *cmd = ucmd->cmd; in iommufd_ioas_unmap()
347 ioas = iommufd_get_ioas(ucmd->ictx, cmd->ioas_id); in iommufd_ioas_unmap()
351 if (cmd->iova == 0 && cmd->length == U64_MAX) { in iommufd_ioas_unmap()
352 rc = iopt_unmap_all(&ioas->iopt, &unmapped); in iommufd_ioas_unmap()
356 if (cmd->iova >= ULONG_MAX || cmd->length >= ULONG_MAX) { in iommufd_ioas_unmap()
357 rc = -EOVERFLOW; in iommufd_ioas_unmap()
360 rc = iopt_unmap_iova(&ioas->iopt, cmd->iova, cmd->length, in iommufd_ioas_unmap()
365 rc = -ENOENT; in iommufd_ioas_unmap()
370 cmd->length = unmapped; in iommufd_ioas_unmap()
374 iommufd_put_object(ucmd->ictx, &ioas->obj); in iommufd_ioas_unmap()
385 up_write(&ioas->iopt.iova_rwsem); in iommufd_release_all_iova_rwsem()
386 refcount_dec(&ioas->obj.users); in iommufd_release_all_iova_rwsem()
388 up_write(&ictx->ioas_creation_lock); in iommufd_release_all_iova_rwsem()
401 * pages->source_mm, which is a performance path for mdev, we just in iommufd_take_all_iova_rwsem()
403 * pages->source_*. Due to copies we can't know which IOAS could read in iommufd_take_all_iova_rwsem()
405 * locks are nested and they are uniformly taken in ID order. in iommufd_take_all_iova_rwsem()
411 down_write(&ictx->ioas_creation_lock); in iommufd_take_all_iova_rwsem()
412 xa_lock(&ictx->objects); in iommufd_take_all_iova_rwsem()
413 xa_for_each(&ictx->objects, index, obj) { in iommufd_take_all_iova_rwsem()
416 if (!obj || obj->type != IOMMUFD_OBJ_IOAS) in iommufd_take_all_iova_rwsem()
419 if (!refcount_inc_not_zero(&obj->users)) in iommufd_take_all_iova_rwsem()
422 xa_unlock(&ictx->objects); in iommufd_take_all_iova_rwsem()
425 down_write_nest_lock(&ioas->iopt.iova_rwsem, in iommufd_take_all_iova_rwsem()
426 &ictx->ioas_creation_lock); in iommufd_take_all_iova_rwsem()
434 xa_lock(&ictx->objects); in iommufd_take_all_iova_rwsem()
436 xa_unlock(&ictx->objects); in iommufd_take_all_iova_rwsem()
442 switch (pages->account_mode) { in need_charge_update()
446 return pages->source_mm != current->mm; in need_charge_update()
450 * in mm->pinned_vm. in need_charge_update()
452 return (pages->source_user != current_user()) || in need_charge_update()
453 (pages->source_mm != current->mm); in need_charge_update()
461 .source_mm = current->mm, in charge_current()
462 .source_task = current->group_leader, in charge_current()
483 account_mode--; in charge_current()
495 struct task_struct *old_task = pages->source_task; in change_mm()
496 struct user_struct *old_user = pages->source_user; in change_mm()
497 struct mm_struct *old_mm = pages->source_mm; in change_mm()
499 pages->source_mm = current->mm; in change_mm()
500 mmgrab(pages->source_mm); in change_mm()
503 pages->source_task = current->group_leader; in change_mm()
504 get_task_struct(pages->source_task); in change_mm()
507 pages->source_user = get_uid(current_user()); in change_mm()
513 for (_area = iopt_area_iter_first(&_ioas->iopt, 0, ULONG_MAX); \
519 struct iommu_ioas_change_process *cmd = ucmd->cmd; in iommufd_ioas_change_process()
520 struct iommufd_ctx *ictx = ucmd->ictx; in iommufd_ioas_change_process()
529 if (cmd->__reserved) in iommufd_ioas_change_process()
530 return -EOPNOTSUPP; in iommufd_ioas_change_process()
538 if (area->pages->type != IOPT_ADDRESS_FILE) { in iommufd_ioas_change_process()
539 rc = -EINVAL; in iommufd_ioas_change_process()
551 pages = area->pages; in iommufd_ioas_change_process()
554 all_npinned[pages->account_mode] += pages->last_npinned; in iommufd_ioas_change_process()
555 pages->last_npinned = 0; in iommufd_ioas_change_process()
564 area->pages->last_npinned = area->pages->npinned; in iommufd_ioas_change_process()
569 pages = area->pages; in iommufd_ioas_change_process()
573 int r = iopt_pages_update_pinned(pages, pages->npinned, in iommufd_ioas_change_process()
590 if (cmd->object_id) in iommufd_option_rlimit_mode()
591 return -EOPNOTSUPP; in iommufd_option_rlimit_mode()
593 if (cmd->op == IOMMU_OPTION_OP_GET) { in iommufd_option_rlimit_mode()
594 cmd->val64 = ictx->account_mode == IOPT_PAGES_ACCOUNT_MM; in iommufd_option_rlimit_mode()
597 if (cmd->op == IOMMU_OPTION_OP_SET) { in iommufd_option_rlimit_mode()
601 return -EPERM; in iommufd_option_rlimit_mode()
603 xa_lock(&ictx->objects); in iommufd_option_rlimit_mode()
604 if (!xa_empty(&ictx->objects)) { in iommufd_option_rlimit_mode()
605 rc = -EBUSY; in iommufd_option_rlimit_mode()
607 if (cmd->val64 == 0) in iommufd_option_rlimit_mode()
608 ictx->account_mode = IOPT_PAGES_ACCOUNT_USER; in iommufd_option_rlimit_mode()
609 else if (cmd->val64 == 1) in iommufd_option_rlimit_mode()
610 ictx->account_mode = IOPT_PAGES_ACCOUNT_MM; in iommufd_option_rlimit_mode()
612 rc = -EINVAL; in iommufd_option_rlimit_mode()
614 xa_unlock(&ictx->objects); in iommufd_option_rlimit_mode()
618 return -EOPNOTSUPP; in iommufd_option_rlimit_mode()
624 if (cmd->op == IOMMU_OPTION_OP_GET) { in iommufd_ioas_option_huge_pages()
625 cmd->val64 = !ioas->iopt.disable_large_pages; in iommufd_ioas_option_huge_pages()
628 if (cmd->op == IOMMU_OPTION_OP_SET) { in iommufd_ioas_option_huge_pages()
629 if (cmd->val64 == 0) in iommufd_ioas_option_huge_pages()
630 return iopt_disable_large_pages(&ioas->iopt); in iommufd_ioas_option_huge_pages()
631 if (cmd->val64 == 1) { in iommufd_ioas_option_huge_pages()
632 iopt_enable_large_pages(&ioas->iopt); in iommufd_ioas_option_huge_pages()
635 return -EINVAL; in iommufd_ioas_option_huge_pages()
637 return -EOPNOTSUPP; in iommufd_ioas_option_huge_pages()
642 struct iommu_option *cmd = ucmd->cmd; in iommufd_ioas_option()
646 if (cmd->__reserved) in iommufd_ioas_option()
647 return -EOPNOTSUPP; in iommufd_ioas_option()
649 ioas = iommufd_get_ioas(ucmd->ictx, cmd->object_id); in iommufd_ioas_option()
653 switch (cmd->option_id) { in iommufd_ioas_option()
658 rc = -EOPNOTSUPP; in iommufd_ioas_option()
661 iommufd_put_object(ucmd->ictx, &ioas->obj); in iommufd_ioas_option()