Lines Matching +full:iommu +full:- +full:ctx
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
60 #include <dev/iommu/iommu.h>
61 #include <dev/iommu/iommu_gas.h>
62 #include <dev/iommu/iommu_msi.h>
66 #include <machine/iommu.h>
67 #include <dev/iommu/busdma_iommu.h>
100 SLIST_INIT(&res->pgtbl_free); in iommu_gas_alloc_entry()
102 res->domain = domain; in iommu_gas_alloc_entry()
103 atomic_add_int(&domain->entries_cnt, 1); in iommu_gas_alloc_entry()
115 n = vm_page_free_pages_toq(&entry->pgtbl_free, false); in iommu_gas_free_entry()
119 domain = entry->domain; in iommu_gas_free_entry()
121 atomic_subtract_int(&domain->entries_cnt, 1); in iommu_gas_free_entry()
130 KASSERT(a->start <= a->end, ("inverted entry %p (%jx, %jx)", in iommu_gas_cmp_entries()
131 a, (uintmax_t)a->start, (uintmax_t)a->end)); in iommu_gas_cmp_entries()
132 KASSERT(b->start <= b->end, ("inverted entry %p (%jx, %jx)", in iommu_gas_cmp_entries()
133 b, (uintmax_t)b->start, (uintmax_t)b->end)); in iommu_gas_cmp_entries()
134 KASSERT(((a->flags | b->flags) & IOMMU_MAP_ENTRY_FAKE) != 0 || in iommu_gas_cmp_entries()
135 a->end <= b->start || b->end <= a->start || in iommu_gas_cmp_entries()
136 a->end == a->start || b->end == b->start, in iommu_gas_cmp_entries()
139 a, (uintmax_t)a->start, (uintmax_t)a->end, a->flags, in iommu_gas_cmp_entries()
140 b, (uintmax_t)b->start, (uintmax_t)b->end, b->flags, in iommu_gas_cmp_entries()
141 a->domain, b->domain)); in iommu_gas_cmp_entries()
143 if (a->end < b->end) in iommu_gas_cmp_entries()
144 return (-1); in iommu_gas_cmp_entries()
145 else if (b->end < a->end) in iommu_gas_cmp_entries()
161 bound = entry->start; in iommu_gas_augment_entry()
163 free_down = MAX(child->free_down, bound - child->last); in iommu_gas_augment_entry()
164 bound = child->first; in iommu_gas_augment_entry()
166 delta = bound - entry->first; in iommu_gas_augment_entry()
167 entry->first = bound; in iommu_gas_augment_entry()
168 bound = entry->end; in iommu_gas_augment_entry()
170 free_down = MAX(free_down, child->free_down); in iommu_gas_augment_entry()
171 free_down = MAX(free_down, child->first - bound); in iommu_gas_augment_entry()
172 bound = child->last; in iommu_gas_augment_entry()
174 delta += entry->last - bound; in iommu_gas_augment_entry()
176 delta = entry->free_down - free_down; in iommu_gas_augment_entry()
177 entry->last = bound; in iommu_gas_augment_entry()
178 entry->free_down = free_down; in iommu_gas_augment_entry()
181 * Return true either if the value of last-first changed, in iommu_gas_augment_entry()
197 RB_FOREACH(entry, iommu_gas_entries_tree, &domain->rb_root) { in iommu_gas_check_free()
198 KASSERT(domain == entry->domain, in iommu_gas_check_free()
199 ("mismatched free domain %p entry %p entry->domain %p", in iommu_gas_check_free()
200 domain, entry, entry->domain)); in iommu_gas_check_free()
205 v = MAX(v, l->free_down); in iommu_gas_check_free()
206 v = MAX(v, entry->start - l->last); in iommu_gas_check_free()
209 v = MAX(v, r->free_down); in iommu_gas_check_free()
210 v = MAX(v, r->first - entry->end); in iommu_gas_check_free()
212 MPASS(entry->free_down == v); in iommu_gas_check_free()
222 /* Removing entry may open a new free gap before domain->start_gap. */ in iommu_gas_rb_remove()
223 if (entry->end <= domain->start_gap->end) { in iommu_gas_rb_remove()
230 domain->start_gap = nbr; in iommu_gas_rb_remove()
232 RB_REMOVE(iommu_gas_entries_tree, &domain->rb_root, entry); in iommu_gas_rb_remove()
236 iommu_get_ctx_domain(struct iommu_ctx *ctx) in iommu_get_ctx_domain() argument
239 return (ctx->domain); in iommu_get_ctx_domain()
251 KASSERT(domain->entries_cnt == 2, ("dirty domain %p", domain)); in iommu_gas_init_domain()
252 KASSERT(RB_EMPTY(&domain->rb_root), in iommu_gas_init_domain()
253 ("non-empty entries %p", domain)); in iommu_gas_init_domain()
255 end->start = domain->end; in iommu_gas_init_domain()
256 end->end = domain->end; in iommu_gas_init_domain()
257 end->flags = IOMMU_MAP_ENTRY_PLACE | IOMMU_MAP_ENTRY_UNMAPPED; in iommu_gas_init_domain()
258 RB_INSERT(iommu_gas_entries_tree, &domain->rb_root, end); in iommu_gas_init_domain()
260 begin->start = 0; in iommu_gas_init_domain()
261 begin->end = 0; in iommu_gas_init_domain()
262 begin->flags = IOMMU_MAP_ENTRY_PLACE | IOMMU_MAP_ENTRY_UNMAPPED; in iommu_gas_init_domain()
263 RB_INSERT_PREV(iommu_gas_entries_tree, &domain->rb_root, end, begin); in iommu_gas_init_domain()
267 domain->start_gap = begin; in iommu_gas_init_domain()
268 domain->first_place = begin; in iommu_gas_init_domain()
269 domain->last_place = end; in iommu_gas_init_domain()
270 domain->flags |= IOMMU_DOMAIN_GAS_INITED; in iommu_gas_init_domain()
280 KASSERT(domain->entries_cnt == 2, in iommu_gas_fini_domain()
283 entry = RB_MIN(iommu_gas_entries_tree, &domain->rb_root); in iommu_gas_fini_domain()
284 KASSERT(entry->start == 0, ("start entry start %p", domain)); in iommu_gas_fini_domain()
285 KASSERT(entry->end == IOMMU_PAGE_SIZE, ("start entry end %p", domain)); in iommu_gas_fini_domain()
286 KASSERT(entry->flags == in iommu_gas_fini_domain()
292 entry = RB_MAX(iommu_gas_entries_tree, &domain->rb_root); in iommu_gas_fini_domain()
293 KASSERT(entry->start == domain->end, ("end entry start %p", domain)); in iommu_gas_fini_domain()
294 KASSERT(entry->end == domain->end, ("end entry end %p", domain)); in iommu_gas_fini_domain()
295 KASSERT(entry->flags == in iommu_gas_fini_domain()
325 * The prev->end is always aligned on the page size, which in iommu_gas_match_one()
326 * causes page alignment for the entry->start too. in iommu_gas_match_one()
329 * to ensure that out-of-bounds accesses fault. in iommu_gas_match_one()
332 start = roundup2(beg, a->common->alignment); in iommu_gas_match_one()
337 end = MIN(end - IOMMU_PAGE_SIZE - 1, ubound); in iommu_gas_match_one()
338 offset = a->offset; in iommu_gas_match_one()
339 size = a->size; in iommu_gas_match_one()
340 if (start + offset + size - 1 > end) in iommu_gas_match_one()
344 if (!vm_addr_bound_ok(start + offset, size, a->common->boundary)) { in iommu_gas_match_one()
351 beg = roundup2(start + offset + 1, a->common->boundary); in iommu_gas_match_one()
352 start = roundup2(beg, a->common->alignment); in iommu_gas_match_one()
354 if (start + offset + size - 1 > end || in iommu_gas_match_one()
356 a->common->boundary)) { in iommu_gas_match_one()
367 if ((a->gas_flags & IOMMU_MF_CANSPLIT) == 0) in iommu_gas_match_one()
369 size = beg - first - offset; in iommu_gas_match_one()
373 entry = a->entry; in iommu_gas_match_one()
374 entry->start = start; in iommu_gas_match_one()
375 entry->end = start + roundup2(size + offset, IOMMU_PAGE_SIZE); in iommu_gas_match_one()
376 entry->flags = IOMMU_MAP_ENTRY_MAP; in iommu_gas_match_one()
380 /* Find the next entry that might abut a big-enough range. */
387 next->free_down >= min_free) { in iommu_gas_next()
392 next->free_down >= min_free); in iommu_gas_next()
394 /* Find next entry in a left-parent ancestor. */ in iommu_gas_next()
404 * Address-ordered first-fit search of 'domain' for free space satisfying the
407 * 'domain' is described by an rb-tree of map entries at domain->rb_root, and
408 * domain->start_gap points to a map entry less than or adjacent to the first
409 * free-space of size at least 3 pages.
419 KASSERT(a->entry->flags == 0, in iommu_gas_find_space()
420 ("dirty entry %p %p", domain, a->entry)); in iommu_gas_find_space()
428 first = domain->start_gap; in iommu_gas_find_space()
429 while (first != NULL && first->free_down < min_free) in iommu_gas_find_space()
434 first->last + min_free <= curr->start) in iommu_gas_find_space()
437 curr->end + min_free <= first->first) in iommu_gas_find_space()
440 domain->start_gap = curr; in iommu_gas_find_space()
447 roundup2(a->size + a->offset, IOMMU_PAGE_SIZE); in iommu_gas_find_space()
449 /* Climb to find a node in the subtree of big-enough ranges. */ in iommu_gas_find_space()
451 while (first != NULL && first->free_down < min_free) in iommu_gas_find_space()
455 * Walk the big-enough ranges tree until one satisfies alignment in iommu_gas_find_space()
458 addr = a->common->lowaddr; in iommu_gas_find_space()
462 iommu_gas_match_one(a, first->last, curr->start, in iommu_gas_find_space()
465 &domain->rb_root, curr, a->entry); in iommu_gas_find_space()
468 if (curr->end >= addr) { in iommu_gas_find_space()
473 iommu_gas_match_one(a, curr->end, first->first, in iommu_gas_find_space()
476 &domain->rb_root, curr, a->entry); in iommu_gas_find_space()
484 * before highaddr that could abut a big-enough range. in iommu_gas_find_space()
486 addr = a->common->highaddr; in iommu_gas_find_space()
487 while (curr != NULL && curr->last < addr) in iommu_gas_find_space()
490 while (curr != NULL && curr->free_down >= min_free) { in iommu_gas_find_space()
491 if (addr < curr->end) in iommu_gas_find_space()
500 * Walk the remaining big-enough ranges until one satisfies alignment in iommu_gas_find_space()
506 iommu_gas_match_one(a, first->last, curr->start, in iommu_gas_find_space()
507 addr + 1, domain->end - 1)) { in iommu_gas_find_space()
509 &domain->rb_root, curr, a->entry); in iommu_gas_find_space()
513 iommu_gas_match_one(a, curr->end, first->first, in iommu_gas_find_space()
514 addr + 1, domain->end - 1)) { in iommu_gas_find_space()
516 &domain->rb_root, curr, a->entry); in iommu_gas_find_space()
532 if ((entry->start & IOMMU_PAGE_MASK) != 0 || in iommu_gas_alloc_region()
533 (entry->end & IOMMU_PAGE_MASK) != 0) in iommu_gas_alloc_region()
535 if (entry->start >= entry->end) in iommu_gas_alloc_region()
537 if (entry->end >= domain->end) in iommu_gas_alloc_region()
540 entry->flags |= IOMMU_MAP_ENTRY_FAKE; in iommu_gas_alloc_region()
541 next = RB_NFIND(iommu_gas_entries_tree, &domain->rb_root, entry); in iommu_gas_alloc_region()
542 KASSERT(next != NULL, ("next must be non-null %p %jx", domain, in iommu_gas_alloc_region()
543 (uintmax_t)entry->start)); in iommu_gas_alloc_region()
544 prev = RB_PREV(iommu_gas_entries_tree, &domain->rb_root, next); in iommu_gas_alloc_region()
546 entry->flags &= ~IOMMU_MAP_ENTRY_FAKE; in iommu_gas_alloc_region()
556 if (prev != NULL && prev->end > entry->start && in iommu_gas_alloc_region()
557 (prev->flags & IOMMU_MAP_ENTRY_PLACE) == 0) { in iommu_gas_alloc_region()
559 (prev->flags & IOMMU_MAP_ENTRY_RMRR) == 0) in iommu_gas_alloc_region()
561 entry->start = prev->end; in iommu_gas_alloc_region()
563 if (next->start < entry->end && in iommu_gas_alloc_region()
564 (next->flags & IOMMU_MAP_ENTRY_PLACE) == 0) { in iommu_gas_alloc_region()
566 (next->flags & IOMMU_MAP_ENTRY_RMRR) == 0) in iommu_gas_alloc_region()
568 entry->end = next->start; in iommu_gas_alloc_region()
570 if (entry->end == entry->start) in iommu_gas_alloc_region()
573 if (prev != NULL && prev->end > entry->start) { in iommu_gas_alloc_region()
579 &domain->rb_root, next, entry); in iommu_gas_alloc_region()
580 if (next->start < entry->end) { in iommu_gas_alloc_region()
586 entry->flags = IOMMU_MAP_ENTRY_RMRR; in iommu_gas_alloc_region()
590 ip = RB_PREV(iommu_gas_entries_tree, &domain->rb_root, entry); in iommu_gas_alloc_region()
591 in = RB_NEXT(iommu_gas_entries_tree, &domain->rb_root, entry); in iommu_gas_alloc_region()
594 entry, entry->start, entry->end, prev, in iommu_gas_alloc_region()
595 prev == NULL ? 0 : prev->start, prev == NULL ? 0 : prev->end, in iommu_gas_alloc_region()
596 ip, ip == NULL ? 0 : ip->start, ip == NULL ? 0 : ip->end)); in iommu_gas_alloc_region()
599 entry, entry->start, entry->end, next, in iommu_gas_alloc_region()
600 next == NULL ? 0 : next->start, next == NULL ? 0 : next->end, in iommu_gas_alloc_region()
601 in, in == NULL ? 0 : in->start, in == NULL ? 0 : in->end)); in iommu_gas_alloc_region()
612 domain = entry->domain; in iommu_gas_free_space()
613 KASSERT((entry->flags & (IOMMU_MAP_ENTRY_PLACE | IOMMU_MAP_ENTRY_RMRR | in iommu_gas_free_space()
619 entry->flags &= ~IOMMU_MAP_ENTRY_MAP; in iommu_gas_free_space()
632 domain = entry->domain; in iommu_gas_free_region()
633 KASSERT((entry->flags & (IOMMU_MAP_ENTRY_PLACE | IOMMU_MAP_ENTRY_RMRR | in iommu_gas_free_region()
635 ("non-RMRR entry %p %p", domain, entry)); in iommu_gas_free_region()
638 if (entry != domain->first_place && in iommu_gas_free_region()
639 entry != domain->last_place) in iommu_gas_free_region()
641 entry->flags &= ~IOMMU_MAP_ENTRY_RMRR; in iommu_gas_free_region()
653 MPASS(end <= domain->end); in iommu_gas_remove_clip_left()
665 entry = RB_NFIND(iommu_gas_entries_tree, &domain->rb_root, &fentry); in iommu_gas_remove_clip_left()
667 if (entry->start >= start || in iommu_gas_remove_clip_left()
668 (entry->flags & IOMMU_MAP_ENTRY_RMRR) != 0) in iommu_gas_remove_clip_left()
674 res->start = entry->end = start; in iommu_gas_remove_clip_left()
677 &domain->rb_root, entry, res); in iommu_gas_remove_clip_left()
686 if (entry->start >= end || (entry->flags & IOMMU_MAP_ENTRY_RMRR) != 0) in iommu_gas_remove_clip_right()
690 r->end = entry->start = end; in iommu_gas_remove_clip_right()
693 &domain->rb_root, entry, r); in iommu_gas_remove_clip_right()
703 if ((entry->flags & (IOMMU_MAP_ENTRY_UNMAPPED | in iommu_gas_remove_unmap()
707 MPASS((entry->flags & IOMMU_MAP_ENTRY_PLACE) == 0); in iommu_gas_remove_unmap()
708 entry->flags |= IOMMU_MAP_ENTRY_REMOVING; in iommu_gas_remove_unmap()
727 if (entry->start >= end) in iommu_gas_remove_locked()
729 KASSERT(start <= entry->start, in iommu_gas_remove_locked()
731 entry->start, entry->end, start)); in iommu_gas_remove_locked()
740 RB_FOREACH(entry, iommu_gas_entries_tree, &domain->rb_root) { in iommu_gas_remove_locked()
741 if ((entry->flags & (IOMMU_MAP_ENTRY_RMRR | in iommu_gas_remove_locked()
744 KASSERT(entry->end <= start || entry->start >= end, in iommu_gas_remove_locked()
747 entry->start, entry->end, start, end)); in iommu_gas_remove_locked()
832 KASSERT(entry->end < domain->end, ("allocated GPA %jx, max GPA %jx", in iommu_gas_map()
833 (uintmax_t)entry->end, (uintmax_t)domain->end)); in iommu_gas_map()
834 entry->flags |= eflags; in iommu_gas_map()
837 error = domain->ops->map(domain, entry, ma, eflags, in iommu_gas_map()
858 KASSERT(entry->domain == domain, in iommu_gas_map_region()
859 ("mismatched domain %p entry %p entry->domain %p", domain, in iommu_gas_map_region()
860 entry, entry->domain)); in iommu_gas_map_region()
861 KASSERT(entry->flags == 0, ("used RMRR entry %p %p %x", domain, in iommu_gas_map_region()
862 entry, entry->flags)); in iommu_gas_map_region()
866 start = entry->start; in iommu_gas_map_region()
873 entry->flags |= eflags; in iommu_gas_map_region()
875 if (entry->end == entry->start) in iommu_gas_map_region()
878 error = domain->ops->map(domain, entry, in iommu_gas_map_region()
879 ma + OFF_TO_IDX(start - entry->start), eflags, in iommu_gas_map_region()
900 entry->start = start; in iommu_gas_reserve_region_locked()
901 entry->end = end; in iommu_gas_reserve_region_locked()
904 entry->flags |= IOMMU_MAP_ENTRY_UNMAPPED; in iommu_gas_reserve_region_locked()
940 end = ummin(end, domain->end); in iommu_gas_reserve_region_extend()
949 next = RB_NFIND(iommu_gas_entries_tree, &domain->rb_root, &key); in iommu_gas_reserve_region_extend()
951 "after %#jx", domain, (uintmax_t)domain->end, in iommu_gas_reserve_region_extend()
953 entry_end = ummin(end, next->start); in iommu_gas_reserve_region_extend()
954 prev = RB_PREV(iommu_gas_entries_tree, &domain->rb_root, next); in iommu_gas_reserve_region_extend()
956 entry_start = ummax(start, prev->end); in iommu_gas_reserve_region_extend()
959 start = next->end; in iommu_gas_reserve_region_extend()
960 /* Reserve the region if non-empty. */ in iommu_gas_reserve_region_extend()
979 iommu_unmap_msi(struct iommu_ctx *ctx) in iommu_unmap_msi() argument
984 domain = ctx->domain; in iommu_unmap_msi()
985 entry = domain->msi_entry; in iommu_unmap_msi()
989 domain->ops->unmap(domain, entry, IOMMU_PGF_WAITOK); in iommu_unmap_msi()
995 domain->msi_entry = NULL; in iommu_unmap_msi()
996 domain->msi_base = 0; in iommu_unmap_msi()
997 domain->msi_phys = 0; in iommu_unmap_msi()
1001 iommu_map_msi(struct iommu_ctx *ctx, iommu_gaddr_t size, int offset, in iommu_map_msi() argument
1009 domain = ctx->domain; in iommu_map_msi()
1013 entry = domain->msi_entry; in iommu_map_msi()
1017 error = iommu_gas_map(domain, &ctx->tag->common, size, offset, in iommu_map_msi()
1021 if (domain->msi_entry == NULL) { in iommu_map_msi()
1022 MPASS(domain->msi_base == 0); in iommu_map_msi()
1023 MPASS(domain->msi_phys == 0); in iommu_map_msi()
1025 domain->msi_entry = entry; in iommu_map_msi()
1026 domain->msi_base = entry->start; in iommu_map_msi()
1027 domain->msi_phys = VM_PAGE_TO_PHYS(ma[0]); in iommu_map_msi()
1035 } else if (domain->msi_entry != NULL) { in iommu_map_msi()
1052 *addr = (*addr - domain->msi_phys) + domain->msi_base; in iommu_translate_msi()
1054 KASSERT(*addr >= domain->msi_entry->start, in iommu_translate_msi()
1056 __func__, (uintmax_t)*addr, (uintmax_t)domain->msi_entry->start)); in iommu_translate_msi()
1058 KASSERT(*addr + sizeof(*addr) <= domain->msi_entry->end, in iommu_translate_msi()
1060 __func__, (uintmax_t)*addr, (uintmax_t)domain->msi_entry->end)); in iommu_translate_msi()
1063 SYSCTL_NODE(_hw, OID_AUTO, iommu, CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, "");
1081 db_printf("iommu_domain %p tree %p iommu %p fl %#x\n", domain, in iommu_debug_dump_gas()
1082 &domain->rb_root, domain->iommu, domain->flags); in iommu_debug_dump_gas()
1083 db_printf("iommu_domain %p tree %p\n", domain, &domain->rb_root); in iommu_debug_dump_gas()
1084 RB_FOREACH(entry, iommu_gas_entries_tree, &domain->rb_root) { in iommu_debug_dump_gas()
1087 entry, (uintmax_t)entry->start, (uintmax_t)entry->end, in iommu_debug_dump_gas()
1088 entry->flags, in iommu_debug_dump_gas()
1089 (uintmax_t)entry->first, (uintmax_t)entry->last, in iommu_debug_dump_gas()
1090 (uintmax_t)entry->free_down); in iommu_debug_dump_gas()
1091 if (entry == domain->start_gap) in iommu_debug_dump_gas()
1093 if (entry == domain->first_place) in iommu_debug_dump_gas()
1095 if (entry == domain->last_place) in iommu_debug_dump_gas()