Lines Matching full:entry
31 #define RB_AUGMENT_CHECK(entry) iommu_gas_augment_entry(entry) argument
110 iommu_gas_free_entry(struct iommu_map_entry *entry) in iommu_gas_free_entry() argument
115 n = vm_page_free_pages_toq(&entry->pgtbl_free, false); in iommu_gas_free_entry()
119 domain = entry->domain; in iommu_gas_free_entry()
122 uma_zfree(iommu_map_entry_zone, entry); in iommu_gas_free_entry()
130 KASSERT(a->start <= a->end, ("inverted entry %p (%jx, %jx)", in iommu_gas_cmp_entries()
132 KASSERT(b->start <= b->end, ("inverted entry %p (%jx, %jx)", in iommu_gas_cmp_entries()
155 iommu_gas_augment_entry(struct iommu_map_entry *entry) in iommu_gas_augment_entry() argument
161 bound = entry->start; in iommu_gas_augment_entry()
162 if ((child = RB_LEFT(entry, rb_entry)) != NULL) { in iommu_gas_augment_entry()
166 delta = bound - entry->first; in iommu_gas_augment_entry()
167 entry->first = bound; in iommu_gas_augment_entry()
168 bound = entry->end; in iommu_gas_augment_entry()
169 if ((child = RB_RIGHT(entry, rb_entry)) != NULL) { in iommu_gas_augment_entry()
174 delta += entry->last - bound; in iommu_gas_augment_entry()
176 delta = entry->free_down - free_down; in iommu_gas_augment_entry()
177 entry->last = bound; in iommu_gas_augment_entry()
178 entry->free_down = free_down; in iommu_gas_augment_entry()
194 struct iommu_map_entry *entry, *l, *r; in iommu_gas_check_free() local
197 RB_FOREACH(entry, iommu_gas_entries_tree, &domain->rb_root) { in iommu_gas_check_free()
198 KASSERT(domain == entry->domain, in iommu_gas_check_free()
199 ("mismatched free domain %p entry %p entry->domain %p", in iommu_gas_check_free()
200 domain, entry, entry->domain)); in iommu_gas_check_free()
201 l = RB_LEFT(entry, rb_entry); in iommu_gas_check_free()
202 r = RB_RIGHT(entry, rb_entry); in iommu_gas_check_free()
206 v = MAX(v, entry->start - l->last); in iommu_gas_check_free()
210 v = MAX(v, r->first - entry->end); in iommu_gas_check_free()
212 MPASS(entry->free_down == v); in iommu_gas_check_free()
218 iommu_gas_rb_remove(struct iommu_domain *domain, struct iommu_map_entry *entry) in iommu_gas_rb_remove() argument
222 /* Removing entry may open a new free gap before domain->start_gap. */ in iommu_gas_rb_remove()
223 if (entry->end <= domain->start_gap->end) { in iommu_gas_rb_remove()
224 if (RB_RIGHT(entry, rb_entry) != NULL) in iommu_gas_rb_remove()
225 nbr = iommu_gas_entries_tree_RB_NEXT(entry); in iommu_gas_rb_remove()
226 else if (RB_LEFT(entry, rb_entry) != NULL) in iommu_gas_rb_remove()
227 nbr = RB_LEFT(entry, rb_entry); in iommu_gas_rb_remove()
229 nbr = RB_PARENT(entry, rb_entry); in iommu_gas_rb_remove()
232 RB_REMOVE(iommu_gas_entries_tree, &domain->rb_root, entry); in iommu_gas_rb_remove()
277 struct iommu_map_entry *entry; in iommu_gas_fini_domain() local
283 entry = RB_MIN(iommu_gas_entries_tree, &domain->rb_root); in iommu_gas_fini_domain()
284 KASSERT(entry->start == 0, ("start entry start %p", domain)); in iommu_gas_fini_domain()
285 KASSERT(entry->end == IOMMU_PAGE_SIZE, ("start entry end %p", domain)); in iommu_gas_fini_domain()
286 KASSERT(entry->flags == in iommu_gas_fini_domain()
288 ("start entry flags %p", domain)); in iommu_gas_fini_domain()
289 iommu_gas_rb_remove(domain, entry); in iommu_gas_fini_domain()
290 iommu_gas_free_entry(entry); in iommu_gas_fini_domain()
292 entry = RB_MAX(iommu_gas_entries_tree, &domain->rb_root); in iommu_gas_fini_domain()
293 KASSERT(entry->start == domain->end, ("end entry start %p", domain)); in iommu_gas_fini_domain()
294 KASSERT(entry->end == domain->end, ("end entry end %p", domain)); in iommu_gas_fini_domain()
295 KASSERT(entry->flags == in iommu_gas_fini_domain()
297 ("end entry flags %p", domain)); in iommu_gas_fini_domain()
298 iommu_gas_rb_remove(domain, entry); in iommu_gas_fini_domain()
299 iommu_gas_free_entry(entry); in iommu_gas_fini_domain()
307 struct iommu_map_entry *entry; member
320 struct iommu_map_entry *entry; in iommu_gas_match_one() local
326 * causes page alignment for the entry->start too. in iommu_gas_match_one()
328 * Create IOMMU_PAGE_SIZE gaps before, after new entry in iommu_gas_match_one()
364 * start of the next entry, then we do not have gap. in iommu_gas_match_one()
373 entry = a->entry; in iommu_gas_match_one()
374 entry->start = start; in iommu_gas_match_one()
375 entry->end = start + roundup2(size + offset, IOMMU_PAGE_SIZE); in iommu_gas_match_one()
376 entry->flags = IOMMU_MAP_ENTRY_MAP; in iommu_gas_match_one()
380 /* Find the next entry that might abut a big-enough range. */
388 /* Find next entry in right subtree. */ in iommu_gas_next()
394 /* Find next entry in a left-parent ancestor. */ in iommu_gas_next()
408 * domain->start_gap points to a map entry less than or adjacent to the first
419 KASSERT(a->entry->flags == 0, in iommu_gas_find_space()
420 ("dirty entry %p %p", domain, a->entry)); in iommu_gas_find_space()
423 * start_gap may point to an entry adjacent to gaps too small for any in iommu_gas_find_space()
465 &domain->rb_root, curr, a->entry); in iommu_gas_find_space()
476 &domain->rb_root, curr, a->entry); in iommu_gas_find_space()
483 * the nearest ancestor that spans highaddr. Then find the last entry in iommu_gas_find_space()
509 &domain->rb_root, curr, a->entry); in iommu_gas_find_space()
516 &domain->rb_root, curr, a->entry); in iommu_gas_find_space()
525 iommu_gas_alloc_region(struct iommu_domain *domain, struct iommu_map_entry *entry, in iommu_gas_alloc_region() argument
532 if ((entry->start & IOMMU_PAGE_MASK) != 0 || in iommu_gas_alloc_region()
533 (entry->end & IOMMU_PAGE_MASK) != 0) in iommu_gas_alloc_region()
535 if (entry->start >= entry->end) in iommu_gas_alloc_region()
537 if (entry->end >= domain->end) in iommu_gas_alloc_region()
540 entry->flags |= IOMMU_MAP_ENTRY_FAKE; in iommu_gas_alloc_region()
541 next = RB_NFIND(iommu_gas_entries_tree, &domain->rb_root, entry); in iommu_gas_alloc_region()
543 (uintmax_t)entry->start)); in iommu_gas_alloc_region()
546 entry->flags &= ~IOMMU_MAP_ENTRY_FAKE; in iommu_gas_alloc_region()
556 if (prev != NULL && prev->end > entry->start && in iommu_gas_alloc_region()
561 entry->start = prev->end; in iommu_gas_alloc_region()
563 if (next->start < entry->end && in iommu_gas_alloc_region()
568 entry->end = next->start; in iommu_gas_alloc_region()
570 if (entry->end == entry->start) in iommu_gas_alloc_region()
573 if (prev != NULL && prev->end > entry->start) { in iommu_gas_alloc_region()
574 /* This assumes that prev is the placeholder entry. */ in iommu_gas_alloc_region()
579 &domain->rb_root, next, entry); in iommu_gas_alloc_region()
580 if (next->start < entry->end) { in iommu_gas_alloc_region()
586 entry->flags = IOMMU_MAP_ENTRY_RMRR; in iommu_gas_alloc_region()
590 ip = RB_PREV(iommu_gas_entries_tree, &domain->rb_root, entry); in iommu_gas_alloc_region()
591 in = RB_NEXT(iommu_gas_entries_tree, &domain->rb_root, entry); in iommu_gas_alloc_region()
594 entry, entry->start, entry->end, prev, in iommu_gas_alloc_region()
599 entry, entry->start, entry->end, next, in iommu_gas_alloc_region()
608 iommu_gas_free_space(struct iommu_map_entry *entry) in iommu_gas_free_space() argument
612 domain = entry->domain; in iommu_gas_free_space()
613 KASSERT((entry->flags & (IOMMU_MAP_ENTRY_PLACE | IOMMU_MAP_ENTRY_RMRR | in iommu_gas_free_space()
615 ("permanent entry %p %p", domain, entry)); in iommu_gas_free_space()
618 iommu_gas_rb_remove(domain, entry); in iommu_gas_free_space()
619 entry->flags &= ~IOMMU_MAP_ENTRY_MAP; in iommu_gas_free_space()
628 iommu_gas_free_region(struct iommu_map_entry *entry) in iommu_gas_free_region() argument
632 domain = entry->domain; in iommu_gas_free_region()
633 KASSERT((entry->flags & (IOMMU_MAP_ENTRY_PLACE | IOMMU_MAP_ENTRY_RMRR | in iommu_gas_free_region()
635 ("non-RMRR entry %p %p", domain, entry)); in iommu_gas_free_region()
638 if (entry != domain->first_place && in iommu_gas_free_region()
639 entry != domain->last_place) in iommu_gas_free_region()
640 iommu_gas_rb_remove(domain, entry); in iommu_gas_free_region()
641 entry->flags &= ~IOMMU_MAP_ENTRY_RMRR; in iommu_gas_free_region()
649 struct iommu_map_entry *entry, *res, fentry; in iommu_gas_remove_clip_left() local
656 * Find an entry which contains the supplied guest's address in iommu_gas_remove_clip_left()
657 * start, or the first entry after the start. Since we in iommu_gas_remove_clip_left()
658 * asserted that start is below domain end, entry should in iommu_gas_remove_clip_left()
665 entry = RB_NFIND(iommu_gas_entries_tree, &domain->rb_root, &fentry); in iommu_gas_remove_clip_left()
667 if (entry->start >= start || in iommu_gas_remove_clip_left()
668 (entry->flags & IOMMU_MAP_ENTRY_RMRR) != 0) in iommu_gas_remove_clip_left()
669 return (entry); in iommu_gas_remove_clip_left()
673 *res = *entry; in iommu_gas_remove_clip_left()
674 res->start = entry->end = start; in iommu_gas_remove_clip_left()
675 RB_UPDATE_AUGMENT(entry, rb_entry); in iommu_gas_remove_clip_left()
677 &domain->rb_root, entry, res); in iommu_gas_remove_clip_left()
683 iommu_gaddr_t end, struct iommu_map_entry *entry, in iommu_gas_remove_clip_right() argument
686 if (entry->start >= end || (entry->flags & IOMMU_MAP_ENTRY_RMRR) != 0) in iommu_gas_remove_clip_right()
689 *r = *entry; in iommu_gas_remove_clip_right()
690 r->end = entry->start = end; in iommu_gas_remove_clip_right()
691 RB_UPDATE_AUGMENT(entry, rb_entry); in iommu_gas_remove_clip_right()
693 &domain->rb_root, entry, r); in iommu_gas_remove_clip_right()
699 struct iommu_map_entry *entry, struct iommu_map_entries_tailq *gcp) in iommu_gas_remove_unmap() argument
703 if ((entry->flags & (IOMMU_MAP_ENTRY_UNMAPPED | in iommu_gas_remove_unmap()
707 MPASS((entry->flags & IOMMU_MAP_ENTRY_PLACE) == 0); in iommu_gas_remove_unmap()
708 entry->flags |= IOMMU_MAP_ENTRY_REMOVING; in iommu_gas_remove_unmap()
709 TAILQ_INSERT_TAIL(gcp, entry, dmamap_link); in iommu_gas_remove_unmap()
718 struct iommu_map_entry *entry, *nentry; in iommu_gas_remove_locked() local
726 RB_FOREACH_FROM(entry, iommu_gas_entries_tree, nentry) { in iommu_gas_remove_locked()
727 if (entry->start >= end) in iommu_gas_remove_locked()
729 KASSERT(start <= entry->start, in iommu_gas_remove_locked()
730 ("iommu_gas_remove entry (%#jx, %#jx) start %#jx", in iommu_gas_remove_locked()
731 entry->start, entry->end, start)); in iommu_gas_remove_locked()
732 iommu_gas_remove_unmap(domain, entry, gc); in iommu_gas_remove_locked()
734 if (iommu_gas_remove_clip_right(domain, end, entry, *r2)) { in iommu_gas_remove_locked()
740 RB_FOREACH(entry, iommu_gas_entries_tree, &domain->rb_root) { in iommu_gas_remove_locked()
741 if ((entry->flags & (IOMMU_MAP_ENTRY_RMRR | in iommu_gas_remove_locked()
744 KASSERT(entry->end <= start || entry->start >= end, in iommu_gas_remove_locked()
745 ("iommu_gas_remove leftover entry (%#jx, %#jx) range " in iommu_gas_remove_locked()
747 entry->start, entry->end, start, end)); in iommu_gas_remove_locked()
804 struct iommu_map_entry *entry; in iommu_gas_map() local
814 entry = iommu_gas_alloc_entry(domain, in iommu_gas_map()
816 if (entry == NULL) in iommu_gas_map()
818 a.entry = entry; in iommu_gas_map()
823 iommu_gas_free_entry(entry); in iommu_gas_map()
832 KASSERT(entry->end < domain->end, ("allocated GPA %jx, max GPA %jx", in iommu_gas_map()
833 (uintmax_t)entry->end, (uintmax_t)domain->end)); in iommu_gas_map()
834 entry->flags |= eflags; in iommu_gas_map()
837 error = domain->ops->map(domain, entry, ma, eflags, in iommu_gas_map()
840 iommu_domain_unload_entry(entry, true, in iommu_gas_map()
847 *res = entry; in iommu_gas_map()
852 iommu_gas_map_region(struct iommu_domain *domain, struct iommu_map_entry *entry, in iommu_gas_map_region() argument
858 KASSERT(entry->domain == domain, in iommu_gas_map_region()
859 ("mismatched domain %p entry %p entry->domain %p", domain, in iommu_gas_map_region()
860 entry, entry->domain)); in iommu_gas_map_region()
861 KASSERT(entry->flags == 0, ("used RMRR entry %p %p %x", domain, in iommu_gas_map_region()
862 entry, entry->flags)); in iommu_gas_map_region()
866 start = entry->start; in iommu_gas_map_region()
868 error = iommu_gas_alloc_region(domain, entry, flags); in iommu_gas_map_region()
873 entry->flags |= eflags; in iommu_gas_map_region()
875 if (entry->end == entry->start) in iommu_gas_map_region()
878 error = domain->ops->map(domain, entry, in iommu_gas_map_region()
879 ma + OFF_TO_IDX(start - entry->start), eflags, in iommu_gas_map_region()
882 iommu_domain_unload_entry(entry, false, in iommu_gas_map_region()
894 iommu_gaddr_t start, iommu_gaddr_t end, struct iommu_map_entry *entry) in iommu_gas_reserve_region_locked() argument
900 entry->start = start; in iommu_gas_reserve_region_locked()
901 entry->end = end; in iommu_gas_reserve_region_locked()
902 error = iommu_gas_alloc_region(domain, entry, IOMMU_MF_CANWAIT); in iommu_gas_reserve_region_locked()
904 entry->flags |= IOMMU_MAP_ENTRY_UNMAPPED; in iommu_gas_reserve_region_locked()
912 struct iommu_map_entry *entry; in iommu_gas_reserve_region() local
915 entry = iommu_gas_alloc_entry(domain, IOMMU_PGF_WAITOK); in iommu_gas_reserve_region()
917 error = iommu_gas_reserve_region_locked(domain, start, end, entry); in iommu_gas_reserve_region()
920 iommu_gas_free_entry(entry); in iommu_gas_reserve_region()
922 *entry0 = entry; in iommu_gas_reserve_region()
934 struct iommu_map_entry *entry, *next, *prev, key = {}; in iommu_gas_reserve_region_extend() local
939 entry = NULL; in iommu_gas_reserve_region_extend()
942 /* Preallocate an entry. */ in iommu_gas_reserve_region_extend()
943 if (entry == NULL) in iommu_gas_reserve_region_extend()
944 entry = iommu_gas_alloc_entry(domain, in iommu_gas_reserve_region_extend()
946 /* Calculate the free region from here to the next entry. */ in iommu_gas_reserve_region_extend()
950 KASSERT(next != NULL, ("domain %p with end %#jx has no entry " in iommu_gas_reserve_region_extend()
963 entry_start, entry_end, entry); in iommu_gas_reserve_region_extend()
968 entry = NULL; in iommu_gas_reserve_region_extend()
972 /* Release a preallocated entry if it was not used. */ in iommu_gas_reserve_region_extend()
973 if (entry != NULL) in iommu_gas_reserve_region_extend()
974 iommu_gas_free_entry(entry); in iommu_gas_reserve_region_extend()
981 struct iommu_map_entry *entry; in iommu_unmap_msi() local
985 entry = domain->msi_entry; in iommu_unmap_msi()
986 if (entry == NULL) in iommu_unmap_msi()
989 domain->ops->unmap(domain, entry, IOMMU_PGF_WAITOK); in iommu_unmap_msi()
991 iommu_gas_free_space(entry); in iommu_unmap_msi()
993 iommu_gas_free_entry(entry); in iommu_unmap_msi()
1005 struct iommu_map_entry *entry; in iommu_map_msi() local
1013 entry = domain->msi_entry; in iommu_map_msi()
1016 if (entry == NULL) { in iommu_map_msi()
1018 eflags, flags, ma, &entry); in iommu_map_msi()
1025 domain->msi_entry = entry; in iommu_map_msi()
1026 domain->msi_base = entry->start; in iommu_map_msi()
1031 * MSI page allocated. Free the unneeded entry. in iommu_map_msi()
1033 iommu_gas_free_entry(entry); in iommu_map_msi()
1055 ("%s: Address is below the MSI entry start address (%jx < %jx)", in iommu_translate_msi()
1059 ("%s: Address is above the MSI entry end address (%jx < %jx)", in iommu_translate_msi()
1079 struct iommu_map_entry *entry; in iommu_debug_dump_gas() local
1084 RB_FOREACH(entry, iommu_gas_entries_tree, &domain->rb_root) { in iommu_debug_dump_gas()
1087 entry, (uintmax_t)entry->start, (uintmax_t)entry->end, in iommu_debug_dump_gas()
1088 entry->flags, in iommu_debug_dump_gas()
1089 (uintmax_t)entry->first, (uintmax_t)entry->last, in iommu_debug_dump_gas()
1090 (uintmax_t)entry->free_down); in iommu_debug_dump_gas()
1091 if (entry == domain->start_gap) in iommu_debug_dump_gas()
1093 if (entry == domain->first_place) in iommu_debug_dump_gas()
1095 if (entry == domain->last_place) in iommu_debug_dump_gas()