32 					struct io_pagetable *iopt,  in iopt_area_contig_init()  argument
36 	lockdep_assert_held(&iopt->iova_rwsem);  in iopt_area_contig_init()
40 	iter->area = iopt_area_iter_first(iopt, iova, iova);  in iopt_area_contig_init()
109 static int iopt_alloc_iova(struct io_pagetable *iopt, unsigned long *iova,  in iopt_alloc_iova()  argument
118 	lockdep_assert_held(&iopt->iova_rwsem);  in iopt_alloc_iova()
142 	if (iova_alignment < iopt->iova_alignment)  in iopt_alloc_iova()
145 	interval_tree_for_each_span(&allowed_span, &iopt->allowed_itree,  in iopt_alloc_iova()
147 		if (RB_EMPTY_ROOT(&iopt->allowed_itree.rb_root)) {  in iopt_alloc_iova()
158 			&used_span, &iopt->reserved_itree, &iopt->area_itree,  in iopt_alloc_iova()
172 static int iopt_check_iova(struct io_pagetable *iopt, unsigned long iova,  in iopt_check_iova()  argument
177 	lockdep_assert_held(&iopt->iova_rwsem);  in iopt_check_iova()
179 	if ((iova & (iopt->iova_alignment - 1)))  in iopt_check_iova()
186 	if (iopt_reserved_iter_first(iopt, iova, last))  in iopt_check_iova()
190 	if (iopt_area_iter_first(iopt, iova, last))  in iopt_check_iova()
198 static int iopt_insert_area(struct io_pagetable *iopt, struct iopt_area *area,  in iopt_insert_area()  argument
203 	lockdep_assert_held_write(&iopt->iova_rwsem);  in iopt_insert_area()
210 	if (area->page_offset & (iopt->iova_alignment - 1))  in iopt_insert_area()
228 	area->iopt = iopt;  in iopt_insert_area()
229 	interval_tree_insert(&area->node, &iopt->area_itree);  in iopt_insert_area()
245 static int iopt_alloc_area_pages(struct io_pagetable *iopt,  in iopt_alloc_area_pages()  argument
261 	down_write(&iopt->iova_rwsem);  in iopt_alloc_area_pages()
262 	if ((length & (iopt->iova_alignment - 1)) || !length) {  in iopt_alloc_area_pages()
279 		rc = iopt_alloc_iova(iopt, dst_iova, start, length);  in iopt_alloc_area_pages()
283 		    WARN_ON(iopt_check_iova(iopt, *dst_iova, length))) {  in iopt_alloc_area_pages()
288 		rc = iopt_check_iova(iopt, *dst_iova, length);  in iopt_alloc_area_pages()
299 		rc = iopt_insert_area(iopt, elm->area, elm->pages, iova,  in iopt_alloc_area_pages()
307 	up_write(&iopt->iova_rwsem);  in iopt_alloc_area_pages()
315 	if (area->iopt) {  in iopt_abort_area()
316 		down_write(&area->iopt->iova_rwsem);  in iopt_abort_area()
317 		interval_tree_remove(&area->node, &area->iopt->area_itree);  in iopt_abort_area()
318 		up_write(&area->iopt->iova_rwsem);  in iopt_abort_area()
360 int iopt_map_pages(struct io_pagetable *iopt, struct list_head *pages_list,  in iopt_map_pages()  argument
367 	rc = iopt_alloc_area_pages(iopt, pages_list, length, dst_iova,  in iopt_map_pages()
372 	down_read(&iopt->domains_rwsem);  in iopt_map_pages()
377 	down_write(&iopt->iova_rwsem);  in iopt_map_pages()
388 	up_write(&iopt->iova_rwsem);  in iopt_map_pages()
390 	up_read(&iopt->domains_rwsem);  in iopt_map_pages()
394 static int iopt_map_common(struct iommufd_ctx *ictx, struct io_pagetable *iopt,  in iopt_map_common()  argument
411 	rc = iopt_map_pages(iopt, &pages_list, length, iova, iommu_prot, flags);  in iopt_map_common()
441 int iopt_map_user_pages(struct iommufd_ctx *ictx, struct io_pagetable *iopt,  in iopt_map_user_pages()  argument
452 	return iopt_map_common(ictx, iopt, pages, iova, length,  in iopt_map_user_pages()
468 int iopt_map_file_pages(struct iommufd_ctx *ictx, struct io_pagetable *iopt,  in iopt_map_file_pages()  argument
479 	return iopt_map_common(ictx, iopt, pages, iova, length,  in iopt_map_file_pages()
485 	struct io_pagetable *iopt;  member
504 	iopt_for_each_contig_area(&iter, area, arg->iopt, iova, last_iova) {  in __iommu_read_and_clear_dirty()
521 			   struct io_pagetable *iopt, unsigned long flags,  in iommu_read_and_clear_dirty()  argument
543 	arg.iopt = iopt;  in iommu_read_and_clear_dirty()
556 int iommufd_check_iova_range(struct io_pagetable *iopt,  in iommufd_check_iova_range()  argument
559 	size_t iommu_pgsize = iopt->iova_alignment;  in iommufd_check_iova_range()
582 int iopt_read_and_clear_dirty_data(struct io_pagetable *iopt,  in iopt_read_and_clear_dirty_data()  argument
589 	ret = iommufd_check_iova_range(iopt, bitmap);  in iopt_read_and_clear_dirty_data()
593 	down_read(&iopt->iova_rwsem);  in iopt_read_and_clear_dirty_data()
594 	ret = iommu_read_and_clear_dirty(domain, iopt, flags, bitmap);  in iopt_read_and_clear_dirty_data()
595 	up_read(&iopt->iova_rwsem);  in iopt_read_and_clear_dirty_data()
600 static int iopt_clear_dirty_data(struct io_pagetable *iopt,  in iopt_clear_dirty_data()  argument
609 	lockdep_assert_held_read(&iopt->iova_rwsem);  in iopt_clear_dirty_data()
613 	for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area;  in iopt_clear_dirty_data()
629 int iopt_set_dirty_tracking(struct io_pagetable *iopt,  in iopt_set_dirty_tracking()  argument
638 	down_read(&iopt->iova_rwsem);  in iopt_set_dirty_tracking()
642 		ret = iopt_clear_dirty_data(iopt, domain);  in iopt_set_dirty_tracking()
650 	up_read(&iopt->iova_rwsem);  in iopt_set_dirty_tracking()
654 int iopt_get_pages(struct io_pagetable *iopt, unsigned long iova,  in iopt_get_pages()  argument
667 	down_read(&iopt->iova_rwsem);  in iopt_get_pages()
668 	iopt_for_each_contig_area(&iter, area, iopt, iova, last_iova) {  in iopt_get_pages()
687 	up_read(&iopt->iova_rwsem);  in iopt_get_pages()
690 	up_read(&iopt->iova_rwsem);  in iopt_get_pages()
695 static int iopt_unmap_iova_range(struct io_pagetable *iopt, unsigned long start,  in iopt_unmap_iova_range()  argument
709 	down_read(&iopt->domains_rwsem);  in iopt_unmap_iova_range()
710 	down_write(&iopt->iova_rwsem);  in iopt_unmap_iova_range()
711 	while ((area = iopt_area_iter_first(iopt, start, last))) {  in iopt_unmap_iova_range()
740 			up_write(&iopt->iova_rwsem);  in iopt_unmap_iova_range()
741 			up_read(&iopt->domains_rwsem);  in iopt_unmap_iova_range()
743 			iommufd_access_notify_unmap(iopt, area_first, length);  in iopt_unmap_iova_range()
753 		up_write(&iopt->iova_rwsem);  in iopt_unmap_iova_range()
761 		down_write(&iopt->iova_rwsem);  in iopt_unmap_iova_range()
767 	up_write(&iopt->iova_rwsem);  in iopt_unmap_iova_range()
768 	up_read(&iopt->domains_rwsem);  in iopt_unmap_iova_range()
784 int iopt_unmap_iova(struct io_pagetable *iopt, unsigned long iova,  in iopt_unmap_iova()  argument
795 	return iopt_unmap_iova_range(iopt, iova, iova_last, unmapped);  in iopt_unmap_iova()
798 int iopt_unmap_all(struct io_pagetable *iopt, unsigned long *unmapped)  in iopt_unmap_all()  argument
802 	rc = iopt_unmap_iova_range(iopt, 0, ULONG_MAX, unmapped);  in iopt_unmap_all()
810 int iopt_set_allow_iova(struct io_pagetable *iopt,  in iopt_set_allow_iova()  argument
815 	down_write(&iopt->iova_rwsem);  in iopt_set_allow_iova()
816 	swap(*allowed_iova, iopt->allowed_itree);  in iopt_set_allow_iova()
818 	for (allowed = iopt_allowed_iter_first(iopt, 0, ULONG_MAX); allowed;  in iopt_set_allow_iova()
820 		if (iopt_reserved_iter_first(iopt, allowed->node.start,  in iopt_set_allow_iova()
822 			swap(*allowed_iova, iopt->allowed_itree);  in iopt_set_allow_iova()
823 			up_write(&iopt->iova_rwsem);  in iopt_set_allow_iova()
827 	up_write(&iopt->iova_rwsem);  in iopt_set_allow_iova()
831 int iopt_reserve_iova(struct io_pagetable *iopt, unsigned long start,  in iopt_reserve_iova()  argument
836 	lockdep_assert_held_write(&iopt->iova_rwsem);  in iopt_reserve_iova()
838 	if (iopt_area_iter_first(iopt, start, last) ||  in iopt_reserve_iova()
839 	    iopt_allowed_iter_first(iopt, start, last))  in iopt_reserve_iova()
848 	interval_tree_insert(&reserved->node, &iopt->reserved_itree);  in iopt_reserve_iova()
852 static void __iopt_remove_reserved_iova(struct io_pagetable *iopt, void *owner)  in __iopt_remove_reserved_iova()  argument
856 	lockdep_assert_held_write(&iopt->iova_rwsem);  in __iopt_remove_reserved_iova()
858 	for (reserved = iopt_reserved_iter_first(iopt, 0, ULONG_MAX); reserved;  in __iopt_remove_reserved_iova()
864 					     &iopt->reserved_itree);  in __iopt_remove_reserved_iova()
870 void iopt_remove_reserved_iova(struct io_pagetable *iopt, void *owner)  in iopt_remove_reserved_iova()  argument
872 	down_write(&iopt->iova_rwsem);  in iopt_remove_reserved_iova()
873 	__iopt_remove_reserved_iova(iopt, owner);  in iopt_remove_reserved_iova()
874 	up_write(&iopt->iova_rwsem);  in iopt_remove_reserved_iova()
877 void iopt_init_table(struct io_pagetable *iopt)  in iopt_init_table()  argument
879 	init_rwsem(&iopt->iova_rwsem);  in iopt_init_table()
880 	init_rwsem(&iopt->domains_rwsem);  in iopt_init_table()
881 	iopt->area_itree = RB_ROOT_CACHED;  in iopt_init_table()
882 	iopt->allowed_itree = RB_ROOT_CACHED;  in iopt_init_table()
883 	iopt->reserved_itree = RB_ROOT_CACHED;  in iopt_init_table()
884 	xa_init_flags(&iopt->domains, XA_FLAGS_ACCOUNT);  in iopt_init_table()
885 	xa_init_flags(&iopt->access_list, XA_FLAGS_ALLOC);  in iopt_init_table()
892 	iopt->iova_alignment = 1;  in iopt_init_table()
895 void iopt_destroy_table(struct io_pagetable *iopt)  in iopt_destroy_table()  argument
900 		iopt_remove_reserved_iova(iopt, NULL);  in iopt_destroy_table()
902 	while ((node = interval_tree_iter_first(&iopt->allowed_itree, 0,  in iopt_destroy_table()
904 		interval_tree_remove(node, &iopt->allowed_itree);  in iopt_destroy_table()
908 	WARN_ON(!RB_EMPTY_ROOT(&iopt->reserved_itree.rb_root));  in iopt_destroy_table()
909 	WARN_ON(!xa_empty(&iopt->domains));  in iopt_destroy_table()
910 	WARN_ON(!xa_empty(&iopt->access_list));  in iopt_destroy_table()
911 	WARN_ON(!RB_EMPTY_ROOT(&iopt->area_itree.rb_root));  in iopt_destroy_table()
923 static void iopt_unfill_domain(struct io_pagetable *iopt,  in iopt_unfill_domain()  argument
928 	lockdep_assert_held(&iopt->iova_rwsem);  in iopt_unfill_domain()
929 	lockdep_assert_held_write(&iopt->domains_rwsem);  in iopt_unfill_domain()
935 	if (iopt->next_domain_id != 0) {  in iopt_unfill_domain()
938 			xa_load(&iopt->domains, 0);  in iopt_unfill_domain()
940 		for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area;  in iopt_unfill_domain()
959 	for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area;  in iopt_unfill_domain()
983 static int iopt_fill_domain(struct io_pagetable *iopt,  in iopt_fill_domain()  argument
990 	lockdep_assert_held(&iopt->iova_rwsem);  in iopt_fill_domain()
991 	lockdep_assert_held_write(&iopt->domains_rwsem);  in iopt_fill_domain()
993 	for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area;  in iopt_fill_domain()
1007 			WARN_ON(iopt->next_domain_id != 0);  in iopt_fill_domain()
1018 	for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area;  in iopt_fill_domain()
1027 		if (iopt->next_domain_id == 0) {  in iopt_fill_domain()
1039 static int iopt_check_iova_alignment(struct io_pagetable *iopt,  in iopt_check_iova_alignment()  argument
1045 	lockdep_assert_held(&iopt->iova_rwsem);  in iopt_check_iova_alignment()
1046 	lockdep_assert_held(&iopt->domains_rwsem);  in iopt_check_iova_alignment()
1048 	for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area;  in iopt_check_iova_alignment()
1059 		xa_for_each(&iopt->access_list, index, access)  in iopt_check_iova_alignment()
1067 int iopt_table_add_domain(struct io_pagetable *iopt,  in iopt_table_add_domain()  argument
1076 	down_write(&iopt->domains_rwsem);  in iopt_table_add_domain()
1077 	down_write(&iopt->iova_rwsem);  in iopt_table_add_domain()
1079 	xa_for_each(&iopt->domains, index, iter_domain) {  in iopt_table_add_domain()
1096 				   iopt->iova_alignment);  in iopt_table_add_domain()
1101 	if (new_iova_alignment != iopt->iova_alignment) {  in iopt_table_add_domain()
1102 		rc = iopt_check_iova_alignment(iopt, new_iova_alignment);  in iopt_table_add_domain()
1109 		rc = iopt_reserve_iova(iopt, 0, geometry->aperture_start - 1,  in iopt_table_add_domain()
1115 		rc = iopt_reserve_iova(iopt, geometry->aperture_end + 1,  in iopt_table_add_domain()
1121 	rc = xa_reserve(&iopt->domains, iopt->next_domain_id, GFP_KERNEL);  in iopt_table_add_domain()
1125 	rc = iopt_fill_domain(iopt, domain);  in iopt_table_add_domain()
1129 	iopt->iova_alignment = new_iova_alignment;  in iopt_table_add_domain()
1130 	xa_store(&iopt->domains, iopt->next_domain_id, domain, GFP_KERNEL);  in iopt_table_add_domain()
1131 	iopt->next_domain_id++;  in iopt_table_add_domain()
1132 	up_write(&iopt->iova_rwsem);  in iopt_table_add_domain()
1133 	up_write(&iopt->domains_rwsem);  in iopt_table_add_domain()
1136 	xa_release(&iopt->domains, iopt->next_domain_id);  in iopt_table_add_domain()
1138 	__iopt_remove_reserved_iova(iopt, domain);  in iopt_table_add_domain()
1140 	up_write(&iopt->iova_rwsem);  in iopt_table_add_domain()
1141 	up_write(&iopt->domains_rwsem);  in iopt_table_add_domain()
1145 static int iopt_calculate_iova_alignment(struct io_pagetable *iopt)  in iopt_calculate_iova_alignment()  argument
1152 	lockdep_assert_held_write(&iopt->iova_rwsem);  in iopt_calculate_iova_alignment()
1153 	lockdep_assert_held(&iopt->domains_rwsem);  in iopt_calculate_iova_alignment()
1156 	if (iopt->disable_large_pages)  in iopt_calculate_iova_alignment()
1161 	xa_for_each(&iopt->domains, index, domain)  in iopt_calculate_iova_alignment()
1165 	xa_for_each(&iopt->access_list, index, access)  in iopt_calculate_iova_alignment()
1170 	if (new_iova_alignment > iopt->iova_alignment) {  in iopt_calculate_iova_alignment()
1173 		rc = iopt_check_iova_alignment(iopt, new_iova_alignment);  in iopt_calculate_iova_alignment()
1177 	iopt->iova_alignment = new_iova_alignment;  in iopt_calculate_iova_alignment()
1181 void iopt_table_remove_domain(struct io_pagetable *iopt,  in iopt_table_remove_domain()  argument
1187 	down_write(&iopt->domains_rwsem);  in iopt_table_remove_domain()
1188 	down_write(&iopt->iova_rwsem);  in iopt_table_remove_domain()
1190 	xa_for_each(&iopt->domains, index, iter_domain)  in iopt_table_remove_domain()
1193 	if (WARN_ON(iter_domain != domain) || index >= iopt->next_domain_id)  in iopt_table_remove_domain()
1200 	iopt->next_domain_id--;  in iopt_table_remove_domain()
1201 	iter_domain = xa_erase(&iopt->domains, iopt->next_domain_id);  in iopt_table_remove_domain()
1202 	if (index != iopt->next_domain_id)  in iopt_table_remove_domain()
1203 		xa_store(&iopt->domains, index, iter_domain, GFP_KERNEL);  in iopt_table_remove_domain()
1205 	iopt_unfill_domain(iopt, domain);  in iopt_table_remove_domain()
1206 	__iopt_remove_reserved_iova(iopt, domain);  in iopt_table_remove_domain()
1208 	WARN_ON(iopt_calculate_iova_alignment(iopt));  in iopt_table_remove_domain()
1210 	up_write(&iopt->iova_rwsem);  in iopt_table_remove_domain()
1211 	up_write(&iopt->domains_rwsem);  in iopt_table_remove_domain()
1225 	unsigned long alignment = area->iopt->iova_alignment;  in iopt_area_split()
1229 	struct io_pagetable *iopt = area->iopt;  in iopt_area_split()  local
1235 	lockdep_assert_held_write(&iopt->iova_rwsem);  in iopt_area_split()
1271 	if (area->storage_domain && !iopt->disable_large_pages) {  in iopt_area_split()
1276 	interval_tree_remove(&area->node, &iopt->area_itree);  in iopt_area_split()
1277 	rc = iopt_insert_area(iopt, lhs, area->pages, start_iova,  in iopt_area_split()
1284 	rc = iopt_insert_area(iopt, rhs, area->pages, new_start,  in iopt_area_split()
1315 	interval_tree_remove(&lhs->node, &iopt->area_itree);  in iopt_area_split()
1317 	interval_tree_insert(&area->node, &iopt->area_itree);  in iopt_area_split()
1326 int iopt_cut_iova(struct io_pagetable *iopt, unsigned long *iovas,  in iopt_cut_iova()  argument
1332 	down_write(&iopt->iova_rwsem);  in iopt_cut_iova()
1336 		area = iopt_area_iter_first(iopt, iovas[i], iovas[i]);  in iopt_cut_iova()
1343 	up_write(&iopt->iova_rwsem);  in iopt_cut_iova()
1347 void iopt_enable_large_pages(struct io_pagetable *iopt)  in iopt_enable_large_pages()  argument
1351 	down_write(&iopt->domains_rwsem);  in iopt_enable_large_pages()
1352 	down_write(&iopt->iova_rwsem);  in iopt_enable_large_pages()
1353 	WRITE_ONCE(iopt->disable_large_pages, false);  in iopt_enable_large_pages()
1354 	rc = iopt_calculate_iova_alignment(iopt);  in iopt_enable_large_pages()
1356 	up_write(&iopt->iova_rwsem);  in iopt_enable_large_pages()
1357 	up_write(&iopt->domains_rwsem);  in iopt_enable_large_pages()
1360 int iopt_disable_large_pages(struct io_pagetable *iopt)  in iopt_disable_large_pages()  argument
1364 	down_write(&iopt->domains_rwsem);  in iopt_disable_large_pages()
1365 	down_write(&iopt->iova_rwsem);  in iopt_disable_large_pages()
1366 	if (iopt->disable_large_pages)  in iopt_disable_large_pages()
1370 	if (!xa_empty(&iopt->domains) &&  in iopt_disable_large_pages()
1371 	    !RB_EMPTY_ROOT(&iopt->area_itree.rb_root)) {  in iopt_disable_large_pages()
1376 	WRITE_ONCE(iopt->disable_large_pages, true);  in iopt_disable_large_pages()
1377 	rc = iopt_calculate_iova_alignment(iopt);  in iopt_disable_large_pages()
1379 		WRITE_ONCE(iopt->disable_large_pages, false);  in iopt_disable_large_pages()
1381 	up_write(&iopt->iova_rwsem);  in iopt_disable_large_pages()
1382 	up_write(&iopt->domains_rwsem);  in iopt_disable_large_pages()
1386 int iopt_add_access(struct io_pagetable *iopt, struct iommufd_access *access)  in iopt_add_access()  argument
1391 	down_write(&iopt->domains_rwsem);  in iopt_add_access()
1392 	down_write(&iopt->iova_rwsem);  in iopt_add_access()
1393 	rc = xa_alloc(&iopt->access_list, &new_id, access, xa_limit_16b,  in iopt_add_access()
1399 	rc = iopt_calculate_iova_alignment(iopt);  in iopt_add_access()
1401 		xa_erase(&iopt->access_list, new_id);  in iopt_add_access()
1407 	up_write(&iopt->iova_rwsem);  in iopt_add_access()
1408 	up_write(&iopt->domains_rwsem);  in iopt_add_access()
1412 void iopt_remove_access(struct io_pagetable *iopt,  in iopt_remove_access()  argument
1416 	down_write(&iopt->domains_rwsem);  in iopt_remove_access()
1417 	down_write(&iopt->iova_rwsem);  in iopt_remove_access()
1418 	WARN_ON(xa_erase(&iopt->access_list, iopt_access_list_id) != access);  in iopt_remove_access()
1419 	WARN_ON(iopt_calculate_iova_alignment(iopt));  in iopt_remove_access()
1420 	up_write(&iopt->iova_rwsem);  in iopt_remove_access()
1421 	up_write(&iopt->domains_rwsem);  in iopt_remove_access()
1425 int iopt_table_enforce_dev_resv_regions(struct io_pagetable *iopt,  in iopt_table_enforce_dev_resv_regions()  argument
1438 	down_write(&iopt->iova_rwsem);  in iopt_table_enforce_dev_resv_regions()
1453 		rc = iopt_reserve_iova(iopt, resv->start,  in iopt_table_enforce_dev_resv_regions()
1469 	__iopt_remove_reserved_iova(iopt, dev);  in iopt_table_enforce_dev_resv_regions()
1472 	up_write(&iopt->iova_rwsem);  in iopt_table_enforce_dev_resv_regions()