1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES 3 */ 4 #ifndef __IOMMUFD_PRIVATE_H 5 #define __IOMMUFD_PRIVATE_H 6 7 #include <linux/rwsem.h> 8 #include <linux/xarray.h> 9 #include <linux/refcount.h> 10 #include <linux/uaccess.h> 11 #include <linux/iommu.h> 12 #include <linux/iova_bitmap.h> 13 #include <uapi/linux/iommufd.h> 14 15 struct iommu_domain; 16 struct iommu_group; 17 struct iommu_option; 18 struct iommufd_device; 19 20 struct iommufd_ctx { 21 struct file *file; 22 struct xarray objects; 23 struct xarray groups; 24 25 u8 account_mode; 26 /* Compatibility with VFIO no iommu */ 27 u8 no_iommu_mode; 28 struct iommufd_ioas *vfio_ioas; 29 }; 30 31 /* 32 * The IOVA to PFN map. The map automatically copies the PFNs into multiple 33 * domains and permits sharing of PFNs between io_pagetable instances. This 34 * supports both a design where IOAS's are 1:1 with a domain (eg because the 35 * domain is HW customized), or where the IOAS is 1:N with multiple generic 36 * domains. The io_pagetable holds an interval tree of iopt_areas which point 37 * to shared iopt_pages which hold the pfns mapped to the page table. 38 * 39 * The locking order is domains_rwsem -> iova_rwsem -> pages::mutex 40 */ 41 struct io_pagetable { 42 struct rw_semaphore domains_rwsem; 43 struct xarray domains; 44 struct xarray access_list; 45 unsigned int next_domain_id; 46 47 struct rw_semaphore iova_rwsem; 48 struct rb_root_cached area_itree; 49 /* IOVA that cannot become reserved, struct iopt_allowed */ 50 struct rb_root_cached allowed_itree; 51 /* IOVA that cannot be allocated, struct iopt_reserved */ 52 struct rb_root_cached reserved_itree; 53 u8 disable_large_pages; 54 unsigned long iova_alignment; 55 }; 56 57 void iopt_init_table(struct io_pagetable *iopt); 58 void iopt_destroy_table(struct io_pagetable *iopt); 59 int iopt_get_pages(struct io_pagetable *iopt, unsigned long iova, 60 unsigned long length, struct list_head *pages_list); 61 void iopt_free_pages_list(struct list_head *pages_list); 62 enum { 63 IOPT_ALLOC_IOVA = 1 << 0, 64 }; 65 int iopt_map_user_pages(struct iommufd_ctx *ictx, struct io_pagetable *iopt, 66 unsigned long *iova, void __user *uptr, 67 unsigned long length, int iommu_prot, 68 unsigned int flags); 69 int iopt_map_pages(struct io_pagetable *iopt, struct list_head *pages_list, 70 unsigned long length, unsigned long *dst_iova, 71 int iommu_prot, unsigned int flags); 72 int iopt_unmap_iova(struct io_pagetable *iopt, unsigned long iova, 73 unsigned long length, unsigned long *unmapped); 74 int iopt_unmap_all(struct io_pagetable *iopt, unsigned long *unmapped); 75 76 int iopt_read_and_clear_dirty_data(struct io_pagetable *iopt, 77 struct iommu_domain *domain, 78 unsigned long flags, 79 struct iommu_hwpt_get_dirty_bitmap *bitmap); 80 int iopt_set_dirty_tracking(struct io_pagetable *iopt, 81 struct iommu_domain *domain, bool enable); 82 83 void iommufd_access_notify_unmap(struct io_pagetable *iopt, unsigned long iova, 84 unsigned long length); 85 int iopt_table_add_domain(struct io_pagetable *iopt, 86 struct iommu_domain *domain); 87 void iopt_table_remove_domain(struct io_pagetable *iopt, 88 struct iommu_domain *domain); 89 int iopt_table_enforce_dev_resv_regions(struct io_pagetable *iopt, 90 struct device *dev, 91 phys_addr_t *sw_msi_start); 92 int iopt_set_allow_iova(struct io_pagetable *iopt, 93 struct rb_root_cached *allowed_iova); 94 int iopt_reserve_iova(struct io_pagetable *iopt, unsigned long start, 95 unsigned long last, void *owner); 96 void iopt_remove_reserved_iova(struct io_pagetable *iopt, void *owner); 97 int iopt_cut_iova(struct io_pagetable *iopt, unsigned long *iovas, 98 size_t num_iovas); 99 void iopt_enable_large_pages(struct io_pagetable *iopt); 100 int iopt_disable_large_pages(struct io_pagetable *iopt); 101 102 struct iommufd_ucmd { 103 struct iommufd_ctx *ictx; 104 void __user *ubuffer; 105 u32 user_size; 106 void *cmd; 107 }; 108 109 int iommufd_vfio_ioctl(struct iommufd_ctx *ictx, unsigned int cmd, 110 unsigned long arg); 111 112 /* Copy the response in ucmd->cmd back to userspace. */ 113 static inline int iommufd_ucmd_respond(struct iommufd_ucmd *ucmd, 114 size_t cmd_len) 115 { 116 if (copy_to_user(ucmd->ubuffer, ucmd->cmd, 117 min_t(size_t, ucmd->user_size, cmd_len))) 118 return -EFAULT; 119 return 0; 120 } 121 122 enum iommufd_object_type { 123 IOMMUFD_OBJ_NONE, 124 IOMMUFD_OBJ_ANY = IOMMUFD_OBJ_NONE, 125 IOMMUFD_OBJ_DEVICE, 126 IOMMUFD_OBJ_HWPT_PAGING, 127 IOMMUFD_OBJ_HWPT_NESTED, 128 IOMMUFD_OBJ_IOAS, 129 IOMMUFD_OBJ_ACCESS, 130 #ifdef CONFIG_IOMMUFD_TEST 131 IOMMUFD_OBJ_SELFTEST, 132 #endif 133 IOMMUFD_OBJ_MAX, 134 }; 135 136 /* Base struct for all objects with a userspace ID handle. */ 137 struct iommufd_object { 138 struct rw_semaphore destroy_rwsem; 139 refcount_t users; 140 enum iommufd_object_type type; 141 unsigned int id; 142 }; 143 144 static inline bool iommufd_lock_obj(struct iommufd_object *obj) 145 { 146 if (!down_read_trylock(&obj->destroy_rwsem)) 147 return false; 148 if (!refcount_inc_not_zero(&obj->users)) { 149 up_read(&obj->destroy_rwsem); 150 return false; 151 } 152 return true; 153 } 154 155 struct iommufd_object *iommufd_get_object(struct iommufd_ctx *ictx, u32 id, 156 enum iommufd_object_type type); 157 static inline void iommufd_put_object(struct iommufd_object *obj) 158 { 159 refcount_dec(&obj->users); 160 up_read(&obj->destroy_rwsem); 161 } 162 163 void iommufd_object_abort(struct iommufd_ctx *ictx, struct iommufd_object *obj); 164 void iommufd_object_abort_and_destroy(struct iommufd_ctx *ictx, 165 struct iommufd_object *obj); 166 void iommufd_object_finalize(struct iommufd_ctx *ictx, 167 struct iommufd_object *obj); 168 void __iommufd_object_destroy_user(struct iommufd_ctx *ictx, 169 struct iommufd_object *obj, bool allow_fail); 170 static inline void iommufd_object_destroy_user(struct iommufd_ctx *ictx, 171 struct iommufd_object *obj) 172 { 173 __iommufd_object_destroy_user(ictx, obj, false); 174 } 175 static inline void iommufd_object_deref_user(struct iommufd_ctx *ictx, 176 struct iommufd_object *obj) 177 { 178 __iommufd_object_destroy_user(ictx, obj, true); 179 } 180 181 struct iommufd_object *_iommufd_object_alloc(struct iommufd_ctx *ictx, 182 size_t size, 183 enum iommufd_object_type type); 184 185 #define __iommufd_object_alloc(ictx, ptr, type, obj) \ 186 container_of(_iommufd_object_alloc( \ 187 ictx, \ 188 sizeof(*(ptr)) + BUILD_BUG_ON_ZERO( \ 189 offsetof(typeof(*(ptr)), \ 190 obj) != 0), \ 191 type), \ 192 typeof(*(ptr)), obj) 193 194 #define iommufd_object_alloc(ictx, ptr, type) \ 195 __iommufd_object_alloc(ictx, ptr, type, obj) 196 197 /* 198 * The IO Address Space (IOAS) pagetable is a virtual page table backed by the 199 * io_pagetable object. It is a user controlled mapping of IOVA -> PFNs. The 200 * mapping is copied into all of the associated domains and made available to 201 * in-kernel users. 202 * 203 * Every iommu_domain that is created is wrapped in a iommufd_hw_pagetable 204 * object. When we go to attach a device to an IOAS we need to get an 205 * iommu_domain and wrapping iommufd_hw_pagetable for it. 206 * 207 * An iommu_domain & iommfd_hw_pagetable will be automatically selected 208 * for a device based on the hwpt_list. If no suitable iommu_domain 209 * is found a new iommu_domain will be created. 210 */ 211 struct iommufd_ioas { 212 struct iommufd_object obj; 213 struct io_pagetable iopt; 214 struct mutex mutex; 215 struct list_head hwpt_list; 216 }; 217 218 static inline struct iommufd_ioas *iommufd_get_ioas(struct iommufd_ctx *ictx, 219 u32 id) 220 { 221 return container_of(iommufd_get_object(ictx, id, 222 IOMMUFD_OBJ_IOAS), 223 struct iommufd_ioas, obj); 224 } 225 226 struct iommufd_ioas *iommufd_ioas_alloc(struct iommufd_ctx *ictx); 227 int iommufd_ioas_alloc_ioctl(struct iommufd_ucmd *ucmd); 228 void iommufd_ioas_destroy(struct iommufd_object *obj); 229 int iommufd_ioas_iova_ranges(struct iommufd_ucmd *ucmd); 230 int iommufd_ioas_allow_iovas(struct iommufd_ucmd *ucmd); 231 int iommufd_ioas_map(struct iommufd_ucmd *ucmd); 232 int iommufd_ioas_copy(struct iommufd_ucmd *ucmd); 233 int iommufd_ioas_unmap(struct iommufd_ucmd *ucmd); 234 int iommufd_ioas_option(struct iommufd_ucmd *ucmd); 235 int iommufd_option_rlimit_mode(struct iommu_option *cmd, 236 struct iommufd_ctx *ictx); 237 238 int iommufd_vfio_ioas(struct iommufd_ucmd *ucmd); 239 int iommufd_check_iova_range(struct io_pagetable *iopt, 240 struct iommu_hwpt_get_dirty_bitmap *bitmap); 241 242 /* 243 * A HW pagetable is called an iommu_domain inside the kernel. This user object 244 * allows directly creating and inspecting the domains. Domains that have kernel 245 * owned page tables will be associated with an iommufd_ioas that provides the 246 * IOVA to PFN map. 247 */ 248 struct iommufd_hw_pagetable { 249 struct iommufd_object obj; 250 struct iommu_domain *domain; 251 }; 252 253 struct iommufd_hwpt_paging { 254 struct iommufd_hw_pagetable common; 255 struct iommufd_ioas *ioas; 256 bool auto_domain : 1; 257 bool enforce_cache_coherency : 1; 258 bool msi_cookie : 1; 259 bool nest_parent : 1; 260 /* Head at iommufd_ioas::hwpt_list */ 261 struct list_head hwpt_item; 262 }; 263 264 struct iommufd_hwpt_nested { 265 struct iommufd_hw_pagetable common; 266 struct iommufd_hwpt_paging *parent; 267 }; 268 269 static inline bool hwpt_is_paging(struct iommufd_hw_pagetable *hwpt) 270 { 271 return hwpt->obj.type == IOMMUFD_OBJ_HWPT_PAGING; 272 } 273 274 static inline struct iommufd_hwpt_paging * 275 to_hwpt_paging(struct iommufd_hw_pagetable *hwpt) 276 { 277 return container_of(hwpt, struct iommufd_hwpt_paging, common); 278 } 279 280 static inline struct iommufd_hwpt_paging * 281 iommufd_get_hwpt_paging(struct iommufd_ucmd *ucmd, u32 id) 282 { 283 return container_of(iommufd_get_object(ucmd->ictx, id, 284 IOMMUFD_OBJ_HWPT_PAGING), 285 struct iommufd_hwpt_paging, common.obj); 286 } 287 int iommufd_hwpt_set_dirty_tracking(struct iommufd_ucmd *ucmd); 288 int iommufd_hwpt_get_dirty_bitmap(struct iommufd_ucmd *ucmd); 289 290 struct iommufd_hwpt_paging * 291 iommufd_hwpt_paging_alloc(struct iommufd_ctx *ictx, struct iommufd_ioas *ioas, 292 struct iommufd_device *idev, u32 flags, 293 bool immediate_attach, 294 const struct iommu_user_data *user_data); 295 int iommufd_hw_pagetable_attach(struct iommufd_hw_pagetable *hwpt, 296 struct iommufd_device *idev); 297 struct iommufd_hw_pagetable * 298 iommufd_hw_pagetable_detach(struct iommufd_device *idev); 299 void iommufd_hwpt_paging_destroy(struct iommufd_object *obj); 300 void iommufd_hwpt_paging_abort(struct iommufd_object *obj); 301 void iommufd_hwpt_nested_destroy(struct iommufd_object *obj); 302 void iommufd_hwpt_nested_abort(struct iommufd_object *obj); 303 int iommufd_hwpt_alloc(struct iommufd_ucmd *ucmd); 304 305 static inline void iommufd_hw_pagetable_put(struct iommufd_ctx *ictx, 306 struct iommufd_hw_pagetable *hwpt) 307 { 308 if (hwpt->obj.type == IOMMUFD_OBJ_HWPT_PAGING) { 309 struct iommufd_hwpt_paging *hwpt_paging = to_hwpt_paging(hwpt); 310 311 lockdep_assert_not_held(&hwpt_paging->ioas->mutex); 312 313 if (hwpt_paging->auto_domain) { 314 iommufd_object_deref_user(ictx, &hwpt->obj); 315 return; 316 } 317 } 318 refcount_dec(&hwpt->obj.users); 319 } 320 321 struct iommufd_group { 322 struct kref ref; 323 struct mutex lock; 324 struct iommufd_ctx *ictx; 325 struct iommu_group *group; 326 struct iommufd_hw_pagetable *hwpt; 327 struct list_head device_list; 328 phys_addr_t sw_msi_start; 329 }; 330 331 /* 332 * A iommufd_device object represents the binding relationship between a 333 * consuming driver and the iommufd. These objects are created/destroyed by 334 * external drivers, not by userspace. 335 */ 336 struct iommufd_device { 337 struct iommufd_object obj; 338 struct iommufd_ctx *ictx; 339 struct iommufd_group *igroup; 340 struct list_head group_item; 341 /* always the physical device */ 342 struct device *dev; 343 bool enforce_cache_coherency; 344 }; 345 346 static inline struct iommufd_device * 347 iommufd_get_device(struct iommufd_ucmd *ucmd, u32 id) 348 { 349 return container_of(iommufd_get_object(ucmd->ictx, id, 350 IOMMUFD_OBJ_DEVICE), 351 struct iommufd_device, obj); 352 } 353 354 void iommufd_device_destroy(struct iommufd_object *obj); 355 int iommufd_get_hw_info(struct iommufd_ucmd *ucmd); 356 357 struct iommufd_access { 358 struct iommufd_object obj; 359 struct iommufd_ctx *ictx; 360 struct iommufd_ioas *ioas; 361 struct iommufd_ioas *ioas_unpin; 362 struct mutex ioas_lock; 363 const struct iommufd_access_ops *ops; 364 void *data; 365 unsigned long iova_alignment; 366 u32 iopt_access_list_id; 367 }; 368 369 int iopt_add_access(struct io_pagetable *iopt, struct iommufd_access *access); 370 void iopt_remove_access(struct io_pagetable *iopt, 371 struct iommufd_access *access, 372 u32 iopt_access_list_id); 373 void iommufd_access_destroy_object(struct iommufd_object *obj); 374 375 #ifdef CONFIG_IOMMUFD_TEST 376 int iommufd_test(struct iommufd_ucmd *ucmd); 377 void iommufd_selftest_destroy(struct iommufd_object *obj); 378 extern size_t iommufd_test_memory_limit; 379 void iommufd_test_syz_conv_iova_id(struct iommufd_ucmd *ucmd, 380 unsigned int ioas_id, u64 *iova, u32 *flags); 381 bool iommufd_should_fail(void); 382 int __init iommufd_test_init(void); 383 void iommufd_test_exit(void); 384 bool iommufd_selftest_is_mock_dev(struct device *dev); 385 #else 386 static inline void iommufd_test_syz_conv_iova_id(struct iommufd_ucmd *ucmd, 387 unsigned int ioas_id, 388 u64 *iova, u32 *flags) 389 { 390 } 391 static inline bool iommufd_should_fail(void) 392 { 393 return false; 394 } 395 static inline int __init iommufd_test_init(void) 396 { 397 return 0; 398 } 399 static inline void iommufd_test_exit(void) 400 { 401 } 402 static inline bool iommufd_selftest_is_mock_dev(struct device *dev) 403 { 404 return false; 405 } 406 #endif 407 #endif 408