1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES 3 */ 4 #ifndef __IOMMUFD_PRIVATE_H 5 #define __IOMMUFD_PRIVATE_H 6 7 #include <linux/rwsem.h> 8 #include <linux/xarray.h> 9 #include <linux/refcount.h> 10 #include <linux/uaccess.h> 11 12 struct iommu_domain; 13 struct iommu_group; 14 struct iommu_option; 15 struct iommufd_device; 16 17 struct iommufd_ctx { 18 struct file *file; 19 struct xarray objects; 20 struct xarray groups; 21 22 u8 account_mode; 23 /* Compatibility with VFIO no iommu */ 24 u8 no_iommu_mode; 25 struct iommufd_ioas *vfio_ioas; 26 }; 27 28 /* 29 * The IOVA to PFN map. The map automatically copies the PFNs into multiple 30 * domains and permits sharing of PFNs between io_pagetable instances. This 31 * supports both a design where IOAS's are 1:1 with a domain (eg because the 32 * domain is HW customized), or where the IOAS is 1:N with multiple generic 33 * domains. The io_pagetable holds an interval tree of iopt_areas which point 34 * to shared iopt_pages which hold the pfns mapped to the page table. 35 * 36 * The locking order is domains_rwsem -> iova_rwsem -> pages::mutex 37 */ 38 struct io_pagetable { 39 struct rw_semaphore domains_rwsem; 40 struct xarray domains; 41 struct xarray access_list; 42 unsigned int next_domain_id; 43 44 struct rw_semaphore iova_rwsem; 45 struct rb_root_cached area_itree; 46 /* IOVA that cannot become reserved, struct iopt_allowed */ 47 struct rb_root_cached allowed_itree; 48 /* IOVA that cannot be allocated, struct iopt_reserved */ 49 struct rb_root_cached reserved_itree; 50 u8 disable_large_pages; 51 unsigned long iova_alignment; 52 }; 53 54 void iopt_init_table(struct io_pagetable *iopt); 55 void iopt_destroy_table(struct io_pagetable *iopt); 56 int iopt_get_pages(struct io_pagetable *iopt, unsigned long iova, 57 unsigned long length, struct list_head *pages_list); 58 void iopt_free_pages_list(struct list_head *pages_list); 59 enum { 60 IOPT_ALLOC_IOVA = 1 << 0, 61 }; 62 int iopt_map_user_pages(struct iommufd_ctx *ictx, struct io_pagetable *iopt, 63 unsigned long *iova, void __user *uptr, 64 unsigned long length, int iommu_prot, 65 unsigned int flags); 66 int iopt_map_pages(struct io_pagetable *iopt, struct list_head *pages_list, 67 unsigned long length, unsigned long *dst_iova, 68 int iommu_prot, unsigned int flags); 69 int iopt_unmap_iova(struct io_pagetable *iopt, unsigned long iova, 70 unsigned long length, unsigned long *unmapped); 71 int iopt_unmap_all(struct io_pagetable *iopt, unsigned long *unmapped); 72 73 void iommufd_access_notify_unmap(struct io_pagetable *iopt, unsigned long iova, 74 unsigned long length); 75 int iopt_table_add_domain(struct io_pagetable *iopt, 76 struct iommu_domain *domain); 77 void iopt_table_remove_domain(struct io_pagetable *iopt, 78 struct iommu_domain *domain); 79 int iopt_table_enforce_dev_resv_regions(struct io_pagetable *iopt, 80 struct device *dev, 81 phys_addr_t *sw_msi_start); 82 int iopt_set_allow_iova(struct io_pagetable *iopt, 83 struct rb_root_cached *allowed_iova); 84 int iopt_reserve_iova(struct io_pagetable *iopt, unsigned long start, 85 unsigned long last, void *owner); 86 void iopt_remove_reserved_iova(struct io_pagetable *iopt, void *owner); 87 int iopt_cut_iova(struct io_pagetable *iopt, unsigned long *iovas, 88 size_t num_iovas); 89 void iopt_enable_large_pages(struct io_pagetable *iopt); 90 int iopt_disable_large_pages(struct io_pagetable *iopt); 91 92 struct iommufd_ucmd { 93 struct iommufd_ctx *ictx; 94 void __user *ubuffer; 95 u32 user_size; 96 void *cmd; 97 }; 98 99 int iommufd_vfio_ioctl(struct iommufd_ctx *ictx, unsigned int cmd, 100 unsigned long arg); 101 102 /* Copy the response in ucmd->cmd back to userspace. */ 103 static inline int iommufd_ucmd_respond(struct iommufd_ucmd *ucmd, 104 size_t cmd_len) 105 { 106 if (copy_to_user(ucmd->ubuffer, ucmd->cmd, 107 min_t(size_t, ucmd->user_size, cmd_len))) 108 return -EFAULT; 109 return 0; 110 } 111 112 enum iommufd_object_type { 113 IOMMUFD_OBJ_NONE, 114 IOMMUFD_OBJ_ANY = IOMMUFD_OBJ_NONE, 115 IOMMUFD_OBJ_DEVICE, 116 IOMMUFD_OBJ_HW_PAGETABLE, 117 IOMMUFD_OBJ_IOAS, 118 IOMMUFD_OBJ_ACCESS, 119 #ifdef CONFIG_IOMMUFD_TEST 120 IOMMUFD_OBJ_SELFTEST, 121 #endif 122 IOMMUFD_OBJ_MAX, 123 }; 124 125 /* Base struct for all objects with a userspace ID handle. */ 126 struct iommufd_object { 127 struct rw_semaphore destroy_rwsem; 128 refcount_t users; 129 enum iommufd_object_type type; 130 unsigned int id; 131 }; 132 133 static inline bool iommufd_lock_obj(struct iommufd_object *obj) 134 { 135 if (!down_read_trylock(&obj->destroy_rwsem)) 136 return false; 137 if (!refcount_inc_not_zero(&obj->users)) { 138 up_read(&obj->destroy_rwsem); 139 return false; 140 } 141 return true; 142 } 143 144 struct iommufd_object *iommufd_get_object(struct iommufd_ctx *ictx, u32 id, 145 enum iommufd_object_type type); 146 static inline void iommufd_put_object(struct iommufd_object *obj) 147 { 148 refcount_dec(&obj->users); 149 up_read(&obj->destroy_rwsem); 150 } 151 152 void iommufd_object_abort(struct iommufd_ctx *ictx, struct iommufd_object *obj); 153 void iommufd_object_abort_and_destroy(struct iommufd_ctx *ictx, 154 struct iommufd_object *obj); 155 void iommufd_object_finalize(struct iommufd_ctx *ictx, 156 struct iommufd_object *obj); 157 void __iommufd_object_destroy_user(struct iommufd_ctx *ictx, 158 struct iommufd_object *obj, bool allow_fail); 159 static inline void iommufd_object_destroy_user(struct iommufd_ctx *ictx, 160 struct iommufd_object *obj) 161 { 162 __iommufd_object_destroy_user(ictx, obj, false); 163 } 164 static inline void iommufd_object_deref_user(struct iommufd_ctx *ictx, 165 struct iommufd_object *obj) 166 { 167 __iommufd_object_destroy_user(ictx, obj, true); 168 } 169 170 struct iommufd_object *_iommufd_object_alloc(struct iommufd_ctx *ictx, 171 size_t size, 172 enum iommufd_object_type type); 173 174 #define iommufd_object_alloc(ictx, ptr, type) \ 175 container_of(_iommufd_object_alloc( \ 176 ictx, \ 177 sizeof(*(ptr)) + BUILD_BUG_ON_ZERO( \ 178 offsetof(typeof(*(ptr)), \ 179 obj) != 0), \ 180 type), \ 181 typeof(*(ptr)), obj) 182 183 /* 184 * The IO Address Space (IOAS) pagetable is a virtual page table backed by the 185 * io_pagetable object. It is a user controlled mapping of IOVA -> PFNs. The 186 * mapping is copied into all of the associated domains and made available to 187 * in-kernel users. 188 * 189 * Every iommu_domain that is created is wrapped in a iommufd_hw_pagetable 190 * object. When we go to attach a device to an IOAS we need to get an 191 * iommu_domain and wrapping iommufd_hw_pagetable for it. 192 * 193 * An iommu_domain & iommfd_hw_pagetable will be automatically selected 194 * for a device based on the hwpt_list. If no suitable iommu_domain 195 * is found a new iommu_domain will be created. 196 */ 197 struct iommufd_ioas { 198 struct iommufd_object obj; 199 struct io_pagetable iopt; 200 struct mutex mutex; 201 struct list_head hwpt_list; 202 }; 203 204 static inline struct iommufd_ioas *iommufd_get_ioas(struct iommufd_ctx *ictx, 205 u32 id) 206 { 207 return container_of(iommufd_get_object(ictx, id, 208 IOMMUFD_OBJ_IOAS), 209 struct iommufd_ioas, obj); 210 } 211 212 struct iommufd_ioas *iommufd_ioas_alloc(struct iommufd_ctx *ictx); 213 int iommufd_ioas_alloc_ioctl(struct iommufd_ucmd *ucmd); 214 void iommufd_ioas_destroy(struct iommufd_object *obj); 215 int iommufd_ioas_iova_ranges(struct iommufd_ucmd *ucmd); 216 int iommufd_ioas_allow_iovas(struct iommufd_ucmd *ucmd); 217 int iommufd_ioas_map(struct iommufd_ucmd *ucmd); 218 int iommufd_ioas_copy(struct iommufd_ucmd *ucmd); 219 int iommufd_ioas_unmap(struct iommufd_ucmd *ucmd); 220 int iommufd_ioas_option(struct iommufd_ucmd *ucmd); 221 int iommufd_option_rlimit_mode(struct iommu_option *cmd, 222 struct iommufd_ctx *ictx); 223 224 int iommufd_vfio_ioas(struct iommufd_ucmd *ucmd); 225 226 /* 227 * A HW pagetable is called an iommu_domain inside the kernel. This user object 228 * allows directly creating and inspecting the domains. Domains that have kernel 229 * owned page tables will be associated with an iommufd_ioas that provides the 230 * IOVA to PFN map. 231 */ 232 struct iommufd_hw_pagetable { 233 struct iommufd_object obj; 234 struct iommufd_ioas *ioas; 235 struct iommu_domain *domain; 236 bool auto_domain : 1; 237 bool enforce_cache_coherency : 1; 238 bool msi_cookie : 1; 239 /* Head at iommufd_ioas::hwpt_list */ 240 struct list_head hwpt_item; 241 }; 242 243 struct iommufd_hw_pagetable * 244 iommufd_hw_pagetable_alloc(struct iommufd_ctx *ictx, struct iommufd_ioas *ioas, 245 struct iommufd_device *idev, bool immediate_attach); 246 int iommufd_hw_pagetable_enforce_cc(struct iommufd_hw_pagetable *hwpt); 247 int iommufd_hw_pagetable_attach(struct iommufd_hw_pagetable *hwpt, 248 struct iommufd_device *idev); 249 struct iommufd_hw_pagetable * 250 iommufd_hw_pagetable_detach(struct iommufd_device *idev); 251 void iommufd_hw_pagetable_destroy(struct iommufd_object *obj); 252 void iommufd_hw_pagetable_abort(struct iommufd_object *obj); 253 int iommufd_hwpt_alloc(struct iommufd_ucmd *ucmd); 254 255 static inline void iommufd_hw_pagetable_put(struct iommufd_ctx *ictx, 256 struct iommufd_hw_pagetable *hwpt) 257 { 258 lockdep_assert_not_held(&hwpt->ioas->mutex); 259 if (hwpt->auto_domain) 260 iommufd_object_deref_user(ictx, &hwpt->obj); 261 else 262 refcount_dec(&hwpt->obj.users); 263 } 264 265 struct iommufd_group { 266 struct kref ref; 267 struct mutex lock; 268 struct iommufd_ctx *ictx; 269 struct iommu_group *group; 270 struct iommufd_hw_pagetable *hwpt; 271 struct list_head device_list; 272 phys_addr_t sw_msi_start; 273 }; 274 275 /* 276 * A iommufd_device object represents the binding relationship between a 277 * consuming driver and the iommufd. These objects are created/destroyed by 278 * external drivers, not by userspace. 279 */ 280 struct iommufd_device { 281 struct iommufd_object obj; 282 struct iommufd_ctx *ictx; 283 struct iommufd_group *igroup; 284 struct list_head group_item; 285 /* always the physical device */ 286 struct device *dev; 287 bool enforce_cache_coherency; 288 }; 289 290 static inline struct iommufd_device * 291 iommufd_get_device(struct iommufd_ucmd *ucmd, u32 id) 292 { 293 return container_of(iommufd_get_object(ucmd->ictx, id, 294 IOMMUFD_OBJ_DEVICE), 295 struct iommufd_device, obj); 296 } 297 298 void iommufd_device_destroy(struct iommufd_object *obj); 299 int iommufd_get_hw_info(struct iommufd_ucmd *ucmd); 300 301 struct iommufd_access { 302 struct iommufd_object obj; 303 struct iommufd_ctx *ictx; 304 struct iommufd_ioas *ioas; 305 struct iommufd_ioas *ioas_unpin; 306 struct mutex ioas_lock; 307 const struct iommufd_access_ops *ops; 308 void *data; 309 unsigned long iova_alignment; 310 u32 iopt_access_list_id; 311 }; 312 313 int iopt_add_access(struct io_pagetable *iopt, struct iommufd_access *access); 314 void iopt_remove_access(struct io_pagetable *iopt, 315 struct iommufd_access *access, 316 u32 iopt_access_list_id); 317 void iommufd_access_destroy_object(struct iommufd_object *obj); 318 319 #ifdef CONFIG_IOMMUFD_TEST 320 int iommufd_test(struct iommufd_ucmd *ucmd); 321 void iommufd_selftest_destroy(struct iommufd_object *obj); 322 extern size_t iommufd_test_memory_limit; 323 void iommufd_test_syz_conv_iova_id(struct iommufd_ucmd *ucmd, 324 unsigned int ioas_id, u64 *iova, u32 *flags); 325 bool iommufd_should_fail(void); 326 int __init iommufd_test_init(void); 327 void iommufd_test_exit(void); 328 bool iommufd_selftest_is_mock_dev(struct device *dev); 329 #else 330 static inline void iommufd_test_syz_conv_iova_id(struct iommufd_ucmd *ucmd, 331 unsigned int ioas_id, 332 u64 *iova, u32 *flags) 333 { 334 } 335 static inline bool iommufd_should_fail(void) 336 { 337 return false; 338 } 339 static inline int __init iommufd_test_init(void) 340 { 341 return 0; 342 } 343 static inline void iommufd_test_exit(void) 344 { 345 } 346 static inline bool iommufd_selftest_is_mock_dev(struct device *dev) 347 { 348 return false; 349 } 350 #endif 351 #endif 352