1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES 3 */ 4 #ifndef __IOMMUFD_PRIVATE_H 5 #define __IOMMUFD_PRIVATE_H 6 7 #include <linux/rwsem.h> 8 #include <linux/xarray.h> 9 #include <linux/refcount.h> 10 #include <linux/uaccess.h> 11 12 struct iommu_domain; 13 struct iommu_group; 14 struct iommu_option; 15 16 struct iommufd_ctx { 17 struct file *file; 18 struct xarray objects; 19 20 u8 account_mode; 21 /* Compatibility with VFIO no iommu */ 22 u8 no_iommu_mode; 23 struct iommufd_ioas *vfio_ioas; 24 }; 25 26 /* 27 * The IOVA to PFN map. The map automatically copies the PFNs into multiple 28 * domains and permits sharing of PFNs between io_pagetable instances. This 29 * supports both a design where IOAS's are 1:1 with a domain (eg because the 30 * domain is HW customized), or where the IOAS is 1:N with multiple generic 31 * domains. The io_pagetable holds an interval tree of iopt_areas which point 32 * to shared iopt_pages which hold the pfns mapped to the page table. 33 * 34 * The locking order is domains_rwsem -> iova_rwsem -> pages::mutex 35 */ 36 struct io_pagetable { 37 struct rw_semaphore domains_rwsem; 38 struct xarray domains; 39 struct xarray access_list; 40 unsigned int next_domain_id; 41 42 struct rw_semaphore iova_rwsem; 43 struct rb_root_cached area_itree; 44 /* IOVA that cannot become reserved, struct iopt_allowed */ 45 struct rb_root_cached allowed_itree; 46 /* IOVA that cannot be allocated, struct iopt_reserved */ 47 struct rb_root_cached reserved_itree; 48 u8 disable_large_pages; 49 unsigned long iova_alignment; 50 }; 51 52 void iopt_init_table(struct io_pagetable *iopt); 53 void iopt_destroy_table(struct io_pagetable *iopt); 54 int iopt_get_pages(struct io_pagetable *iopt, unsigned long iova, 55 unsigned long length, struct list_head *pages_list); 56 void iopt_free_pages_list(struct list_head *pages_list); 57 enum { 58 IOPT_ALLOC_IOVA = 1 << 0, 59 }; 60 int iopt_map_user_pages(struct iommufd_ctx *ictx, struct io_pagetable *iopt, 61 unsigned long *iova, void __user *uptr, 62 unsigned long length, int iommu_prot, 63 unsigned int flags); 64 int iopt_map_pages(struct io_pagetable *iopt, struct list_head *pages_list, 65 unsigned long length, unsigned long *dst_iova, 66 int iommu_prot, unsigned int flags); 67 int iopt_unmap_iova(struct io_pagetable *iopt, unsigned long iova, 68 unsigned long length, unsigned long *unmapped); 69 int iopt_unmap_all(struct io_pagetable *iopt, unsigned long *unmapped); 70 71 void iommufd_access_notify_unmap(struct io_pagetable *iopt, unsigned long iova, 72 unsigned long length); 73 int iopt_table_add_domain(struct io_pagetable *iopt, 74 struct iommu_domain *domain); 75 void iopt_table_remove_domain(struct io_pagetable *iopt, 76 struct iommu_domain *domain); 77 int iopt_table_enforce_group_resv_regions(struct io_pagetable *iopt, 78 struct device *device, 79 struct iommu_group *group, 80 phys_addr_t *sw_msi_start); 81 int iopt_set_allow_iova(struct io_pagetable *iopt, 82 struct rb_root_cached *allowed_iova); 83 int iopt_reserve_iova(struct io_pagetable *iopt, unsigned long start, 84 unsigned long last, void *owner); 85 void iopt_remove_reserved_iova(struct io_pagetable *iopt, void *owner); 86 int iopt_cut_iova(struct io_pagetable *iopt, unsigned long *iovas, 87 size_t num_iovas); 88 void iopt_enable_large_pages(struct io_pagetable *iopt); 89 int iopt_disable_large_pages(struct io_pagetable *iopt); 90 91 struct iommufd_ucmd { 92 struct iommufd_ctx *ictx; 93 void __user *ubuffer; 94 u32 user_size; 95 void *cmd; 96 }; 97 98 int iommufd_vfio_ioctl(struct iommufd_ctx *ictx, unsigned int cmd, 99 unsigned long arg); 100 101 /* Copy the response in ucmd->cmd back to userspace. */ 102 static inline int iommufd_ucmd_respond(struct iommufd_ucmd *ucmd, 103 size_t cmd_len) 104 { 105 if (copy_to_user(ucmd->ubuffer, ucmd->cmd, 106 min_t(size_t, ucmd->user_size, cmd_len))) 107 return -EFAULT; 108 return 0; 109 } 110 111 enum iommufd_object_type { 112 IOMMUFD_OBJ_NONE, 113 IOMMUFD_OBJ_ANY = IOMMUFD_OBJ_NONE, 114 IOMMUFD_OBJ_DEVICE, 115 IOMMUFD_OBJ_HW_PAGETABLE, 116 IOMMUFD_OBJ_IOAS, 117 IOMMUFD_OBJ_ACCESS, 118 #ifdef CONFIG_IOMMUFD_TEST 119 IOMMUFD_OBJ_SELFTEST, 120 #endif 121 }; 122 123 /* Base struct for all objects with a userspace ID handle. */ 124 struct iommufd_object { 125 struct rw_semaphore destroy_rwsem; 126 refcount_t users; 127 enum iommufd_object_type type; 128 unsigned int id; 129 }; 130 131 static inline bool iommufd_lock_obj(struct iommufd_object *obj) 132 { 133 if (!down_read_trylock(&obj->destroy_rwsem)) 134 return false; 135 if (!refcount_inc_not_zero(&obj->users)) { 136 up_read(&obj->destroy_rwsem); 137 return false; 138 } 139 return true; 140 } 141 142 struct iommufd_object *iommufd_get_object(struct iommufd_ctx *ictx, u32 id, 143 enum iommufd_object_type type); 144 static inline void iommufd_put_object(struct iommufd_object *obj) 145 { 146 refcount_dec(&obj->users); 147 up_read(&obj->destroy_rwsem); 148 } 149 150 /** 151 * iommufd_ref_to_users() - Switch from destroy_rwsem to users refcount 152 * protection 153 * @obj - Object to release 154 * 155 * Objects have two refcount protections (destroy_rwsem and the refcount_t 156 * users). Holding either of these will prevent the object from being destroyed. 157 * 158 * Depending on the use case, one protection or the other is appropriate. In 159 * most cases references are being protected by the destroy_rwsem. This allows 160 * orderly destruction of the object because iommufd_object_destroy_user() will 161 * wait for it to become unlocked. However, as a rwsem, it cannot be held across 162 * a system call return. So cases that have longer term needs must switch 163 * to the weaker users refcount_t. 164 * 165 * With users protection iommufd_object_destroy_user() will return false, 166 * refusing to destroy the object, causing -EBUSY to userspace. 167 */ 168 static inline void iommufd_ref_to_users(struct iommufd_object *obj) 169 { 170 up_read(&obj->destroy_rwsem); 171 /* iommufd_lock_obj() obtains users as well */ 172 } 173 void iommufd_object_abort(struct iommufd_ctx *ictx, struct iommufd_object *obj); 174 void iommufd_object_abort_and_destroy(struct iommufd_ctx *ictx, 175 struct iommufd_object *obj); 176 void iommufd_object_finalize(struct iommufd_ctx *ictx, 177 struct iommufd_object *obj); 178 bool iommufd_object_destroy_user(struct iommufd_ctx *ictx, 179 struct iommufd_object *obj); 180 struct iommufd_object *_iommufd_object_alloc(struct iommufd_ctx *ictx, 181 size_t size, 182 enum iommufd_object_type type); 183 184 #define iommufd_object_alloc(ictx, ptr, type) \ 185 container_of(_iommufd_object_alloc( \ 186 ictx, \ 187 sizeof(*(ptr)) + BUILD_BUG_ON_ZERO( \ 188 offsetof(typeof(*(ptr)), \ 189 obj) != 0), \ 190 type), \ 191 typeof(*(ptr)), obj) 192 193 /* 194 * The IO Address Space (IOAS) pagetable is a virtual page table backed by the 195 * io_pagetable object. It is a user controlled mapping of IOVA -> PFNs. The 196 * mapping is copied into all of the associated domains and made available to 197 * in-kernel users. 198 * 199 * Every iommu_domain that is created is wrapped in a iommufd_hw_pagetable 200 * object. When we go to attach a device to an IOAS we need to get an 201 * iommu_domain and wrapping iommufd_hw_pagetable for it. 202 * 203 * An iommu_domain & iommfd_hw_pagetable will be automatically selected 204 * for a device based on the hwpt_list. If no suitable iommu_domain 205 * is found a new iommu_domain will be created. 206 */ 207 struct iommufd_ioas { 208 struct iommufd_object obj; 209 struct io_pagetable iopt; 210 struct mutex mutex; 211 struct list_head hwpt_list; 212 }; 213 214 static inline struct iommufd_ioas *iommufd_get_ioas(struct iommufd_ucmd *ucmd, 215 u32 id) 216 { 217 return container_of(iommufd_get_object(ucmd->ictx, id, 218 IOMMUFD_OBJ_IOAS), 219 struct iommufd_ioas, obj); 220 } 221 222 struct iommufd_ioas *iommufd_ioas_alloc(struct iommufd_ctx *ictx); 223 int iommufd_ioas_alloc_ioctl(struct iommufd_ucmd *ucmd); 224 void iommufd_ioas_destroy(struct iommufd_object *obj); 225 int iommufd_ioas_iova_ranges(struct iommufd_ucmd *ucmd); 226 int iommufd_ioas_allow_iovas(struct iommufd_ucmd *ucmd); 227 int iommufd_ioas_map(struct iommufd_ucmd *ucmd); 228 int iommufd_ioas_copy(struct iommufd_ucmd *ucmd); 229 int iommufd_ioas_unmap(struct iommufd_ucmd *ucmd); 230 int iommufd_ioas_option(struct iommufd_ucmd *ucmd); 231 int iommufd_option_rlimit_mode(struct iommu_option *cmd, 232 struct iommufd_ctx *ictx); 233 234 int iommufd_vfio_ioas(struct iommufd_ucmd *ucmd); 235 236 /* 237 * A HW pagetable is called an iommu_domain inside the kernel. This user object 238 * allows directly creating and inspecting the domains. Domains that have kernel 239 * owned page tables will be associated with an iommufd_ioas that provides the 240 * IOVA to PFN map. 241 */ 242 struct iommufd_hw_pagetable { 243 struct iommufd_object obj; 244 struct iommufd_ioas *ioas; 245 struct iommu_domain *domain; 246 bool auto_domain : 1; 247 bool enforce_cache_coherency : 1; 248 bool msi_cookie : 1; 249 /* Head at iommufd_ioas::hwpt_list */ 250 struct list_head hwpt_item; 251 struct mutex devices_lock; 252 struct list_head devices; 253 }; 254 255 struct iommufd_hw_pagetable * 256 iommufd_hw_pagetable_alloc(struct iommufd_ctx *ictx, struct iommufd_ioas *ioas, 257 struct device *dev); 258 void iommufd_hw_pagetable_destroy(struct iommufd_object *obj); 259 260 void iommufd_device_destroy(struct iommufd_object *obj); 261 262 struct iommufd_access { 263 struct iommufd_object obj; 264 struct iommufd_ctx *ictx; 265 struct iommufd_ioas *ioas; 266 const struct iommufd_access_ops *ops; 267 void *data; 268 unsigned long iova_alignment; 269 u32 iopt_access_list_id; 270 }; 271 272 int iopt_add_access(struct io_pagetable *iopt, struct iommufd_access *access); 273 void iopt_remove_access(struct io_pagetable *iopt, 274 struct iommufd_access *access); 275 void iommufd_access_destroy_object(struct iommufd_object *obj); 276 277 #ifdef CONFIG_IOMMUFD_TEST 278 struct iommufd_hw_pagetable * 279 iommufd_device_selftest_attach(struct iommufd_ctx *ictx, 280 struct iommufd_ioas *ioas, 281 struct device *mock_dev); 282 void iommufd_device_selftest_detach(struct iommufd_ctx *ictx, 283 struct iommufd_hw_pagetable *hwpt); 284 int iommufd_test(struct iommufd_ucmd *ucmd); 285 void iommufd_selftest_destroy(struct iommufd_object *obj); 286 extern size_t iommufd_test_memory_limit; 287 void iommufd_test_syz_conv_iova_id(struct iommufd_ucmd *ucmd, 288 unsigned int ioas_id, u64 *iova, u32 *flags); 289 bool iommufd_should_fail(void); 290 void __init iommufd_test_init(void); 291 void iommufd_test_exit(void); 292 #else 293 static inline void iommufd_test_syz_conv_iova_id(struct iommufd_ucmd *ucmd, 294 unsigned int ioas_id, 295 u64 *iova, u32 *flags) 296 { 297 } 298 static inline bool iommufd_should_fail(void) 299 { 300 return false; 301 } 302 static inline void __init iommufd_test_init(void) 303 { 304 } 305 static inline void iommufd_test_exit(void) 306 { 307 } 308 #endif 309 #endif 310