1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2023 Intel Corporation. 4 */ 5 #include <linux/vfio.h> 6 #include <linux/iommufd.h> 7 8 #include "vfio.h" 9 10 static dev_t device_devt; 11 12 void vfio_init_device_cdev(struct vfio_device *device) 13 { 14 if (vfio_device_is_noiommu(device) && 15 !IS_ENABLED(CONFIG_IOMMUFD_NOIOMMU)) 16 return; 17 18 device->device.devt = MKDEV(MAJOR(device_devt), device->index); 19 cdev_init(&device->cdev, &vfio_device_fops); 20 device->cdev.owner = THIS_MODULE; 21 } 22 23 /* 24 * device access via the fd opened by this function is blocked until 25 * .open_device() is called successfully during BIND_IOMMUFD. 26 */ 27 int vfio_device_fops_cdev_open(struct inode *inode, struct file *filep) 28 { 29 struct vfio_device *device = container_of(inode->i_cdev, 30 struct vfio_device, cdev); 31 struct vfio_device_file *df; 32 int ret; 33 34 /* Paired with the put in vfio_device_fops_release() */ 35 if (!vfio_device_try_get_registration(device)) 36 return -ENODEV; 37 38 if (vfio_device_is_noiommu(device) && !capable(CAP_SYS_RAWIO)) { 39 ret = -EPERM; 40 goto err_put_registration; 41 } 42 43 df = vfio_allocate_device_file(device); 44 if (IS_ERR(df)) { 45 ret = PTR_ERR(df); 46 goto err_put_registration; 47 } 48 49 filep->private_data = df; 50 51 /* 52 * Use the pseudo fs inode on the device to link all mmaps 53 * to the same address space, allowing us to unmap all vmas 54 * associated to this device using unmap_mapping_range(). 55 */ 56 filep->f_mapping = device->inode->i_mapping; 57 58 return 0; 59 60 err_put_registration: 61 vfio_device_put_registration(device); 62 return ret; 63 } 64 65 static void vfio_df_get_kvm_safe(struct vfio_device_file *df) 66 { 67 spin_lock(&df->kvm_ref_lock); 68 vfio_device_get_kvm_safe(df->device, df->kvm); 69 spin_unlock(&df->kvm_ref_lock); 70 } 71 72 static int vfio_df_check_token(struct vfio_device *device, 73 const struct vfio_device_bind_iommufd *bind) 74 { 75 uuid_t uuid; 76 77 if (!device->ops->match_token_uuid) { 78 if (bind->flags & VFIO_DEVICE_BIND_FLAG_TOKEN) 79 return -EINVAL; 80 return 0; 81 } 82 83 if (!(bind->flags & VFIO_DEVICE_BIND_FLAG_TOKEN)) 84 return device->ops->match_token_uuid(device, NULL); 85 86 if (copy_from_user(&uuid, u64_to_user_ptr(bind->token_uuid_ptr), 87 sizeof(uuid))) 88 return -EFAULT; 89 return device->ops->match_token_uuid(device, &uuid); 90 } 91 92 long vfio_df_ioctl_bind_iommufd(struct vfio_device_file *df, 93 struct vfio_device_bind_iommufd __user *arg) 94 { 95 const u32 VALID_FLAGS = VFIO_DEVICE_BIND_FLAG_TOKEN; 96 struct vfio_device *device = df->device; 97 struct vfio_device_bind_iommufd bind; 98 unsigned long minsz; 99 u32 user_size; 100 int ret; 101 102 static_assert(__same_type(arg->out_devid, df->devid)); 103 104 minsz = offsetofend(struct vfio_device_bind_iommufd, out_devid); 105 106 ret = get_user(user_size, &arg->argsz); 107 if (ret) 108 return ret; 109 if (user_size < minsz) 110 return -EINVAL; 111 ret = copy_struct_from_user(&bind, sizeof(bind), arg, user_size); 112 if (ret) 113 return ret; 114 115 if (bind.iommufd < 0 || bind.flags & ~VALID_FLAGS) 116 return -EINVAL; 117 118 /* BIND_IOMMUFD only allowed for cdev fds */ 119 if (df->group) 120 return -EINVAL; 121 122 ret = vfio_device_block_group(device); 123 if (ret) 124 return ret; 125 126 mutex_lock(&device->dev_set->lock); 127 /* one device cannot be bound twice */ 128 if (df->access_granted) { 129 ret = -EINVAL; 130 goto out_unlock; 131 } 132 133 ret = vfio_df_check_token(device, &bind); 134 if (ret) 135 goto out_unlock; 136 137 df->iommufd = iommufd_ctx_from_fd(bind.iommufd); 138 if (IS_ERR(df->iommufd)) { 139 ret = PTR_ERR(df->iommufd); 140 df->iommufd = NULL; 141 goto out_unlock; 142 } 143 144 /* 145 * Before the device open, get the KVM pointer currently 146 * associated with the device file (if there is) and obtain 147 * a reference. This reference is held until device closed. 148 * Save the pointer in the device for use by drivers. 149 */ 150 vfio_df_get_kvm_safe(df); 151 152 ret = vfio_df_open(df); 153 if (ret) 154 goto out_put_kvm; 155 156 ret = copy_to_user(&arg->out_devid, &df->devid, 157 sizeof(df->devid)) ? -EFAULT : 0; 158 if (ret) 159 goto out_close_device; 160 161 device->cdev_opened = true; 162 /* 163 * Paired with smp_load_acquire() in vfio_device_fops::ioctl/ 164 * read/write/mmap 165 */ 166 smp_store_release(&df->access_granted, true); 167 mutex_unlock(&device->dev_set->lock); 168 return 0; 169 170 out_close_device: 171 vfio_df_close(df); 172 out_put_kvm: 173 vfio_device_put_kvm(device); 174 iommufd_ctx_put(df->iommufd); 175 df->iommufd = NULL; 176 out_unlock: 177 mutex_unlock(&device->dev_set->lock); 178 vfio_device_unblock_group(device); 179 return ret; 180 } 181 182 void vfio_df_unbind_iommufd(struct vfio_device_file *df) 183 { 184 struct vfio_device *device = df->device; 185 186 /* 187 * In the time of close, there is no contention with another one 188 * changing this flag. So read df->access_granted without lock 189 * and no smp_load_acquire() is ok. 190 */ 191 if (!df->access_granted) 192 return; 193 194 mutex_lock(&device->dev_set->lock); 195 vfio_df_close(df); 196 vfio_device_put_kvm(device); 197 iommufd_ctx_put(df->iommufd); 198 device->cdev_opened = false; 199 mutex_unlock(&device->dev_set->lock); 200 vfio_device_unblock_group(device); 201 } 202 203 int vfio_df_ioctl_attach_pt(struct vfio_device_file *df, 204 struct vfio_device_attach_iommufd_pt __user *arg) 205 { 206 struct vfio_device_attach_iommufd_pt attach; 207 struct vfio_device *device = df->device; 208 unsigned long minsz, xend = 0; 209 int ret; 210 211 minsz = offsetofend(struct vfio_device_attach_iommufd_pt, pt_id); 212 213 if (copy_from_user(&attach, arg, minsz)) 214 return -EFAULT; 215 216 if (attach.argsz < minsz) 217 return -EINVAL; 218 219 if (attach.flags & ~VFIO_DEVICE_ATTACH_PASID) 220 return -EINVAL; 221 222 if (attach.flags & VFIO_DEVICE_ATTACH_PASID) { 223 if (!device->ops->pasid_attach_ioas) 224 return -EOPNOTSUPP; 225 xend = offsetofend(struct vfio_device_attach_iommufd_pt, pasid); 226 } 227 228 if (xend) { 229 if (attach.argsz < xend) 230 return -EINVAL; 231 232 if (copy_from_user((void *)&attach + minsz, 233 (void __user *)arg + minsz, xend - minsz)) 234 return -EFAULT; 235 } 236 237 mutex_lock(&device->dev_set->lock); 238 if (attach.flags & VFIO_DEVICE_ATTACH_PASID) 239 ret = device->ops->pasid_attach_ioas(device, 240 attach.pasid, 241 &attach.pt_id); 242 else 243 ret = device->ops->attach_ioas(device, &attach.pt_id); 244 if (ret) 245 goto out_unlock; 246 247 if (copy_to_user(&arg->pt_id, &attach.pt_id, sizeof(attach.pt_id))) { 248 ret = -EFAULT; 249 goto out_detach; 250 } 251 mutex_unlock(&device->dev_set->lock); 252 253 return 0; 254 255 out_detach: 256 device->ops->detach_ioas(device); 257 out_unlock: 258 mutex_unlock(&device->dev_set->lock); 259 return ret; 260 } 261 262 int vfio_df_ioctl_detach_pt(struct vfio_device_file *df, 263 struct vfio_device_detach_iommufd_pt __user *arg) 264 { 265 struct vfio_device_detach_iommufd_pt detach; 266 struct vfio_device *device = df->device; 267 unsigned long minsz, xend = 0; 268 269 minsz = offsetofend(struct vfio_device_detach_iommufd_pt, flags); 270 271 if (copy_from_user(&detach, arg, minsz)) 272 return -EFAULT; 273 274 if (detach.argsz < minsz) 275 return -EINVAL; 276 277 if (detach.flags & ~VFIO_DEVICE_DETACH_PASID) 278 return -EINVAL; 279 280 if (detach.flags & VFIO_DEVICE_DETACH_PASID) { 281 if (!device->ops->pasid_detach_ioas) 282 return -EOPNOTSUPP; 283 xend = offsetofend(struct vfio_device_detach_iommufd_pt, pasid); 284 } 285 286 if (xend) { 287 if (detach.argsz < xend) 288 return -EINVAL; 289 290 if (copy_from_user((void *)&detach + minsz, 291 (void __user *)arg + minsz, xend - minsz)) 292 return -EFAULT; 293 } 294 295 mutex_lock(&device->dev_set->lock); 296 if (detach.flags & VFIO_DEVICE_DETACH_PASID) 297 device->ops->pasid_detach_ioas(device, detach.pasid); 298 else 299 device->ops->detach_ioas(device); 300 mutex_unlock(&device->dev_set->lock); 301 302 return 0; 303 } 304 305 int vfio_cdev_init(void) 306 { 307 return alloc_chrdev_region(&device_devt, 0, 308 MINORMASK + 1, "vfio-dev"); 309 } 310 311 void vfio_cdev_cleanup(void) 312 { 313 unregister_chrdev_region(device_devt, MINORMASK + 1); 314 } 315