1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 2 /* 3 * Copyright 2013-2016 Freescale Semiconductor Inc. 4 * Copyright 2016-2017,2019-2020 NXP 5 */ 6 7 #include <linux/device.h> 8 #include <linux/iommu.h> 9 #include <linux/module.h> 10 #include <linux/mutex.h> 11 #include <linux/slab.h> 12 #include <linux/types.h> 13 #include <linux/vfio.h> 14 #include <linux/fsl/mc.h> 15 #include <linux/delay.h> 16 #include <linux/io-64-nonatomic-hi-lo.h> 17 18 #include "vfio_fsl_mc_private.h" 19 20 static struct fsl_mc_driver vfio_fsl_mc_driver; 21 22 static int vfio_fsl_mc_open_device(struct vfio_device *core_vdev) 23 { 24 struct vfio_fsl_mc_device *vdev = 25 container_of(core_vdev, struct vfio_fsl_mc_device, vdev); 26 struct fsl_mc_device *mc_dev = vdev->mc_dev; 27 int count = mc_dev->obj_desc.region_count; 28 int i; 29 30 vdev->regions = kzalloc_objs(struct vfio_fsl_mc_region, count, 31 GFP_KERNEL_ACCOUNT); 32 if (!vdev->regions) 33 return -ENOMEM; 34 35 for (i = 0; i < count; i++) { 36 struct resource *res = &mc_dev->regions[i]; 37 int no_mmap = is_fsl_mc_bus_dprc(mc_dev); 38 39 vdev->regions[i].addr = res->start; 40 vdev->regions[i].size = resource_size(res); 41 vdev->regions[i].type = mc_dev->regions[i].flags & IORESOURCE_BITS; 42 /* 43 * Only regions addressed with PAGE granularity may be 44 * MMAPed securely. 45 */ 46 if (!no_mmap && !(vdev->regions[i].addr & ~PAGE_MASK) && 47 !(vdev->regions[i].size & ~PAGE_MASK)) 48 vdev->regions[i].flags |= 49 VFIO_REGION_INFO_FLAG_MMAP; 50 vdev->regions[i].flags |= VFIO_REGION_INFO_FLAG_READ; 51 if (!(mc_dev->regions[i].flags & IORESOURCE_READONLY)) 52 vdev->regions[i].flags |= VFIO_REGION_INFO_FLAG_WRITE; 53 } 54 55 return 0; 56 } 57 58 static void vfio_fsl_mc_regions_cleanup(struct vfio_fsl_mc_device *vdev) 59 { 60 struct fsl_mc_device *mc_dev = vdev->mc_dev; 61 int i; 62 63 for (i = 0; i < mc_dev->obj_desc.region_count; i++) 64 iounmap(vdev->regions[i].ioaddr); 65 kfree(vdev->regions); 66 } 67 68 static int vfio_fsl_mc_reset_device(struct vfio_fsl_mc_device *vdev) 69 { 70 struct fsl_mc_device *mc_dev = vdev->mc_dev; 71 int ret = 0; 72 73 if (is_fsl_mc_bus_dprc(vdev->mc_dev)) { 74 return dprc_reset_container(mc_dev->mc_io, 0, 75 mc_dev->mc_handle, 76 mc_dev->obj_desc.id, 77 DPRC_RESET_OPTION_NON_RECURSIVE); 78 } else { 79 u16 token; 80 81 ret = fsl_mc_obj_open(mc_dev->mc_io, 0, mc_dev->obj_desc.id, 82 mc_dev->obj_desc.type, 83 &token); 84 if (ret) 85 goto out; 86 ret = fsl_mc_obj_reset(mc_dev->mc_io, 0, token); 87 if (ret) { 88 fsl_mc_obj_close(mc_dev->mc_io, 0, token); 89 goto out; 90 } 91 ret = fsl_mc_obj_close(mc_dev->mc_io, 0, token); 92 } 93 out: 94 return ret; 95 } 96 97 static void vfio_fsl_mc_close_device(struct vfio_device *core_vdev) 98 { 99 struct vfio_fsl_mc_device *vdev = 100 container_of(core_vdev, struct vfio_fsl_mc_device, vdev); 101 struct fsl_mc_device *mc_dev = vdev->mc_dev; 102 struct device *cont_dev = fsl_mc_cont_dev(&mc_dev->dev); 103 struct fsl_mc_device *mc_cont = to_fsl_mc_device(cont_dev); 104 int ret; 105 106 vfio_fsl_mc_regions_cleanup(vdev); 107 108 /* reset the device before cleaning up the interrupts */ 109 ret = vfio_fsl_mc_reset_device(vdev); 110 111 if (ret) 112 dev_warn(&mc_cont->dev, 113 "VFIO_FSL_MC: reset device has failed (%d)\n", ret); 114 115 vfio_fsl_mc_irqs_cleanup(vdev); 116 117 fsl_mc_cleanup_irq_pool(mc_cont); 118 } 119 120 static int vfio_fsl_mc_ioctl_get_region_info(struct vfio_device *core_vdev, 121 struct vfio_region_info *info, 122 struct vfio_info_cap *caps) 123 { 124 struct vfio_fsl_mc_device *vdev = 125 container_of(core_vdev, struct vfio_fsl_mc_device, vdev); 126 struct fsl_mc_device *mc_dev = vdev->mc_dev; 127 128 if (info->index >= mc_dev->obj_desc.region_count) 129 return -EINVAL; 130 131 /* map offset to the physical address */ 132 info->offset = VFIO_FSL_MC_INDEX_TO_OFFSET(info->index); 133 info->size = vdev->regions[info->index].size; 134 info->flags = vdev->regions[info->index].flags; 135 return 0; 136 } 137 138 static long vfio_fsl_mc_ioctl(struct vfio_device *core_vdev, 139 unsigned int cmd, unsigned long arg) 140 { 141 unsigned long minsz; 142 struct vfio_fsl_mc_device *vdev = 143 container_of(core_vdev, struct vfio_fsl_mc_device, vdev); 144 struct fsl_mc_device *mc_dev = vdev->mc_dev; 145 146 switch (cmd) { 147 case VFIO_DEVICE_GET_INFO: 148 { 149 struct vfio_device_info info; 150 151 minsz = offsetofend(struct vfio_device_info, num_irqs); 152 153 if (copy_from_user(&info, (void __user *)arg, minsz)) 154 return -EFAULT; 155 156 if (info.argsz < minsz) 157 return -EINVAL; 158 159 info.flags = VFIO_DEVICE_FLAGS_FSL_MC; 160 161 if (is_fsl_mc_bus_dprc(mc_dev)) 162 info.flags |= VFIO_DEVICE_FLAGS_RESET; 163 164 info.num_regions = mc_dev->obj_desc.region_count; 165 info.num_irqs = mc_dev->obj_desc.irq_count; 166 167 return copy_to_user((void __user *)arg, &info, minsz) ? 168 -EFAULT : 0; 169 } 170 case VFIO_DEVICE_GET_IRQ_INFO: 171 { 172 struct vfio_irq_info info; 173 174 minsz = offsetofend(struct vfio_irq_info, count); 175 if (copy_from_user(&info, (void __user *)arg, minsz)) 176 return -EFAULT; 177 178 if (info.argsz < minsz) 179 return -EINVAL; 180 181 if (info.index >= mc_dev->obj_desc.irq_count) 182 return -EINVAL; 183 184 info.flags = VFIO_IRQ_INFO_EVENTFD; 185 info.count = 1; 186 187 if (copy_to_user((void __user *)arg, &info, minsz)) 188 return -EFAULT; 189 return 0; 190 } 191 case VFIO_DEVICE_SET_IRQS: 192 { 193 struct vfio_irq_set hdr; 194 u8 *data = NULL; 195 int ret = 0; 196 size_t data_size = 0; 197 198 minsz = offsetofend(struct vfio_irq_set, count); 199 200 if (copy_from_user(&hdr, (void __user *)arg, minsz)) 201 return -EFAULT; 202 203 ret = vfio_set_irqs_validate_and_prepare(&hdr, mc_dev->obj_desc.irq_count, 204 mc_dev->obj_desc.irq_count, &data_size); 205 if (ret) 206 return ret; 207 208 if (data_size) { 209 data = memdup_user((void __user *)(arg + minsz), 210 data_size); 211 if (IS_ERR(data)) 212 return PTR_ERR(data); 213 } 214 215 mutex_lock(&vdev->igate); 216 ret = vfio_fsl_mc_set_irqs_ioctl(vdev, hdr.flags, 217 hdr.index, hdr.start, 218 hdr.count, data); 219 mutex_unlock(&vdev->igate); 220 kfree(data); 221 222 return ret; 223 } 224 case VFIO_DEVICE_RESET: 225 { 226 return vfio_fsl_mc_reset_device(vdev); 227 228 } 229 default: 230 return -ENOTTY; 231 } 232 } 233 234 static ssize_t vfio_fsl_mc_read(struct vfio_device *core_vdev, char __user *buf, 235 size_t count, loff_t *ppos) 236 { 237 struct vfio_fsl_mc_device *vdev = 238 container_of(core_vdev, struct vfio_fsl_mc_device, vdev); 239 unsigned int index = VFIO_FSL_MC_OFFSET_TO_INDEX(*ppos); 240 loff_t off = *ppos & VFIO_FSL_MC_OFFSET_MASK; 241 struct fsl_mc_device *mc_dev = vdev->mc_dev; 242 struct vfio_fsl_mc_region *region; 243 u64 data[8]; 244 int i; 245 246 if (index >= mc_dev->obj_desc.region_count) 247 return -EINVAL; 248 249 region = &vdev->regions[index]; 250 251 if (!(region->flags & VFIO_REGION_INFO_FLAG_READ)) 252 return -EINVAL; 253 254 if (!region->ioaddr) { 255 region->ioaddr = ioremap(region->addr, region->size); 256 if (!region->ioaddr) 257 return -ENOMEM; 258 } 259 260 if (count != 64 || off != 0) 261 return -EINVAL; 262 263 for (i = 7; i >= 0; i--) 264 data[i] = readq(region->ioaddr + i * sizeof(uint64_t)); 265 266 if (copy_to_user(buf, data, 64)) 267 return -EFAULT; 268 269 return count; 270 } 271 272 #define MC_CMD_COMPLETION_TIMEOUT_MS 5000 273 #define MC_CMD_COMPLETION_POLLING_MAX_SLEEP_USECS 500 274 275 static int vfio_fsl_mc_send_command(void __iomem *ioaddr, uint64_t *cmd_data) 276 { 277 int i; 278 enum mc_cmd_status status; 279 unsigned long timeout_usecs = MC_CMD_COMPLETION_TIMEOUT_MS * 1000; 280 281 /* Write at command parameter into portal */ 282 for (i = 7; i >= 1; i--) 283 writeq_relaxed(cmd_data[i], ioaddr + i * sizeof(uint64_t)); 284 285 /* Write command header in the end */ 286 writeq(cmd_data[0], ioaddr); 287 288 /* Wait for response before returning to user-space 289 * This can be optimized in future to even prepare response 290 * before returning to user-space and avoid read ioctl. 291 */ 292 for (;;) { 293 u64 header; 294 struct mc_cmd_header *resp_hdr; 295 296 header = cpu_to_le64(readq_relaxed(ioaddr)); 297 298 resp_hdr = (struct mc_cmd_header *)&header; 299 status = (enum mc_cmd_status)resp_hdr->status; 300 if (status != MC_CMD_STATUS_READY) 301 break; 302 303 udelay(MC_CMD_COMPLETION_POLLING_MAX_SLEEP_USECS); 304 timeout_usecs -= MC_CMD_COMPLETION_POLLING_MAX_SLEEP_USECS; 305 if (timeout_usecs == 0) 306 return -ETIMEDOUT; 307 } 308 309 return 0; 310 } 311 312 static ssize_t vfio_fsl_mc_write(struct vfio_device *core_vdev, 313 const char __user *buf, size_t count, 314 loff_t *ppos) 315 { 316 struct vfio_fsl_mc_device *vdev = 317 container_of(core_vdev, struct vfio_fsl_mc_device, vdev); 318 unsigned int index = VFIO_FSL_MC_OFFSET_TO_INDEX(*ppos); 319 loff_t off = *ppos & VFIO_FSL_MC_OFFSET_MASK; 320 struct fsl_mc_device *mc_dev = vdev->mc_dev; 321 struct vfio_fsl_mc_region *region; 322 u64 data[8]; 323 int ret; 324 325 if (index >= mc_dev->obj_desc.region_count) 326 return -EINVAL; 327 328 region = &vdev->regions[index]; 329 330 if (!(region->flags & VFIO_REGION_INFO_FLAG_WRITE)) 331 return -EINVAL; 332 333 if (!region->ioaddr) { 334 region->ioaddr = ioremap(region->addr, region->size); 335 if (!region->ioaddr) 336 return -ENOMEM; 337 } 338 339 if (count != 64 || off != 0) 340 return -EINVAL; 341 342 if (copy_from_user(&data, buf, 64)) 343 return -EFAULT; 344 345 ret = vfio_fsl_mc_send_command(region->ioaddr, data); 346 if (ret) 347 return ret; 348 349 return count; 350 351 } 352 353 static int vfio_fsl_mc_mmap_mmio(struct vfio_fsl_mc_region region, 354 struct vm_area_struct *vma) 355 { 356 u64 size = vma->vm_end - vma->vm_start; 357 u64 pgoff, base; 358 u8 region_cacheable; 359 360 pgoff = vma->vm_pgoff & 361 ((1U << (VFIO_FSL_MC_OFFSET_SHIFT - PAGE_SHIFT)) - 1); 362 base = pgoff << PAGE_SHIFT; 363 364 if (region.size < PAGE_SIZE || base + size > region.size) 365 return -EINVAL; 366 367 region_cacheable = (region.type & FSL_MC_REGION_CACHEABLE) && 368 (region.type & FSL_MC_REGION_SHAREABLE); 369 if (!region_cacheable) 370 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 371 372 vma->vm_pgoff = (region.addr >> PAGE_SHIFT) + pgoff; 373 374 return remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, 375 size, vma->vm_page_prot); 376 } 377 378 static int vfio_fsl_mc_mmap(struct vfio_device *core_vdev, 379 struct vm_area_struct *vma) 380 { 381 struct vfio_fsl_mc_device *vdev = 382 container_of(core_vdev, struct vfio_fsl_mc_device, vdev); 383 struct fsl_mc_device *mc_dev = vdev->mc_dev; 384 unsigned int index; 385 386 index = vma->vm_pgoff >> (VFIO_FSL_MC_OFFSET_SHIFT - PAGE_SHIFT); 387 388 if (vma->vm_end < vma->vm_start) 389 return -EINVAL; 390 if (vma->vm_start & ~PAGE_MASK) 391 return -EINVAL; 392 if (vma->vm_end & ~PAGE_MASK) 393 return -EINVAL; 394 if (!(vma->vm_flags & VM_SHARED)) 395 return -EINVAL; 396 if (index >= mc_dev->obj_desc.region_count) 397 return -EINVAL; 398 399 if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_MMAP)) 400 return -EINVAL; 401 402 if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_READ) 403 && (vma->vm_flags & VM_READ)) 404 return -EINVAL; 405 406 if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_WRITE) 407 && (vma->vm_flags & VM_WRITE)) 408 return -EINVAL; 409 410 vma->vm_private_data = mc_dev; 411 412 return vfio_fsl_mc_mmap_mmio(vdev->regions[index], vma); 413 } 414 415 static const struct vfio_device_ops vfio_fsl_mc_ops; 416 static int vfio_fsl_mc_bus_notifier(struct notifier_block *nb, 417 unsigned long action, void *data) 418 { 419 struct vfio_fsl_mc_device *vdev = container_of(nb, 420 struct vfio_fsl_mc_device, nb); 421 struct device *dev = data; 422 struct fsl_mc_device *mc_dev = to_fsl_mc_device(dev); 423 struct fsl_mc_device *mc_cont = to_fsl_mc_device(mc_dev->dev.parent); 424 425 if (action == BUS_NOTIFY_ADD_DEVICE && 426 vdev->mc_dev == mc_cont) { 427 if (device_set_driver_override(dev, vfio_fsl_mc_ops.name)) 428 dev_warn(dev, "VFIO_FSL_MC: Setting driver override for device in dprc %s failed\n", 429 dev_name(&mc_cont->dev)); 430 else 431 dev_info(dev, "VFIO_FSL_MC: Setting driver override for device in dprc %s\n", 432 dev_name(&mc_cont->dev)); 433 } else if (action == BUS_NOTIFY_BOUND_DRIVER && 434 vdev->mc_dev == mc_cont) { 435 struct fsl_mc_driver *mc_drv = to_fsl_mc_driver(dev->driver); 436 437 if (mc_drv && mc_drv != &vfio_fsl_mc_driver) 438 dev_warn(dev, "VFIO_FSL_MC: Object %s bound to driver %s while DPRC bound to vfio-fsl-mc\n", 439 dev_name(dev), mc_drv->driver.name); 440 } 441 442 return 0; 443 } 444 445 static int vfio_fsl_mc_init_device(struct vfio_fsl_mc_device *vdev) 446 { 447 struct fsl_mc_device *mc_dev = vdev->mc_dev; 448 int ret; 449 450 /* Non-dprc devices share mc_io from parent */ 451 if (!is_fsl_mc_bus_dprc(mc_dev)) { 452 struct fsl_mc_device *mc_cont = to_fsl_mc_device(mc_dev->dev.parent); 453 454 mc_dev->mc_io = mc_cont->mc_io; 455 return 0; 456 } 457 458 vdev->nb.notifier_call = vfio_fsl_mc_bus_notifier; 459 ret = bus_register_notifier(&fsl_mc_bus_type, &vdev->nb); 460 if (ret) 461 return ret; 462 463 /* open DPRC, allocate a MC portal */ 464 ret = dprc_setup(mc_dev); 465 if (ret) { 466 dev_err(&mc_dev->dev, "VFIO_FSL_MC: Failed to setup DPRC (%d)\n", ret); 467 goto out_nc_unreg; 468 } 469 return 0; 470 471 out_nc_unreg: 472 bus_unregister_notifier(&fsl_mc_bus_type, &vdev->nb); 473 return ret; 474 } 475 476 static int vfio_fsl_mc_scan_container(struct fsl_mc_device *mc_dev) 477 { 478 int ret; 479 480 /* non dprc devices do not scan for other devices */ 481 if (!is_fsl_mc_bus_dprc(mc_dev)) 482 return 0; 483 ret = dprc_scan_container(mc_dev, false); 484 if (ret) { 485 dev_err(&mc_dev->dev, 486 "VFIO_FSL_MC: Container scanning failed (%d)\n", ret); 487 dprc_remove_devices(mc_dev, NULL, 0); 488 return ret; 489 } 490 return 0; 491 } 492 493 static void vfio_fsl_uninit_device(struct vfio_fsl_mc_device *vdev) 494 { 495 struct fsl_mc_device *mc_dev = vdev->mc_dev; 496 497 if (!is_fsl_mc_bus_dprc(mc_dev)) 498 return; 499 500 dprc_cleanup(mc_dev); 501 bus_unregister_notifier(&fsl_mc_bus_type, &vdev->nb); 502 } 503 504 static int vfio_fsl_mc_init_dev(struct vfio_device *core_vdev) 505 { 506 struct vfio_fsl_mc_device *vdev = 507 container_of(core_vdev, struct vfio_fsl_mc_device, vdev); 508 struct fsl_mc_device *mc_dev = to_fsl_mc_device(core_vdev->dev); 509 int ret; 510 511 vdev->mc_dev = mc_dev; 512 mutex_init(&vdev->igate); 513 514 if (is_fsl_mc_bus_dprc(mc_dev)) 515 ret = vfio_assign_device_set(core_vdev, &mc_dev->dev); 516 else 517 ret = vfio_assign_device_set(core_vdev, mc_dev->dev.parent); 518 519 if (ret) 520 return ret; 521 522 /* device_set is released by vfio core if @init fails */ 523 return vfio_fsl_mc_init_device(vdev); 524 } 525 526 static int vfio_fsl_mc_probe(struct fsl_mc_device *mc_dev) 527 { 528 struct vfio_fsl_mc_device *vdev; 529 struct device *dev = &mc_dev->dev; 530 int ret; 531 532 vdev = vfio_alloc_device(vfio_fsl_mc_device, vdev, dev, 533 &vfio_fsl_mc_ops); 534 if (IS_ERR(vdev)) 535 return PTR_ERR(vdev); 536 537 ret = vfio_register_group_dev(&vdev->vdev); 538 if (ret) { 539 dev_err(dev, "VFIO_FSL_MC: Failed to add to vfio group\n"); 540 goto out_put_vdev; 541 } 542 543 ret = vfio_fsl_mc_scan_container(mc_dev); 544 if (ret) 545 goto out_group_dev; 546 dev_set_drvdata(dev, vdev); 547 return 0; 548 549 out_group_dev: 550 vfio_unregister_group_dev(&vdev->vdev); 551 out_put_vdev: 552 vfio_put_device(&vdev->vdev); 553 return ret; 554 } 555 556 static void vfio_fsl_mc_release_dev(struct vfio_device *core_vdev) 557 { 558 struct vfio_fsl_mc_device *vdev = 559 container_of(core_vdev, struct vfio_fsl_mc_device, vdev); 560 561 vfio_fsl_uninit_device(vdev); 562 mutex_destroy(&vdev->igate); 563 } 564 565 static void vfio_fsl_mc_remove(struct fsl_mc_device *mc_dev) 566 { 567 struct device *dev = &mc_dev->dev; 568 struct vfio_fsl_mc_device *vdev = dev_get_drvdata(dev); 569 570 vfio_unregister_group_dev(&vdev->vdev); 571 dprc_remove_devices(mc_dev, NULL, 0); 572 vfio_put_device(&vdev->vdev); 573 } 574 575 static const struct vfio_device_ops vfio_fsl_mc_ops = { 576 .name = "vfio-fsl-mc", 577 .init = vfio_fsl_mc_init_dev, 578 .release = vfio_fsl_mc_release_dev, 579 .open_device = vfio_fsl_mc_open_device, 580 .close_device = vfio_fsl_mc_close_device, 581 .ioctl = vfio_fsl_mc_ioctl, 582 .get_region_info_caps = vfio_fsl_mc_ioctl_get_region_info, 583 .read = vfio_fsl_mc_read, 584 .write = vfio_fsl_mc_write, 585 .mmap = vfio_fsl_mc_mmap, 586 .bind_iommufd = vfio_iommufd_physical_bind, 587 .unbind_iommufd = vfio_iommufd_physical_unbind, 588 .attach_ioas = vfio_iommufd_physical_attach_ioas, 589 .detach_ioas = vfio_iommufd_physical_detach_ioas, 590 }; 591 592 static struct fsl_mc_driver vfio_fsl_mc_driver = { 593 .probe = vfio_fsl_mc_probe, 594 .remove = vfio_fsl_mc_remove, 595 .driver = { 596 .name = "vfio-fsl-mc", 597 }, 598 .driver_managed_dma = true, 599 }; 600 601 module_fsl_mc_driver(vfio_fsl_mc_driver); 602 603 MODULE_LICENSE("Dual BSD/GPL"); 604 MODULE_DESCRIPTION("VFIO for FSL-MC devices - User Level meta-driver"); 605