1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 2 /* 3 * Copyright 2013-2016 Freescale Semiconductor Inc. 4 * Copyright 2016-2017,2019-2020 NXP 5 */ 6 7 #include <linux/device.h> 8 #include <linux/iommu.h> 9 #include <linux/module.h> 10 #include <linux/mutex.h> 11 #include <linux/slab.h> 12 #include <linux/types.h> 13 #include <linux/vfio.h> 14 #include <linux/fsl/mc.h> 15 #include <linux/delay.h> 16 #include <linux/io-64-nonatomic-hi-lo.h> 17 18 #include "vfio_fsl_mc_private.h" 19 20 static struct fsl_mc_driver vfio_fsl_mc_driver; 21 22 static int vfio_fsl_mc_open_device(struct vfio_device *core_vdev) 23 { 24 struct vfio_fsl_mc_device *vdev = 25 container_of(core_vdev, struct vfio_fsl_mc_device, vdev); 26 struct fsl_mc_device *mc_dev = vdev->mc_dev; 27 int count = mc_dev->obj_desc.region_count; 28 int i; 29 30 vdev->regions = kcalloc(count, sizeof(struct vfio_fsl_mc_region), 31 GFP_KERNEL_ACCOUNT); 32 if (!vdev->regions) 33 return -ENOMEM; 34 35 for (i = 0; i < count; i++) { 36 struct resource *res = &mc_dev->regions[i]; 37 int no_mmap = is_fsl_mc_bus_dprc(mc_dev); 38 39 vdev->regions[i].addr = res->start; 40 vdev->regions[i].size = resource_size(res); 41 vdev->regions[i].type = mc_dev->regions[i].flags & IORESOURCE_BITS; 42 /* 43 * Only regions addressed with PAGE granularity may be 44 * MMAPed securely. 45 */ 46 if (!no_mmap && !(vdev->regions[i].addr & ~PAGE_MASK) && 47 !(vdev->regions[i].size & ~PAGE_MASK)) 48 vdev->regions[i].flags |= 49 VFIO_REGION_INFO_FLAG_MMAP; 50 vdev->regions[i].flags |= VFIO_REGION_INFO_FLAG_READ; 51 if (!(mc_dev->regions[i].flags & IORESOURCE_READONLY)) 52 vdev->regions[i].flags |= VFIO_REGION_INFO_FLAG_WRITE; 53 } 54 55 return 0; 56 } 57 58 static void vfio_fsl_mc_regions_cleanup(struct vfio_fsl_mc_device *vdev) 59 { 60 struct fsl_mc_device *mc_dev = vdev->mc_dev; 61 int i; 62 63 for (i = 0; i < mc_dev->obj_desc.region_count; i++) 64 iounmap(vdev->regions[i].ioaddr); 65 kfree(vdev->regions); 66 } 67 68 static int vfio_fsl_mc_reset_device(struct vfio_fsl_mc_device *vdev) 69 { 70 struct fsl_mc_device *mc_dev = vdev->mc_dev; 71 int ret = 0; 72 73 if (is_fsl_mc_bus_dprc(vdev->mc_dev)) { 74 return dprc_reset_container(mc_dev->mc_io, 0, 75 mc_dev->mc_handle, 76 mc_dev->obj_desc.id, 77 DPRC_RESET_OPTION_NON_RECURSIVE); 78 } else { 79 u16 token; 80 81 ret = fsl_mc_obj_open(mc_dev->mc_io, 0, mc_dev->obj_desc.id, 82 mc_dev->obj_desc.type, 83 &token); 84 if (ret) 85 goto out; 86 ret = fsl_mc_obj_reset(mc_dev->mc_io, 0, token); 87 if (ret) { 88 fsl_mc_obj_close(mc_dev->mc_io, 0, token); 89 goto out; 90 } 91 ret = fsl_mc_obj_close(mc_dev->mc_io, 0, token); 92 } 93 out: 94 return ret; 95 } 96 97 static void vfio_fsl_mc_close_device(struct vfio_device *core_vdev) 98 { 99 struct vfio_fsl_mc_device *vdev = 100 container_of(core_vdev, struct vfio_fsl_mc_device, vdev); 101 struct fsl_mc_device *mc_dev = vdev->mc_dev; 102 struct device *cont_dev = fsl_mc_cont_dev(&mc_dev->dev); 103 struct fsl_mc_device *mc_cont = to_fsl_mc_device(cont_dev); 104 int ret; 105 106 vfio_fsl_mc_regions_cleanup(vdev); 107 108 /* reset the device before cleaning up the interrupts */ 109 ret = vfio_fsl_mc_reset_device(vdev); 110 111 if (ret) 112 dev_warn(&mc_cont->dev, 113 "VFIO_FSL_MC: reset device has failed (%d)\n", ret); 114 115 vfio_fsl_mc_irqs_cleanup(vdev); 116 117 fsl_mc_cleanup_irq_pool(mc_cont); 118 } 119 120 static int 121 vfio_fsl_mc_ioctl_get_region_info(struct vfio_device *core_vdev, 122 struct vfio_region_info __user *arg) 123 { 124 struct vfio_fsl_mc_device *vdev = 125 container_of(core_vdev, struct vfio_fsl_mc_device, vdev); 126 struct fsl_mc_device *mc_dev = vdev->mc_dev; 127 struct vfio_region_info info; 128 unsigned long minsz; 129 130 minsz = offsetofend(struct vfio_region_info, offset); 131 132 if (copy_from_user(&info, arg, minsz)) 133 return -EFAULT; 134 135 if (info.argsz < minsz) 136 return -EINVAL; 137 138 if (info.index >= mc_dev->obj_desc.region_count) 139 return -EINVAL; 140 141 /* map offset to the physical address */ 142 info.offset = VFIO_FSL_MC_INDEX_TO_OFFSET(info.index); 143 info.size = vdev->regions[info.index].size; 144 info.flags = vdev->regions[info.index].flags; 145 146 if (copy_to_user(arg, &info, minsz)) 147 return -EFAULT; 148 return 0; 149 } 150 151 static long vfio_fsl_mc_ioctl(struct vfio_device *core_vdev, 152 unsigned int cmd, unsigned long arg) 153 { 154 unsigned long minsz; 155 struct vfio_fsl_mc_device *vdev = 156 container_of(core_vdev, struct vfio_fsl_mc_device, vdev); 157 struct fsl_mc_device *mc_dev = vdev->mc_dev; 158 159 switch (cmd) { 160 case VFIO_DEVICE_GET_INFO: 161 { 162 struct vfio_device_info info; 163 164 minsz = offsetofend(struct vfio_device_info, num_irqs); 165 166 if (copy_from_user(&info, (void __user *)arg, minsz)) 167 return -EFAULT; 168 169 if (info.argsz < minsz) 170 return -EINVAL; 171 172 info.flags = VFIO_DEVICE_FLAGS_FSL_MC; 173 174 if (is_fsl_mc_bus_dprc(mc_dev)) 175 info.flags |= VFIO_DEVICE_FLAGS_RESET; 176 177 info.num_regions = mc_dev->obj_desc.region_count; 178 info.num_irqs = mc_dev->obj_desc.irq_count; 179 180 return copy_to_user((void __user *)arg, &info, minsz) ? 181 -EFAULT : 0; 182 } 183 case VFIO_DEVICE_GET_IRQ_INFO: 184 { 185 struct vfio_irq_info info; 186 187 minsz = offsetofend(struct vfio_irq_info, count); 188 if (copy_from_user(&info, (void __user *)arg, minsz)) 189 return -EFAULT; 190 191 if (info.argsz < minsz) 192 return -EINVAL; 193 194 if (info.index >= mc_dev->obj_desc.irq_count) 195 return -EINVAL; 196 197 info.flags = VFIO_IRQ_INFO_EVENTFD; 198 info.count = 1; 199 200 if (copy_to_user((void __user *)arg, &info, minsz)) 201 return -EFAULT; 202 return 0; 203 } 204 case VFIO_DEVICE_SET_IRQS: 205 { 206 struct vfio_irq_set hdr; 207 u8 *data = NULL; 208 int ret = 0; 209 size_t data_size = 0; 210 211 minsz = offsetofend(struct vfio_irq_set, count); 212 213 if (copy_from_user(&hdr, (void __user *)arg, minsz)) 214 return -EFAULT; 215 216 ret = vfio_set_irqs_validate_and_prepare(&hdr, mc_dev->obj_desc.irq_count, 217 mc_dev->obj_desc.irq_count, &data_size); 218 if (ret) 219 return ret; 220 221 if (data_size) { 222 data = memdup_user((void __user *)(arg + minsz), 223 data_size); 224 if (IS_ERR(data)) 225 return PTR_ERR(data); 226 } 227 228 mutex_lock(&vdev->igate); 229 ret = vfio_fsl_mc_set_irqs_ioctl(vdev, hdr.flags, 230 hdr.index, hdr.start, 231 hdr.count, data); 232 mutex_unlock(&vdev->igate); 233 kfree(data); 234 235 return ret; 236 } 237 case VFIO_DEVICE_RESET: 238 { 239 return vfio_fsl_mc_reset_device(vdev); 240 241 } 242 default: 243 return -ENOTTY; 244 } 245 } 246 247 static ssize_t vfio_fsl_mc_read(struct vfio_device *core_vdev, char __user *buf, 248 size_t count, loff_t *ppos) 249 { 250 struct vfio_fsl_mc_device *vdev = 251 container_of(core_vdev, struct vfio_fsl_mc_device, vdev); 252 unsigned int index = VFIO_FSL_MC_OFFSET_TO_INDEX(*ppos); 253 loff_t off = *ppos & VFIO_FSL_MC_OFFSET_MASK; 254 struct fsl_mc_device *mc_dev = vdev->mc_dev; 255 struct vfio_fsl_mc_region *region; 256 u64 data[8]; 257 int i; 258 259 if (index >= mc_dev->obj_desc.region_count) 260 return -EINVAL; 261 262 region = &vdev->regions[index]; 263 264 if (!(region->flags & VFIO_REGION_INFO_FLAG_READ)) 265 return -EINVAL; 266 267 if (!region->ioaddr) { 268 region->ioaddr = ioremap(region->addr, region->size); 269 if (!region->ioaddr) 270 return -ENOMEM; 271 } 272 273 if (count != 64 || off != 0) 274 return -EINVAL; 275 276 for (i = 7; i >= 0; i--) 277 data[i] = readq(region->ioaddr + i * sizeof(uint64_t)); 278 279 if (copy_to_user(buf, data, 64)) 280 return -EFAULT; 281 282 return count; 283 } 284 285 #define MC_CMD_COMPLETION_TIMEOUT_MS 5000 286 #define MC_CMD_COMPLETION_POLLING_MAX_SLEEP_USECS 500 287 288 static int vfio_fsl_mc_send_command(void __iomem *ioaddr, uint64_t *cmd_data) 289 { 290 int i; 291 enum mc_cmd_status status; 292 unsigned long timeout_usecs = MC_CMD_COMPLETION_TIMEOUT_MS * 1000; 293 294 /* Write at command parameter into portal */ 295 for (i = 7; i >= 1; i--) 296 writeq_relaxed(cmd_data[i], ioaddr + i * sizeof(uint64_t)); 297 298 /* Write command header in the end */ 299 writeq(cmd_data[0], ioaddr); 300 301 /* Wait for response before returning to user-space 302 * This can be optimized in future to even prepare response 303 * before returning to user-space and avoid read ioctl. 304 */ 305 for (;;) { 306 u64 header; 307 struct mc_cmd_header *resp_hdr; 308 309 header = cpu_to_le64(readq_relaxed(ioaddr)); 310 311 resp_hdr = (struct mc_cmd_header *)&header; 312 status = (enum mc_cmd_status)resp_hdr->status; 313 if (status != MC_CMD_STATUS_READY) 314 break; 315 316 udelay(MC_CMD_COMPLETION_POLLING_MAX_SLEEP_USECS); 317 timeout_usecs -= MC_CMD_COMPLETION_POLLING_MAX_SLEEP_USECS; 318 if (timeout_usecs == 0) 319 return -ETIMEDOUT; 320 } 321 322 return 0; 323 } 324 325 static ssize_t vfio_fsl_mc_write(struct vfio_device *core_vdev, 326 const char __user *buf, size_t count, 327 loff_t *ppos) 328 { 329 struct vfio_fsl_mc_device *vdev = 330 container_of(core_vdev, struct vfio_fsl_mc_device, vdev); 331 unsigned int index = VFIO_FSL_MC_OFFSET_TO_INDEX(*ppos); 332 loff_t off = *ppos & VFIO_FSL_MC_OFFSET_MASK; 333 struct fsl_mc_device *mc_dev = vdev->mc_dev; 334 struct vfio_fsl_mc_region *region; 335 u64 data[8]; 336 int ret; 337 338 if (index >= mc_dev->obj_desc.region_count) 339 return -EINVAL; 340 341 region = &vdev->regions[index]; 342 343 if (!(region->flags & VFIO_REGION_INFO_FLAG_WRITE)) 344 return -EINVAL; 345 346 if (!region->ioaddr) { 347 region->ioaddr = ioremap(region->addr, region->size); 348 if (!region->ioaddr) 349 return -ENOMEM; 350 } 351 352 if (count != 64 || off != 0) 353 return -EINVAL; 354 355 if (copy_from_user(&data, buf, 64)) 356 return -EFAULT; 357 358 ret = vfio_fsl_mc_send_command(region->ioaddr, data); 359 if (ret) 360 return ret; 361 362 return count; 363 364 } 365 366 static int vfio_fsl_mc_mmap_mmio(struct vfio_fsl_mc_region region, 367 struct vm_area_struct *vma) 368 { 369 u64 size = vma->vm_end - vma->vm_start; 370 u64 pgoff, base; 371 u8 region_cacheable; 372 373 pgoff = vma->vm_pgoff & 374 ((1U << (VFIO_FSL_MC_OFFSET_SHIFT - PAGE_SHIFT)) - 1); 375 base = pgoff << PAGE_SHIFT; 376 377 if (region.size < PAGE_SIZE || base + size > region.size) 378 return -EINVAL; 379 380 region_cacheable = (region.type & FSL_MC_REGION_CACHEABLE) && 381 (region.type & FSL_MC_REGION_SHAREABLE); 382 if (!region_cacheable) 383 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 384 385 vma->vm_pgoff = (region.addr >> PAGE_SHIFT) + pgoff; 386 387 return remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, 388 size, vma->vm_page_prot); 389 } 390 391 static int vfio_fsl_mc_mmap(struct vfio_device *core_vdev, 392 struct vm_area_struct *vma) 393 { 394 struct vfio_fsl_mc_device *vdev = 395 container_of(core_vdev, struct vfio_fsl_mc_device, vdev); 396 struct fsl_mc_device *mc_dev = vdev->mc_dev; 397 unsigned int index; 398 399 index = vma->vm_pgoff >> (VFIO_FSL_MC_OFFSET_SHIFT - PAGE_SHIFT); 400 401 if (vma->vm_end < vma->vm_start) 402 return -EINVAL; 403 if (vma->vm_start & ~PAGE_MASK) 404 return -EINVAL; 405 if (vma->vm_end & ~PAGE_MASK) 406 return -EINVAL; 407 if (!(vma->vm_flags & VM_SHARED)) 408 return -EINVAL; 409 if (index >= mc_dev->obj_desc.region_count) 410 return -EINVAL; 411 412 if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_MMAP)) 413 return -EINVAL; 414 415 if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_READ) 416 && (vma->vm_flags & VM_READ)) 417 return -EINVAL; 418 419 if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_WRITE) 420 && (vma->vm_flags & VM_WRITE)) 421 return -EINVAL; 422 423 vma->vm_private_data = mc_dev; 424 425 return vfio_fsl_mc_mmap_mmio(vdev->regions[index], vma); 426 } 427 428 static const struct vfio_device_ops vfio_fsl_mc_ops; 429 static int vfio_fsl_mc_bus_notifier(struct notifier_block *nb, 430 unsigned long action, void *data) 431 { 432 struct vfio_fsl_mc_device *vdev = container_of(nb, 433 struct vfio_fsl_mc_device, nb); 434 struct device *dev = data; 435 struct fsl_mc_device *mc_dev = to_fsl_mc_device(dev); 436 struct fsl_mc_device *mc_cont = to_fsl_mc_device(mc_dev->dev.parent); 437 438 if (action == BUS_NOTIFY_ADD_DEVICE && 439 vdev->mc_dev == mc_cont) { 440 mc_dev->driver_override = kasprintf(GFP_KERNEL, "%s", 441 vfio_fsl_mc_ops.name); 442 if (!mc_dev->driver_override) 443 dev_warn(dev, "VFIO_FSL_MC: Setting driver override for device in dprc %s failed\n", 444 dev_name(&mc_cont->dev)); 445 else 446 dev_info(dev, "VFIO_FSL_MC: Setting driver override for device in dprc %s\n", 447 dev_name(&mc_cont->dev)); 448 } else if (action == BUS_NOTIFY_BOUND_DRIVER && 449 vdev->mc_dev == mc_cont) { 450 struct fsl_mc_driver *mc_drv = to_fsl_mc_driver(dev->driver); 451 452 if (mc_drv && mc_drv != &vfio_fsl_mc_driver) 453 dev_warn(dev, "VFIO_FSL_MC: Object %s bound to driver %s while DPRC bound to vfio-fsl-mc\n", 454 dev_name(dev), mc_drv->driver.name); 455 } 456 457 return 0; 458 } 459 460 static int vfio_fsl_mc_init_device(struct vfio_fsl_mc_device *vdev) 461 { 462 struct fsl_mc_device *mc_dev = vdev->mc_dev; 463 int ret; 464 465 /* Non-dprc devices share mc_io from parent */ 466 if (!is_fsl_mc_bus_dprc(mc_dev)) { 467 struct fsl_mc_device *mc_cont = to_fsl_mc_device(mc_dev->dev.parent); 468 469 mc_dev->mc_io = mc_cont->mc_io; 470 return 0; 471 } 472 473 vdev->nb.notifier_call = vfio_fsl_mc_bus_notifier; 474 ret = bus_register_notifier(&fsl_mc_bus_type, &vdev->nb); 475 if (ret) 476 return ret; 477 478 /* open DPRC, allocate a MC portal */ 479 ret = dprc_setup(mc_dev); 480 if (ret) { 481 dev_err(&mc_dev->dev, "VFIO_FSL_MC: Failed to setup DPRC (%d)\n", ret); 482 goto out_nc_unreg; 483 } 484 return 0; 485 486 out_nc_unreg: 487 bus_unregister_notifier(&fsl_mc_bus_type, &vdev->nb); 488 return ret; 489 } 490 491 static int vfio_fsl_mc_scan_container(struct fsl_mc_device *mc_dev) 492 { 493 int ret; 494 495 /* non dprc devices do not scan for other devices */ 496 if (!is_fsl_mc_bus_dprc(mc_dev)) 497 return 0; 498 ret = dprc_scan_container(mc_dev, false); 499 if (ret) { 500 dev_err(&mc_dev->dev, 501 "VFIO_FSL_MC: Container scanning failed (%d)\n", ret); 502 dprc_remove_devices(mc_dev, NULL, 0); 503 return ret; 504 } 505 return 0; 506 } 507 508 static void vfio_fsl_uninit_device(struct vfio_fsl_mc_device *vdev) 509 { 510 struct fsl_mc_device *mc_dev = vdev->mc_dev; 511 512 if (!is_fsl_mc_bus_dprc(mc_dev)) 513 return; 514 515 dprc_cleanup(mc_dev); 516 bus_unregister_notifier(&fsl_mc_bus_type, &vdev->nb); 517 } 518 519 static int vfio_fsl_mc_init_dev(struct vfio_device *core_vdev) 520 { 521 struct vfio_fsl_mc_device *vdev = 522 container_of(core_vdev, struct vfio_fsl_mc_device, vdev); 523 struct fsl_mc_device *mc_dev = to_fsl_mc_device(core_vdev->dev); 524 int ret; 525 526 vdev->mc_dev = mc_dev; 527 mutex_init(&vdev->igate); 528 529 if (is_fsl_mc_bus_dprc(mc_dev)) 530 ret = vfio_assign_device_set(core_vdev, &mc_dev->dev); 531 else 532 ret = vfio_assign_device_set(core_vdev, mc_dev->dev.parent); 533 534 if (ret) 535 return ret; 536 537 /* device_set is released by vfio core if @init fails */ 538 return vfio_fsl_mc_init_device(vdev); 539 } 540 541 static int vfio_fsl_mc_probe(struct fsl_mc_device *mc_dev) 542 { 543 struct vfio_fsl_mc_device *vdev; 544 struct device *dev = &mc_dev->dev; 545 int ret; 546 547 dev_err_once(dev, "DEPRECATION: vfio-fsl-mc is deprecated and will be removed in a future kernel release\n"); 548 549 vdev = vfio_alloc_device(vfio_fsl_mc_device, vdev, dev, 550 &vfio_fsl_mc_ops); 551 if (IS_ERR(vdev)) 552 return PTR_ERR(vdev); 553 554 ret = vfio_register_group_dev(&vdev->vdev); 555 if (ret) { 556 dev_err(dev, "VFIO_FSL_MC: Failed to add to vfio group\n"); 557 goto out_put_vdev; 558 } 559 560 ret = vfio_fsl_mc_scan_container(mc_dev); 561 if (ret) 562 goto out_group_dev; 563 dev_set_drvdata(dev, vdev); 564 return 0; 565 566 out_group_dev: 567 vfio_unregister_group_dev(&vdev->vdev); 568 out_put_vdev: 569 vfio_put_device(&vdev->vdev); 570 return ret; 571 } 572 573 static void vfio_fsl_mc_release_dev(struct vfio_device *core_vdev) 574 { 575 struct vfio_fsl_mc_device *vdev = 576 container_of(core_vdev, struct vfio_fsl_mc_device, vdev); 577 578 vfio_fsl_uninit_device(vdev); 579 mutex_destroy(&vdev->igate); 580 } 581 582 static void vfio_fsl_mc_remove(struct fsl_mc_device *mc_dev) 583 { 584 struct device *dev = &mc_dev->dev; 585 struct vfio_fsl_mc_device *vdev = dev_get_drvdata(dev); 586 587 vfio_unregister_group_dev(&vdev->vdev); 588 dprc_remove_devices(mc_dev, NULL, 0); 589 vfio_put_device(&vdev->vdev); 590 } 591 592 static const struct vfio_device_ops vfio_fsl_mc_ops = { 593 .name = "vfio-fsl-mc", 594 .init = vfio_fsl_mc_init_dev, 595 .release = vfio_fsl_mc_release_dev, 596 .open_device = vfio_fsl_mc_open_device, 597 .close_device = vfio_fsl_mc_close_device, 598 .ioctl = vfio_fsl_mc_ioctl, 599 .get_region_info = vfio_fsl_mc_ioctl_get_region_info, 600 .read = vfio_fsl_mc_read, 601 .write = vfio_fsl_mc_write, 602 .mmap = vfio_fsl_mc_mmap, 603 .bind_iommufd = vfio_iommufd_physical_bind, 604 .unbind_iommufd = vfio_iommufd_physical_unbind, 605 .attach_ioas = vfio_iommufd_physical_attach_ioas, 606 .detach_ioas = vfio_iommufd_physical_detach_ioas, 607 }; 608 609 static struct fsl_mc_driver vfio_fsl_mc_driver = { 610 .probe = vfio_fsl_mc_probe, 611 .remove = vfio_fsl_mc_remove, 612 .driver = { 613 .name = "vfio-fsl-mc", 614 }, 615 .driver_managed_dma = true, 616 }; 617 618 module_fsl_mc_driver(vfio_fsl_mc_driver); 619 620 MODULE_LICENSE("Dual BSD/GPL"); 621 MODULE_DESCRIPTION("VFIO for FSL-MC devices - User Level meta-driver"); 622