1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Physical device callbacks for vfio_ccw 4 * 5 * Copyright IBM Corp. 2017 6 * Copyright Red Hat, Inc. 2019 7 * 8 * Author(s): Dong Jia Shi <bjsdjshi@linux.vnet.ibm.com> 9 * Xiao Feng Ren <renxiaof@linux.vnet.ibm.com> 10 * Cornelia Huck <cohuck@redhat.com> 11 */ 12 13 #include <linux/vfio.h> 14 #include <linux/nospec.h> 15 #include <linux/slab.h> 16 17 #include "vfio_ccw_private.h" 18 19 static const struct vfio_device_ops vfio_ccw_dev_ops; 20 21 static int vfio_ccw_mdev_reset(struct vfio_ccw_private *private) 22 { 23 /* 24 * If the FSM state is seen as Not Operational after closing 25 * and re-opening the mdev, return an error. 26 */ 27 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_CLOSE); 28 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_OPEN); 29 if (private->state == VFIO_CCW_STATE_NOT_OPER) 30 return -EINVAL; 31 32 return 0; 33 } 34 35 static void vfio_ccw_dma_unmap(struct vfio_device *vdev, u64 iova, u64 length) 36 { 37 struct vfio_ccw_private *private = 38 container_of(vdev, struct vfio_ccw_private, vdev); 39 40 /* Drivers MUST unpin pages in response to an invalidation. */ 41 if (!cp_iova_pinned(&private->cp, iova, length)) 42 return; 43 44 vfio_ccw_mdev_reset(private); 45 } 46 47 static int vfio_ccw_mdev_init_dev(struct vfio_device *vdev) 48 { 49 struct vfio_ccw_private *private = 50 container_of(vdev, struct vfio_ccw_private, vdev); 51 52 mutex_init(&private->io_mutex); 53 private->state = VFIO_CCW_STATE_STANDBY; 54 INIT_LIST_HEAD(&private->crw); 55 INIT_WORK(&private->io_work, vfio_ccw_sch_io_todo); 56 INIT_WORK(&private->crw_work, vfio_ccw_crw_todo); 57 58 private->cp.guest_cp = kcalloc(CCWCHAIN_LEN_MAX, sizeof(struct ccw1), 59 GFP_KERNEL); 60 if (!private->cp.guest_cp) 61 goto out_free_private; 62 63 private->io_region = kmem_cache_zalloc(vfio_ccw_io_region, 64 GFP_KERNEL | GFP_DMA); 65 if (!private->io_region) 66 goto out_free_cp; 67 68 private->cmd_region = kmem_cache_zalloc(vfio_ccw_cmd_region, 69 GFP_KERNEL | GFP_DMA); 70 if (!private->cmd_region) 71 goto out_free_io; 72 73 private->schib_region = kmem_cache_zalloc(vfio_ccw_schib_region, 74 GFP_KERNEL | GFP_DMA); 75 if (!private->schib_region) 76 goto out_free_cmd; 77 78 private->crw_region = kmem_cache_zalloc(vfio_ccw_crw_region, 79 GFP_KERNEL | GFP_DMA); 80 if (!private->crw_region) 81 goto out_free_schib; 82 83 return 0; 84 85 out_free_schib: 86 kmem_cache_free(vfio_ccw_schib_region, private->schib_region); 87 out_free_cmd: 88 kmem_cache_free(vfio_ccw_cmd_region, private->cmd_region); 89 out_free_io: 90 kmem_cache_free(vfio_ccw_io_region, private->io_region); 91 out_free_cp: 92 kfree(private->cp.guest_cp); 93 out_free_private: 94 mutex_destroy(&private->io_mutex); 95 return -ENOMEM; 96 } 97 98 static int vfio_ccw_mdev_probe(struct mdev_device *mdev) 99 { 100 struct subchannel *sch = to_subchannel(mdev->dev.parent); 101 struct vfio_ccw_parent *parent = dev_get_drvdata(&sch->dev); 102 struct vfio_ccw_private *private; 103 int ret; 104 105 private = vfio_alloc_device(vfio_ccw_private, vdev, &mdev->dev, 106 &vfio_ccw_dev_ops); 107 if (IS_ERR(private)) 108 return PTR_ERR(private); 109 110 dev_set_drvdata(&parent->dev, private); 111 112 VFIO_CCW_MSG_EVENT(2, "sch %x.%x.%04x: create\n", 113 sch->schid.cssid, 114 sch->schid.ssid, 115 sch->schid.sch_no); 116 117 ret = vfio_register_emulated_iommu_dev(&private->vdev); 118 if (ret) 119 goto err_put_vdev; 120 dev_set_drvdata(&mdev->dev, private); 121 return 0; 122 123 err_put_vdev: 124 dev_set_drvdata(&parent->dev, NULL); 125 vfio_put_device(&private->vdev); 126 return ret; 127 } 128 129 static void vfio_ccw_mdev_release_dev(struct vfio_device *vdev) 130 { 131 struct vfio_ccw_private *private = 132 container_of(vdev, struct vfio_ccw_private, vdev); 133 struct vfio_ccw_crw *crw, *temp; 134 135 list_for_each_entry_safe(crw, temp, &private->crw, next) { 136 list_del(&crw->next); 137 kfree(crw); 138 } 139 140 kmem_cache_free(vfio_ccw_crw_region, private->crw_region); 141 kmem_cache_free(vfio_ccw_schib_region, private->schib_region); 142 kmem_cache_free(vfio_ccw_cmd_region, private->cmd_region); 143 kmem_cache_free(vfio_ccw_io_region, private->io_region); 144 kfree(private->cp.guest_cp); 145 mutex_destroy(&private->io_mutex); 146 } 147 148 static void vfio_ccw_mdev_remove(struct mdev_device *mdev) 149 { 150 struct subchannel *sch = to_subchannel(mdev->dev.parent); 151 struct vfio_ccw_parent *parent = dev_get_drvdata(&sch->dev); 152 struct vfio_ccw_private *private = dev_get_drvdata(&parent->dev); 153 154 VFIO_CCW_MSG_EVENT(2, "sch %x.%x.%04x: remove\n", 155 sch->schid.cssid, 156 sch->schid.ssid, 157 sch->schid.sch_no); 158 159 vfio_unregister_group_dev(&private->vdev); 160 161 dev_set_drvdata(&parent->dev, NULL); 162 vfio_put_device(&private->vdev); 163 } 164 165 static int vfio_ccw_mdev_open_device(struct vfio_device *vdev) 166 { 167 struct vfio_ccw_private *private = 168 container_of(vdev, struct vfio_ccw_private, vdev); 169 int ret; 170 171 /* Device cannot simply be opened again from this state */ 172 if (private->state == VFIO_CCW_STATE_NOT_OPER) 173 return -EINVAL; 174 175 ret = vfio_ccw_register_async_dev_regions(private); 176 if (ret) 177 return ret; 178 179 ret = vfio_ccw_register_schib_dev_regions(private); 180 if (ret) 181 goto out_unregister; 182 183 ret = vfio_ccw_register_crw_dev_regions(private); 184 if (ret) 185 goto out_unregister; 186 187 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_OPEN); 188 if (private->state == VFIO_CCW_STATE_NOT_OPER) { 189 ret = -EINVAL; 190 goto out_unregister; 191 } 192 193 return ret; 194 195 out_unregister: 196 vfio_ccw_unregister_dev_regions(private); 197 return ret; 198 } 199 200 static void vfio_ccw_mdev_close_device(struct vfio_device *vdev) 201 { 202 struct vfio_ccw_private *private = 203 container_of(vdev, struct vfio_ccw_private, vdev); 204 205 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_CLOSE); 206 vfio_ccw_unregister_dev_regions(private); 207 } 208 209 static ssize_t vfio_ccw_mdev_read_io_region(struct vfio_ccw_private *private, 210 char __user *buf, size_t count, 211 loff_t *ppos) 212 { 213 loff_t pos = *ppos & VFIO_CCW_OFFSET_MASK; 214 struct ccw_io_region *region; 215 int ret; 216 217 if (pos + count > sizeof(*region)) 218 return -EINVAL; 219 220 mutex_lock(&private->io_mutex); 221 region = private->io_region; 222 if (copy_to_user(buf, (void *)region + pos, count)) 223 ret = -EFAULT; 224 else 225 ret = count; 226 mutex_unlock(&private->io_mutex); 227 return ret; 228 } 229 230 static ssize_t vfio_ccw_mdev_read(struct vfio_device *vdev, 231 char __user *buf, 232 size_t count, 233 loff_t *ppos) 234 { 235 struct vfio_ccw_private *private = 236 container_of(vdev, struct vfio_ccw_private, vdev); 237 unsigned int index = VFIO_CCW_OFFSET_TO_INDEX(*ppos); 238 239 if (index >= VFIO_CCW_NUM_REGIONS + private->num_regions) 240 return -EINVAL; 241 242 switch (index) { 243 case VFIO_CCW_CONFIG_REGION_INDEX: 244 return vfio_ccw_mdev_read_io_region(private, buf, count, ppos); 245 default: 246 index -= VFIO_CCW_NUM_REGIONS; 247 return private->region[index].ops->read(private, buf, count, 248 ppos); 249 } 250 251 return -EINVAL; 252 } 253 254 static ssize_t vfio_ccw_mdev_write_io_region(struct vfio_ccw_private *private, 255 const char __user *buf, 256 size_t count, loff_t *ppos) 257 { 258 loff_t pos = *ppos & VFIO_CCW_OFFSET_MASK; 259 struct ccw_io_region *region; 260 int ret; 261 262 if (pos + count > sizeof(*region)) 263 return -EINVAL; 264 265 if (!mutex_trylock(&private->io_mutex)) 266 return -EAGAIN; 267 268 region = private->io_region; 269 if (copy_from_user((void *)region + pos, buf, count)) { 270 ret = -EFAULT; 271 goto out_unlock; 272 } 273 274 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_IO_REQ); 275 ret = (region->ret_code != 0) ? region->ret_code : count; 276 277 out_unlock: 278 mutex_unlock(&private->io_mutex); 279 return ret; 280 } 281 282 static ssize_t vfio_ccw_mdev_write(struct vfio_device *vdev, 283 const char __user *buf, 284 size_t count, 285 loff_t *ppos) 286 { 287 struct vfio_ccw_private *private = 288 container_of(vdev, struct vfio_ccw_private, vdev); 289 unsigned int index = VFIO_CCW_OFFSET_TO_INDEX(*ppos); 290 291 if (index >= VFIO_CCW_NUM_REGIONS + private->num_regions) 292 return -EINVAL; 293 294 switch (index) { 295 case VFIO_CCW_CONFIG_REGION_INDEX: 296 return vfio_ccw_mdev_write_io_region(private, buf, count, ppos); 297 default: 298 index -= VFIO_CCW_NUM_REGIONS; 299 return private->region[index].ops->write(private, buf, count, 300 ppos); 301 } 302 303 return -EINVAL; 304 } 305 306 static int vfio_ccw_mdev_get_device_info(struct vfio_ccw_private *private, 307 struct vfio_device_info *info) 308 { 309 info->flags = VFIO_DEVICE_FLAGS_CCW | VFIO_DEVICE_FLAGS_RESET; 310 info->num_regions = VFIO_CCW_NUM_REGIONS + private->num_regions; 311 info->num_irqs = VFIO_CCW_NUM_IRQS; 312 313 return 0; 314 } 315 316 static int vfio_ccw_mdev_ioctl_get_region_info(struct vfio_device *vdev, 317 struct vfio_region_info *info, 318 struct vfio_info_cap *caps) 319 { 320 struct vfio_ccw_private *private = 321 container_of(vdev, struct vfio_ccw_private, vdev); 322 int i; 323 324 switch (info->index) { 325 case VFIO_CCW_CONFIG_REGION_INDEX: 326 info->offset = 0; 327 info->size = sizeof(struct ccw_io_region); 328 info->flags = VFIO_REGION_INFO_FLAG_READ 329 | VFIO_REGION_INFO_FLAG_WRITE; 330 return 0; 331 default: /* all other regions are handled via capability chain */ 332 { 333 struct vfio_region_info_cap_type cap_type = { 334 .header.id = VFIO_REGION_INFO_CAP_TYPE, 335 .header.version = 1 }; 336 int ret; 337 338 if (info->index >= 339 VFIO_CCW_NUM_REGIONS + private->num_regions) 340 return -EINVAL; 341 342 info->index = array_index_nospec(info->index, 343 VFIO_CCW_NUM_REGIONS + 344 private->num_regions); 345 346 i = info->index - VFIO_CCW_NUM_REGIONS; 347 348 info->offset = VFIO_CCW_INDEX_TO_OFFSET(info->index); 349 info->size = private->region[i].size; 350 info->flags = private->region[i].flags; 351 352 cap_type.type = private->region[i].type; 353 cap_type.subtype = private->region[i].subtype; 354 355 ret = vfio_info_add_capability(caps, &cap_type.header, 356 sizeof(cap_type)); 357 if (ret) 358 return ret; 359 } 360 } 361 return 0; 362 } 363 364 static int vfio_ccw_mdev_get_irq_info(struct vfio_irq_info *info) 365 { 366 switch (info->index) { 367 case VFIO_CCW_IO_IRQ_INDEX: 368 case VFIO_CCW_CRW_IRQ_INDEX: 369 case VFIO_CCW_REQ_IRQ_INDEX: 370 info->count = 1; 371 info->flags = VFIO_IRQ_INFO_EVENTFD; 372 break; 373 default: 374 return -EINVAL; 375 } 376 377 return 0; 378 } 379 380 static int vfio_ccw_mdev_set_irqs(struct vfio_ccw_private *private, 381 uint32_t flags, 382 uint32_t index, 383 void __user *data) 384 { 385 struct eventfd_ctx **ctx; 386 387 if (!(flags & VFIO_IRQ_SET_ACTION_TRIGGER)) 388 return -EINVAL; 389 390 switch (index) { 391 case VFIO_CCW_IO_IRQ_INDEX: 392 ctx = &private->io_trigger; 393 break; 394 case VFIO_CCW_CRW_IRQ_INDEX: 395 ctx = &private->crw_trigger; 396 break; 397 case VFIO_CCW_REQ_IRQ_INDEX: 398 ctx = &private->req_trigger; 399 break; 400 default: 401 return -EINVAL; 402 } 403 404 switch (flags & VFIO_IRQ_SET_DATA_TYPE_MASK) { 405 case VFIO_IRQ_SET_DATA_NONE: 406 { 407 if (*ctx) 408 eventfd_signal(*ctx); 409 return 0; 410 } 411 case VFIO_IRQ_SET_DATA_BOOL: 412 { 413 uint8_t trigger; 414 415 if (get_user(trigger, (uint8_t __user *)data)) 416 return -EFAULT; 417 418 if (trigger && *ctx) 419 eventfd_signal(*ctx); 420 return 0; 421 } 422 case VFIO_IRQ_SET_DATA_EVENTFD: 423 { 424 int32_t fd; 425 426 if (get_user(fd, (int32_t __user *)data)) 427 return -EFAULT; 428 429 if (fd == -1) { 430 if (*ctx) 431 eventfd_ctx_put(*ctx); 432 *ctx = NULL; 433 } else if (fd >= 0) { 434 struct eventfd_ctx *efdctx; 435 436 efdctx = eventfd_ctx_fdget(fd); 437 if (IS_ERR(efdctx)) 438 return PTR_ERR(efdctx); 439 440 if (*ctx) 441 eventfd_ctx_put(*ctx); 442 443 *ctx = efdctx; 444 } else 445 return -EINVAL; 446 447 return 0; 448 } 449 default: 450 return -EINVAL; 451 } 452 } 453 454 int vfio_ccw_register_dev_region(struct vfio_ccw_private *private, 455 unsigned int subtype, 456 const struct vfio_ccw_regops *ops, 457 size_t size, u32 flags, void *data) 458 { 459 struct vfio_ccw_region *region; 460 461 region = krealloc(private->region, 462 (private->num_regions + 1) * sizeof(*region), 463 GFP_KERNEL); 464 if (!region) 465 return -ENOMEM; 466 467 private->region = region; 468 private->region[private->num_regions].type = VFIO_REGION_TYPE_CCW; 469 private->region[private->num_regions].subtype = subtype; 470 private->region[private->num_regions].ops = ops; 471 private->region[private->num_regions].size = size; 472 private->region[private->num_regions].flags = flags; 473 private->region[private->num_regions].data = data; 474 475 private->num_regions++; 476 477 return 0; 478 } 479 480 void vfio_ccw_unregister_dev_regions(struct vfio_ccw_private *private) 481 { 482 int i; 483 484 for (i = 0; i < private->num_regions; i++) 485 private->region[i].ops->release(private, &private->region[i]); 486 private->num_regions = 0; 487 kfree(private->region); 488 private->region = NULL; 489 } 490 491 static ssize_t vfio_ccw_mdev_ioctl(struct vfio_device *vdev, 492 unsigned int cmd, 493 unsigned long arg) 494 { 495 struct vfio_ccw_private *private = 496 container_of(vdev, struct vfio_ccw_private, vdev); 497 int ret = 0; 498 unsigned long minsz; 499 500 switch (cmd) { 501 case VFIO_DEVICE_GET_INFO: 502 { 503 struct vfio_device_info info; 504 505 minsz = offsetofend(struct vfio_device_info, num_irqs); 506 507 if (copy_from_user(&info, (void __user *)arg, minsz)) 508 return -EFAULT; 509 510 if (info.argsz < minsz) 511 return -EINVAL; 512 513 ret = vfio_ccw_mdev_get_device_info(private, &info); 514 if (ret) 515 return ret; 516 517 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; 518 } 519 case VFIO_DEVICE_GET_IRQ_INFO: 520 { 521 struct vfio_irq_info info; 522 523 minsz = offsetofend(struct vfio_irq_info, count); 524 525 if (copy_from_user(&info, (void __user *)arg, minsz)) 526 return -EFAULT; 527 528 if (info.argsz < minsz || info.index >= VFIO_CCW_NUM_IRQS) 529 return -EINVAL; 530 531 ret = vfio_ccw_mdev_get_irq_info(&info); 532 if (ret) 533 return ret; 534 535 if (info.count == -1) 536 return -EINVAL; 537 538 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; 539 } 540 case VFIO_DEVICE_SET_IRQS: 541 { 542 struct vfio_irq_set hdr; 543 size_t data_size; 544 void __user *data; 545 546 minsz = offsetofend(struct vfio_irq_set, count); 547 548 if (copy_from_user(&hdr, (void __user *)arg, minsz)) 549 return -EFAULT; 550 551 ret = vfio_set_irqs_validate_and_prepare(&hdr, 1, 552 VFIO_CCW_NUM_IRQS, 553 &data_size); 554 if (ret) 555 return ret; 556 557 data = (void __user *)(arg + minsz); 558 return vfio_ccw_mdev_set_irqs(private, hdr.flags, hdr.index, 559 data); 560 } 561 case VFIO_DEVICE_RESET: 562 return vfio_ccw_mdev_reset(private); 563 default: 564 return -ENOTTY; 565 } 566 } 567 568 /* Request removal of the device*/ 569 static void vfio_ccw_mdev_request(struct vfio_device *vdev, unsigned int count) 570 { 571 struct vfio_ccw_private *private = 572 container_of(vdev, struct vfio_ccw_private, vdev); 573 struct device *dev = vdev->dev; 574 575 if (private->req_trigger) { 576 if (!(count % 10)) 577 dev_notice_ratelimited(dev, 578 "Relaying device request to user (#%u)\n", 579 count); 580 581 eventfd_signal(private->req_trigger); 582 } else if (count == 0) { 583 dev_notice(dev, 584 "No device request channel registered, blocked until released by user\n"); 585 } 586 } 587 588 static const struct vfio_device_ops vfio_ccw_dev_ops = { 589 .init = vfio_ccw_mdev_init_dev, 590 .release = vfio_ccw_mdev_release_dev, 591 .open_device = vfio_ccw_mdev_open_device, 592 .close_device = vfio_ccw_mdev_close_device, 593 .read = vfio_ccw_mdev_read, 594 .write = vfio_ccw_mdev_write, 595 .ioctl = vfio_ccw_mdev_ioctl, 596 .get_region_info_caps = vfio_ccw_mdev_ioctl_get_region_info, 597 .request = vfio_ccw_mdev_request, 598 .dma_unmap = vfio_ccw_dma_unmap, 599 .bind_iommufd = vfio_iommufd_emulated_bind, 600 .unbind_iommufd = vfio_iommufd_emulated_unbind, 601 .attach_ioas = vfio_iommufd_emulated_attach_ioas, 602 .detach_ioas = vfio_iommufd_emulated_detach_ioas, 603 }; 604 605 struct mdev_driver vfio_ccw_mdev_driver = { 606 .device_api = VFIO_DEVICE_API_CCW_STRING, 607 .max_instances = 1, 608 .driver = { 609 .name = "vfio_ccw_mdev", 610 .owner = THIS_MODULE, 611 .mod_name = KBUILD_MODNAME, 612 }, 613 .probe = vfio_ccw_mdev_probe, 614 .remove = vfio_ccw_mdev_remove, 615 }; 616