1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Physical device callbacks for vfio_ccw 4 * 5 * Copyright IBM Corp. 2017 6 * Copyright Red Hat, Inc. 2019 7 * 8 * Author(s): Dong Jia Shi <bjsdjshi@linux.vnet.ibm.com> 9 * Xiao Feng Ren <renxiaof@linux.vnet.ibm.com> 10 * Cornelia Huck <cohuck@redhat.com> 11 */ 12 13 #include <linux/vfio.h> 14 #include <linux/nospec.h> 15 #include <linux/slab.h> 16 17 #include "vfio_ccw_private.h" 18 19 static const struct vfio_device_ops vfio_ccw_dev_ops; 20 21 static int vfio_ccw_mdev_reset(struct vfio_ccw_private *private) 22 { 23 /* 24 * If the FSM state is seen as Not Operational after closing 25 * and re-opening the mdev, return an error. 26 */ 27 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_CLOSE); 28 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_OPEN); 29 if (private->state == VFIO_CCW_STATE_NOT_OPER) 30 return -EINVAL; 31 32 return 0; 33 } 34 35 static void vfio_ccw_dma_unmap(struct vfio_device *vdev, u64 iova, u64 length) 36 { 37 struct vfio_ccw_private *private = 38 container_of(vdev, struct vfio_ccw_private, vdev); 39 40 /* Drivers MUST unpin pages in response to an invalidation. */ 41 if (!cp_iova_pinned(&private->cp, iova, length)) 42 return; 43 44 vfio_ccw_mdev_reset(private); 45 } 46 47 static int vfio_ccw_mdev_init_dev(struct vfio_device *vdev) 48 { 49 struct vfio_ccw_private *private = 50 container_of(vdev, struct vfio_ccw_private, vdev); 51 52 mutex_init(&private->io_mutex); 53 private->state = VFIO_CCW_STATE_STANDBY; 54 INIT_LIST_HEAD(&private->crw); 55 INIT_WORK(&private->io_work, vfio_ccw_sch_io_todo); 56 INIT_WORK(&private->crw_work, vfio_ccw_crw_todo); 57 58 private->cp.guest_cp = kzalloc_objs(struct ccw1, CCWCHAIN_LEN_MAX); 59 if (!private->cp.guest_cp) 60 goto out_free_private; 61 62 private->io_region = kmem_cache_zalloc(vfio_ccw_io_region, 63 GFP_KERNEL | GFP_DMA); 64 if (!private->io_region) 65 goto out_free_cp; 66 67 private->cmd_region = kmem_cache_zalloc(vfio_ccw_cmd_region, 68 GFP_KERNEL | GFP_DMA); 69 if (!private->cmd_region) 70 goto out_free_io; 71 72 private->schib_region = kmem_cache_zalloc(vfio_ccw_schib_region, 73 GFP_KERNEL | GFP_DMA); 74 if (!private->schib_region) 75 goto out_free_cmd; 76 77 private->crw_region = kmem_cache_zalloc(vfio_ccw_crw_region, 78 GFP_KERNEL | GFP_DMA); 79 if (!private->crw_region) 80 goto out_free_schib; 81 82 return 0; 83 84 out_free_schib: 85 kmem_cache_free(vfio_ccw_schib_region, private->schib_region); 86 out_free_cmd: 87 kmem_cache_free(vfio_ccw_cmd_region, private->cmd_region); 88 out_free_io: 89 kmem_cache_free(vfio_ccw_io_region, private->io_region); 90 out_free_cp: 91 kfree(private->cp.guest_cp); 92 out_free_private: 93 mutex_destroy(&private->io_mutex); 94 return -ENOMEM; 95 } 96 97 static int vfio_ccw_mdev_probe(struct mdev_device *mdev) 98 { 99 struct subchannel *sch = to_subchannel(mdev->dev.parent); 100 struct vfio_ccw_parent *parent = dev_get_drvdata(&sch->dev); 101 struct vfio_ccw_private *private; 102 int ret; 103 104 private = vfio_alloc_device(vfio_ccw_private, vdev, &mdev->dev, 105 &vfio_ccw_dev_ops); 106 if (IS_ERR(private)) 107 return PTR_ERR(private); 108 109 dev_set_drvdata(&parent->dev, private); 110 111 VFIO_CCW_MSG_EVENT(2, "sch %x.%x.%04x: create\n", 112 sch->schid.cssid, 113 sch->schid.ssid, 114 sch->schid.sch_no); 115 116 ret = vfio_register_emulated_iommu_dev(&private->vdev); 117 if (ret) 118 goto err_put_vdev; 119 dev_set_drvdata(&mdev->dev, private); 120 return 0; 121 122 err_put_vdev: 123 dev_set_drvdata(&parent->dev, NULL); 124 vfio_put_device(&private->vdev); 125 return ret; 126 } 127 128 static void vfio_ccw_mdev_release_dev(struct vfio_device *vdev) 129 { 130 struct vfio_ccw_private *private = 131 container_of(vdev, struct vfio_ccw_private, vdev); 132 struct vfio_ccw_crw *crw, *temp; 133 134 list_for_each_entry_safe(crw, temp, &private->crw, next) { 135 list_del(&crw->next); 136 kfree(crw); 137 } 138 139 kmem_cache_free(vfio_ccw_crw_region, private->crw_region); 140 kmem_cache_free(vfio_ccw_schib_region, private->schib_region); 141 kmem_cache_free(vfio_ccw_cmd_region, private->cmd_region); 142 kmem_cache_free(vfio_ccw_io_region, private->io_region); 143 kfree(private->cp.guest_cp); 144 mutex_destroy(&private->io_mutex); 145 } 146 147 static void vfio_ccw_mdev_remove(struct mdev_device *mdev) 148 { 149 struct subchannel *sch = to_subchannel(mdev->dev.parent); 150 struct vfio_ccw_parent *parent = dev_get_drvdata(&sch->dev); 151 struct vfio_ccw_private *private = dev_get_drvdata(&parent->dev); 152 153 VFIO_CCW_MSG_EVENT(2, "sch %x.%x.%04x: remove\n", 154 sch->schid.cssid, 155 sch->schid.ssid, 156 sch->schid.sch_no); 157 158 vfio_unregister_group_dev(&private->vdev); 159 160 dev_set_drvdata(&parent->dev, NULL); 161 vfio_put_device(&private->vdev); 162 } 163 164 static int vfio_ccw_mdev_open_device(struct vfio_device *vdev) 165 { 166 struct vfio_ccw_private *private = 167 container_of(vdev, struct vfio_ccw_private, vdev); 168 int ret; 169 170 /* Device cannot simply be opened again from this state */ 171 if (private->state == VFIO_CCW_STATE_NOT_OPER) 172 return -EINVAL; 173 174 ret = vfio_ccw_register_async_dev_regions(private); 175 if (ret) 176 return ret; 177 178 ret = vfio_ccw_register_schib_dev_regions(private); 179 if (ret) 180 goto out_unregister; 181 182 ret = vfio_ccw_register_crw_dev_regions(private); 183 if (ret) 184 goto out_unregister; 185 186 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_OPEN); 187 if (private->state == VFIO_CCW_STATE_NOT_OPER) { 188 ret = -EINVAL; 189 goto out_unregister; 190 } 191 192 return ret; 193 194 out_unregister: 195 vfio_ccw_unregister_dev_regions(private); 196 return ret; 197 } 198 199 static void vfio_ccw_mdev_close_device(struct vfio_device *vdev) 200 { 201 struct vfio_ccw_private *private = 202 container_of(vdev, struct vfio_ccw_private, vdev); 203 204 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_CLOSE); 205 vfio_ccw_unregister_dev_regions(private); 206 } 207 208 static ssize_t vfio_ccw_mdev_read_io_region(struct vfio_ccw_private *private, 209 char __user *buf, size_t count, 210 loff_t *ppos) 211 { 212 loff_t pos = *ppos & VFIO_CCW_OFFSET_MASK; 213 struct ccw_io_region *region; 214 int ret; 215 216 if (pos + count > sizeof(*region)) 217 return -EINVAL; 218 219 mutex_lock(&private->io_mutex); 220 region = private->io_region; 221 if (copy_to_user(buf, (void *)region + pos, count)) 222 ret = -EFAULT; 223 else 224 ret = count; 225 mutex_unlock(&private->io_mutex); 226 return ret; 227 } 228 229 static ssize_t vfio_ccw_mdev_read(struct vfio_device *vdev, 230 char __user *buf, 231 size_t count, 232 loff_t *ppos) 233 { 234 struct vfio_ccw_private *private = 235 container_of(vdev, struct vfio_ccw_private, vdev); 236 unsigned int index = VFIO_CCW_OFFSET_TO_INDEX(*ppos); 237 238 if (index >= VFIO_CCW_NUM_REGIONS + private->num_regions) 239 return -EINVAL; 240 241 switch (index) { 242 case VFIO_CCW_CONFIG_REGION_INDEX: 243 return vfio_ccw_mdev_read_io_region(private, buf, count, ppos); 244 default: 245 index -= VFIO_CCW_NUM_REGIONS; 246 return private->region[index].ops->read(private, buf, count, 247 ppos); 248 } 249 250 return -EINVAL; 251 } 252 253 static ssize_t vfio_ccw_mdev_write_io_region(struct vfio_ccw_private *private, 254 const char __user *buf, 255 size_t count, loff_t *ppos) 256 { 257 loff_t pos = *ppos & VFIO_CCW_OFFSET_MASK; 258 struct ccw_io_region *region; 259 int ret; 260 261 if (pos + count > sizeof(*region)) 262 return -EINVAL; 263 264 if (!mutex_trylock(&private->io_mutex)) 265 return -EAGAIN; 266 267 region = private->io_region; 268 if (copy_from_user((void *)region + pos, buf, count)) { 269 ret = -EFAULT; 270 goto out_unlock; 271 } 272 273 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_IO_REQ); 274 ret = (region->ret_code != 0) ? region->ret_code : count; 275 276 out_unlock: 277 mutex_unlock(&private->io_mutex); 278 return ret; 279 } 280 281 static ssize_t vfio_ccw_mdev_write(struct vfio_device *vdev, 282 const char __user *buf, 283 size_t count, 284 loff_t *ppos) 285 { 286 struct vfio_ccw_private *private = 287 container_of(vdev, struct vfio_ccw_private, vdev); 288 unsigned int index = VFIO_CCW_OFFSET_TO_INDEX(*ppos); 289 290 if (index >= VFIO_CCW_NUM_REGIONS + private->num_regions) 291 return -EINVAL; 292 293 switch (index) { 294 case VFIO_CCW_CONFIG_REGION_INDEX: 295 return vfio_ccw_mdev_write_io_region(private, buf, count, ppos); 296 default: 297 index -= VFIO_CCW_NUM_REGIONS; 298 return private->region[index].ops->write(private, buf, count, 299 ppos); 300 } 301 302 return -EINVAL; 303 } 304 305 static int vfio_ccw_mdev_get_device_info(struct vfio_ccw_private *private, 306 struct vfio_device_info *info) 307 { 308 info->flags = VFIO_DEVICE_FLAGS_CCW | VFIO_DEVICE_FLAGS_RESET; 309 info->num_regions = VFIO_CCW_NUM_REGIONS + private->num_regions; 310 info->num_irqs = VFIO_CCW_NUM_IRQS; 311 312 return 0; 313 } 314 315 static int vfio_ccw_mdev_ioctl_get_region_info(struct vfio_device *vdev, 316 struct vfio_region_info *info, 317 struct vfio_info_cap *caps) 318 { 319 struct vfio_ccw_private *private = 320 container_of(vdev, struct vfio_ccw_private, vdev); 321 int i; 322 323 switch (info->index) { 324 case VFIO_CCW_CONFIG_REGION_INDEX: 325 info->offset = 0; 326 info->size = sizeof(struct ccw_io_region); 327 info->flags = VFIO_REGION_INFO_FLAG_READ 328 | VFIO_REGION_INFO_FLAG_WRITE; 329 return 0; 330 default: /* all other regions are handled via capability chain */ 331 { 332 struct vfio_region_info_cap_type cap_type = { 333 .header.id = VFIO_REGION_INFO_CAP_TYPE, 334 .header.version = 1 }; 335 int ret; 336 337 if (info->index >= 338 VFIO_CCW_NUM_REGIONS + private->num_regions) 339 return -EINVAL; 340 341 info->index = array_index_nospec(info->index, 342 VFIO_CCW_NUM_REGIONS + 343 private->num_regions); 344 345 i = info->index - VFIO_CCW_NUM_REGIONS; 346 347 info->offset = VFIO_CCW_INDEX_TO_OFFSET(info->index); 348 info->size = private->region[i].size; 349 info->flags = private->region[i].flags; 350 351 cap_type.type = private->region[i].type; 352 cap_type.subtype = private->region[i].subtype; 353 354 ret = vfio_info_add_capability(caps, &cap_type.header, 355 sizeof(cap_type)); 356 if (ret) 357 return ret; 358 } 359 } 360 return 0; 361 } 362 363 static int vfio_ccw_mdev_get_irq_info(struct vfio_irq_info *info) 364 { 365 switch (info->index) { 366 case VFIO_CCW_IO_IRQ_INDEX: 367 case VFIO_CCW_CRW_IRQ_INDEX: 368 case VFIO_CCW_REQ_IRQ_INDEX: 369 info->count = 1; 370 info->flags = VFIO_IRQ_INFO_EVENTFD; 371 break; 372 default: 373 return -EINVAL; 374 } 375 376 return 0; 377 } 378 379 static int vfio_ccw_mdev_set_irqs(struct vfio_ccw_private *private, 380 uint32_t flags, 381 uint32_t index, 382 void __user *data) 383 { 384 struct eventfd_ctx **ctx; 385 386 if (!(flags & VFIO_IRQ_SET_ACTION_TRIGGER)) 387 return -EINVAL; 388 389 switch (index) { 390 case VFIO_CCW_IO_IRQ_INDEX: 391 ctx = &private->io_trigger; 392 break; 393 case VFIO_CCW_CRW_IRQ_INDEX: 394 ctx = &private->crw_trigger; 395 break; 396 case VFIO_CCW_REQ_IRQ_INDEX: 397 ctx = &private->req_trigger; 398 break; 399 default: 400 return -EINVAL; 401 } 402 403 switch (flags & VFIO_IRQ_SET_DATA_TYPE_MASK) { 404 case VFIO_IRQ_SET_DATA_NONE: 405 { 406 if (*ctx) 407 eventfd_signal(*ctx); 408 return 0; 409 } 410 case VFIO_IRQ_SET_DATA_BOOL: 411 { 412 uint8_t trigger; 413 414 if (get_user(trigger, (uint8_t __user *)data)) 415 return -EFAULT; 416 417 if (trigger && *ctx) 418 eventfd_signal(*ctx); 419 return 0; 420 } 421 case VFIO_IRQ_SET_DATA_EVENTFD: 422 { 423 int32_t fd; 424 425 if (get_user(fd, (int32_t __user *)data)) 426 return -EFAULT; 427 428 if (fd == -1) { 429 if (*ctx) 430 eventfd_ctx_put(*ctx); 431 *ctx = NULL; 432 } else if (fd >= 0) { 433 struct eventfd_ctx *efdctx; 434 435 efdctx = eventfd_ctx_fdget(fd); 436 if (IS_ERR(efdctx)) 437 return PTR_ERR(efdctx); 438 439 if (*ctx) 440 eventfd_ctx_put(*ctx); 441 442 *ctx = efdctx; 443 } else 444 return -EINVAL; 445 446 return 0; 447 } 448 default: 449 return -EINVAL; 450 } 451 } 452 453 int vfio_ccw_register_dev_region(struct vfio_ccw_private *private, 454 unsigned int subtype, 455 const struct vfio_ccw_regops *ops, 456 size_t size, u32 flags, void *data) 457 { 458 struct vfio_ccw_region *region; 459 460 region = krealloc(private->region, 461 (private->num_regions + 1) * sizeof(*region), 462 GFP_KERNEL); 463 if (!region) 464 return -ENOMEM; 465 466 private->region = region; 467 private->region[private->num_regions].type = VFIO_REGION_TYPE_CCW; 468 private->region[private->num_regions].subtype = subtype; 469 private->region[private->num_regions].ops = ops; 470 private->region[private->num_regions].size = size; 471 private->region[private->num_regions].flags = flags; 472 private->region[private->num_regions].data = data; 473 474 private->num_regions++; 475 476 return 0; 477 } 478 479 void vfio_ccw_unregister_dev_regions(struct vfio_ccw_private *private) 480 { 481 int i; 482 483 for (i = 0; i < private->num_regions; i++) 484 private->region[i].ops->release(private, &private->region[i]); 485 private->num_regions = 0; 486 kfree(private->region); 487 private->region = NULL; 488 } 489 490 static ssize_t vfio_ccw_mdev_ioctl(struct vfio_device *vdev, 491 unsigned int cmd, 492 unsigned long arg) 493 { 494 struct vfio_ccw_private *private = 495 container_of(vdev, struct vfio_ccw_private, vdev); 496 int ret = 0; 497 unsigned long minsz; 498 499 switch (cmd) { 500 case VFIO_DEVICE_GET_INFO: 501 { 502 struct vfio_device_info info; 503 504 minsz = offsetofend(struct vfio_device_info, num_irqs); 505 506 if (copy_from_user(&info, (void __user *)arg, minsz)) 507 return -EFAULT; 508 509 if (info.argsz < minsz) 510 return -EINVAL; 511 512 ret = vfio_ccw_mdev_get_device_info(private, &info); 513 if (ret) 514 return ret; 515 516 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; 517 } 518 case VFIO_DEVICE_GET_IRQ_INFO: 519 { 520 struct vfio_irq_info info; 521 522 minsz = offsetofend(struct vfio_irq_info, count); 523 524 if (copy_from_user(&info, (void __user *)arg, minsz)) 525 return -EFAULT; 526 527 if (info.argsz < minsz || info.index >= VFIO_CCW_NUM_IRQS) 528 return -EINVAL; 529 530 ret = vfio_ccw_mdev_get_irq_info(&info); 531 if (ret) 532 return ret; 533 534 if (info.count == -1) 535 return -EINVAL; 536 537 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; 538 } 539 case VFIO_DEVICE_SET_IRQS: 540 { 541 struct vfio_irq_set hdr; 542 size_t data_size; 543 void __user *data; 544 545 minsz = offsetofend(struct vfio_irq_set, count); 546 547 if (copy_from_user(&hdr, (void __user *)arg, minsz)) 548 return -EFAULT; 549 550 ret = vfio_set_irqs_validate_and_prepare(&hdr, 1, 551 VFIO_CCW_NUM_IRQS, 552 &data_size); 553 if (ret) 554 return ret; 555 556 data = (void __user *)(arg + minsz); 557 return vfio_ccw_mdev_set_irqs(private, hdr.flags, hdr.index, 558 data); 559 } 560 case VFIO_DEVICE_RESET: 561 return vfio_ccw_mdev_reset(private); 562 default: 563 return -ENOTTY; 564 } 565 } 566 567 /* Request removal of the device*/ 568 static void vfio_ccw_mdev_request(struct vfio_device *vdev, unsigned int count) 569 { 570 struct vfio_ccw_private *private = 571 container_of(vdev, struct vfio_ccw_private, vdev); 572 struct device *dev = vdev->dev; 573 574 if (private->req_trigger) { 575 if (!(count % 10)) 576 dev_notice_ratelimited(dev, 577 "Relaying device request to user (#%u)\n", 578 count); 579 580 eventfd_signal(private->req_trigger); 581 } else if (count == 0) { 582 dev_notice(dev, 583 "No device request channel registered, blocked until released by user\n"); 584 } 585 } 586 587 static const struct vfio_device_ops vfio_ccw_dev_ops = { 588 .init = vfio_ccw_mdev_init_dev, 589 .release = vfio_ccw_mdev_release_dev, 590 .open_device = vfio_ccw_mdev_open_device, 591 .close_device = vfio_ccw_mdev_close_device, 592 .read = vfio_ccw_mdev_read, 593 .write = vfio_ccw_mdev_write, 594 .ioctl = vfio_ccw_mdev_ioctl, 595 .get_region_info_caps = vfio_ccw_mdev_ioctl_get_region_info, 596 .request = vfio_ccw_mdev_request, 597 .dma_unmap = vfio_ccw_dma_unmap, 598 .bind_iommufd = vfio_iommufd_emulated_bind, 599 .unbind_iommufd = vfio_iommufd_emulated_unbind, 600 .attach_ioas = vfio_iommufd_emulated_attach_ioas, 601 .detach_ioas = vfio_iommufd_emulated_detach_ioas, 602 }; 603 604 struct mdev_driver vfio_ccw_mdev_driver = { 605 .device_api = VFIO_DEVICE_API_CCW_STRING, 606 .max_instances = 1, 607 .driver = { 608 .name = "vfio_ccw_mdev", 609 .owner = THIS_MODULE, 610 .mod_name = KBUILD_MODNAME, 611 }, 612 .probe = vfio_ccw_mdev_probe, 613 .remove = vfio_ccw_mdev_remove, 614 }; 615