1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Physical device callbacks for vfio_ccw
4 *
5 * Copyright IBM Corp. 2017
6 * Copyright Red Hat, Inc. 2019
7 *
8 * Author(s): Dong Jia Shi <bjsdjshi@linux.vnet.ibm.com>
9 * Xiao Feng Ren <renxiaof@linux.vnet.ibm.com>
10 * Cornelia Huck <cohuck@redhat.com>
11 */
12
13 #include <linux/vfio.h>
14 #include <linux/nospec.h>
15 #include <linux/slab.h>
16
17 #include "vfio_ccw_private.h"
18
19 static const struct vfio_device_ops vfio_ccw_dev_ops;
20
vfio_ccw_mdev_reset(struct vfio_ccw_private * private)21 static int vfio_ccw_mdev_reset(struct vfio_ccw_private *private)
22 {
23 /*
24 * If the FSM state is seen as Not Operational after closing
25 * and re-opening the mdev, return an error.
26 */
27 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_CLOSE);
28 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_OPEN);
29 if (private->state == VFIO_CCW_STATE_NOT_OPER)
30 return -EINVAL;
31
32 return 0;
33 }
34
vfio_ccw_dma_unmap(struct vfio_device * vdev,u64 iova,u64 length)35 static void vfio_ccw_dma_unmap(struct vfio_device *vdev, u64 iova, u64 length)
36 {
37 struct vfio_ccw_private *private =
38 container_of(vdev, struct vfio_ccw_private, vdev);
39
40 /* Drivers MUST unpin pages in response to an invalidation. */
41 if (!cp_iova_pinned(&private->cp, iova, length))
42 return;
43
44 vfio_ccw_mdev_reset(private);
45 }
46
vfio_ccw_mdev_init_dev(struct vfio_device * vdev)47 static int vfio_ccw_mdev_init_dev(struct vfio_device *vdev)
48 {
49 struct vfio_ccw_private *private =
50 container_of(vdev, struct vfio_ccw_private, vdev);
51
52 mutex_init(&private->io_mutex);
53 private->state = VFIO_CCW_STATE_STANDBY;
54 INIT_LIST_HEAD(&private->crw);
55 INIT_WORK(&private->io_work, vfio_ccw_sch_io_todo);
56 INIT_WORK(&private->crw_work, vfio_ccw_crw_todo);
57
58 private->cp.guest_cp = kzalloc_objs(struct ccw1, CCWCHAIN_LEN_MAX);
59 if (!private->cp.guest_cp)
60 goto out_free_private;
61
62 private->io_region = kmem_cache_zalloc(vfio_ccw_io_region,
63 GFP_KERNEL | GFP_DMA);
64 if (!private->io_region)
65 goto out_free_cp;
66
67 private->cmd_region = kmem_cache_zalloc(vfio_ccw_cmd_region,
68 GFP_KERNEL | GFP_DMA);
69 if (!private->cmd_region)
70 goto out_free_io;
71
72 private->schib_region = kmem_cache_zalloc(vfio_ccw_schib_region,
73 GFP_KERNEL | GFP_DMA);
74 if (!private->schib_region)
75 goto out_free_cmd;
76
77 private->crw_region = kmem_cache_zalloc(vfio_ccw_crw_region,
78 GFP_KERNEL | GFP_DMA);
79 if (!private->crw_region)
80 goto out_free_schib;
81
82 return 0;
83
84 out_free_schib:
85 kmem_cache_free(vfio_ccw_schib_region, private->schib_region);
86 out_free_cmd:
87 kmem_cache_free(vfio_ccw_cmd_region, private->cmd_region);
88 out_free_io:
89 kmem_cache_free(vfio_ccw_io_region, private->io_region);
90 out_free_cp:
91 kfree(private->cp.guest_cp);
92 out_free_private:
93 mutex_destroy(&private->io_mutex);
94 return -ENOMEM;
95 }
96
vfio_ccw_mdev_probe(struct mdev_device * mdev)97 static int vfio_ccw_mdev_probe(struct mdev_device *mdev)
98 {
99 struct subchannel *sch = to_subchannel(mdev->dev.parent);
100 struct vfio_ccw_parent *parent = dev_get_drvdata(&sch->dev);
101 struct vfio_ccw_private *private;
102 int ret;
103
104 private = vfio_alloc_device(vfio_ccw_private, vdev, &mdev->dev,
105 &vfio_ccw_dev_ops);
106 if (IS_ERR(private))
107 return PTR_ERR(private);
108
109 dev_set_drvdata(&parent->dev, private);
110
111 VFIO_CCW_MSG_EVENT(2, "sch %x.%x.%04x: create\n",
112 sch->schid.cssid,
113 sch->schid.ssid,
114 sch->schid.sch_no);
115
116 ret = vfio_register_emulated_iommu_dev(&private->vdev);
117 if (ret)
118 goto err_put_vdev;
119 dev_set_drvdata(&mdev->dev, private);
120 return 0;
121
122 err_put_vdev:
123 dev_set_drvdata(&parent->dev, NULL);
124 vfio_put_device(&private->vdev);
125 return ret;
126 }
127
vfio_ccw_mdev_release_dev(struct vfio_device * vdev)128 static void vfio_ccw_mdev_release_dev(struct vfio_device *vdev)
129 {
130 struct vfio_ccw_private *private =
131 container_of(vdev, struct vfio_ccw_private, vdev);
132 struct vfio_ccw_crw *crw, *temp;
133
134 list_for_each_entry_safe(crw, temp, &private->crw, next) {
135 list_del(&crw->next);
136 kfree(crw);
137 }
138
139 kmem_cache_free(vfio_ccw_crw_region, private->crw_region);
140 kmem_cache_free(vfio_ccw_schib_region, private->schib_region);
141 kmem_cache_free(vfio_ccw_cmd_region, private->cmd_region);
142 kmem_cache_free(vfio_ccw_io_region, private->io_region);
143 kfree(private->cp.guest_cp);
144 mutex_destroy(&private->io_mutex);
145 }
146
vfio_ccw_mdev_remove(struct mdev_device * mdev)147 static void vfio_ccw_mdev_remove(struct mdev_device *mdev)
148 {
149 struct subchannel *sch = to_subchannel(mdev->dev.parent);
150 struct vfio_ccw_parent *parent = dev_get_drvdata(&sch->dev);
151 struct vfio_ccw_private *private = dev_get_drvdata(&parent->dev);
152
153 VFIO_CCW_MSG_EVENT(2, "sch %x.%x.%04x: remove\n",
154 sch->schid.cssid,
155 sch->schid.ssid,
156 sch->schid.sch_no);
157
158 vfio_unregister_group_dev(&private->vdev);
159
160 dev_set_drvdata(&parent->dev, NULL);
161 vfio_put_device(&private->vdev);
162 }
163
vfio_ccw_mdev_open_device(struct vfio_device * vdev)164 static int vfio_ccw_mdev_open_device(struct vfio_device *vdev)
165 {
166 struct vfio_ccw_private *private =
167 container_of(vdev, struct vfio_ccw_private, vdev);
168 int ret;
169
170 /* Device cannot simply be opened again from this state */
171 if (private->state == VFIO_CCW_STATE_NOT_OPER)
172 return -EINVAL;
173
174 ret = vfio_ccw_register_async_dev_regions(private);
175 if (ret)
176 return ret;
177
178 ret = vfio_ccw_register_schib_dev_regions(private);
179 if (ret)
180 goto out_unregister;
181
182 ret = vfio_ccw_register_crw_dev_regions(private);
183 if (ret)
184 goto out_unregister;
185
186 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_OPEN);
187 if (private->state == VFIO_CCW_STATE_NOT_OPER) {
188 ret = -EINVAL;
189 goto out_unregister;
190 }
191
192 return ret;
193
194 out_unregister:
195 vfio_ccw_unregister_dev_regions(private);
196 return ret;
197 }
198
vfio_ccw_mdev_close_device(struct vfio_device * vdev)199 static void vfio_ccw_mdev_close_device(struct vfio_device *vdev)
200 {
201 struct vfio_ccw_private *private =
202 container_of(vdev, struct vfio_ccw_private, vdev);
203
204 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_CLOSE);
205 vfio_ccw_unregister_dev_regions(private);
206 }
207
vfio_ccw_mdev_read_io_region(struct vfio_ccw_private * private,char __user * buf,size_t count,loff_t * ppos)208 static ssize_t vfio_ccw_mdev_read_io_region(struct vfio_ccw_private *private,
209 char __user *buf, size_t count,
210 loff_t *ppos)
211 {
212 loff_t pos = *ppos & VFIO_CCW_OFFSET_MASK;
213 struct ccw_io_region *region;
214 int ret;
215
216 if (pos + count > sizeof(*region))
217 return -EINVAL;
218
219 mutex_lock(&private->io_mutex);
220 region = private->io_region;
221 if (copy_to_user(buf, (void *)region + pos, count))
222 ret = -EFAULT;
223 else
224 ret = count;
225 mutex_unlock(&private->io_mutex);
226 return ret;
227 }
228
vfio_ccw_mdev_read(struct vfio_device * vdev,char __user * buf,size_t count,loff_t * ppos)229 static ssize_t vfio_ccw_mdev_read(struct vfio_device *vdev,
230 char __user *buf,
231 size_t count,
232 loff_t *ppos)
233 {
234 struct vfio_ccw_private *private =
235 container_of(vdev, struct vfio_ccw_private, vdev);
236 unsigned int index = VFIO_CCW_OFFSET_TO_INDEX(*ppos);
237
238 if (index >= VFIO_CCW_NUM_REGIONS + private->num_regions)
239 return -EINVAL;
240
241 switch (index) {
242 case VFIO_CCW_CONFIG_REGION_INDEX:
243 return vfio_ccw_mdev_read_io_region(private, buf, count, ppos);
244 default:
245 index -= VFIO_CCW_NUM_REGIONS;
246 return private->region[index].ops->read(private, buf, count,
247 ppos);
248 }
249
250 return -EINVAL;
251 }
252
vfio_ccw_mdev_write_io_region(struct vfio_ccw_private * private,const char __user * buf,size_t count,loff_t * ppos)253 static ssize_t vfio_ccw_mdev_write_io_region(struct vfio_ccw_private *private,
254 const char __user *buf,
255 size_t count, loff_t *ppos)
256 {
257 loff_t pos = *ppos & VFIO_CCW_OFFSET_MASK;
258 struct ccw_io_region *region;
259 int ret;
260
261 if (pos + count > sizeof(*region))
262 return -EINVAL;
263
264 if (!mutex_trylock(&private->io_mutex))
265 return -EAGAIN;
266
267 region = private->io_region;
268 if (copy_from_user((void *)region + pos, buf, count)) {
269 ret = -EFAULT;
270 goto out_unlock;
271 }
272
273 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_IO_REQ);
274 ret = (region->ret_code != 0) ? region->ret_code : count;
275
276 out_unlock:
277 mutex_unlock(&private->io_mutex);
278 return ret;
279 }
280
vfio_ccw_mdev_write(struct vfio_device * vdev,const char __user * buf,size_t count,loff_t * ppos)281 static ssize_t vfio_ccw_mdev_write(struct vfio_device *vdev,
282 const char __user *buf,
283 size_t count,
284 loff_t *ppos)
285 {
286 struct vfio_ccw_private *private =
287 container_of(vdev, struct vfio_ccw_private, vdev);
288 unsigned int index = VFIO_CCW_OFFSET_TO_INDEX(*ppos);
289
290 if (index >= VFIO_CCW_NUM_REGIONS + private->num_regions)
291 return -EINVAL;
292
293 switch (index) {
294 case VFIO_CCW_CONFIG_REGION_INDEX:
295 return vfio_ccw_mdev_write_io_region(private, buf, count, ppos);
296 default:
297 index -= VFIO_CCW_NUM_REGIONS;
298 return private->region[index].ops->write(private, buf, count,
299 ppos);
300 }
301
302 return -EINVAL;
303 }
304
vfio_ccw_mdev_get_device_info(struct vfio_ccw_private * private,struct vfio_device_info * info)305 static int vfio_ccw_mdev_get_device_info(struct vfio_ccw_private *private,
306 struct vfio_device_info *info)
307 {
308 info->flags = VFIO_DEVICE_FLAGS_CCW | VFIO_DEVICE_FLAGS_RESET;
309 info->num_regions = VFIO_CCW_NUM_REGIONS + private->num_regions;
310 info->num_irqs = VFIO_CCW_NUM_IRQS;
311
312 return 0;
313 }
314
vfio_ccw_mdev_ioctl_get_region_info(struct vfio_device * vdev,struct vfio_region_info * info,struct vfio_info_cap * caps)315 static int vfio_ccw_mdev_ioctl_get_region_info(struct vfio_device *vdev,
316 struct vfio_region_info *info,
317 struct vfio_info_cap *caps)
318 {
319 struct vfio_ccw_private *private =
320 container_of(vdev, struct vfio_ccw_private, vdev);
321 int i;
322
323 switch (info->index) {
324 case VFIO_CCW_CONFIG_REGION_INDEX:
325 info->offset = 0;
326 info->size = sizeof(struct ccw_io_region);
327 info->flags = VFIO_REGION_INFO_FLAG_READ
328 | VFIO_REGION_INFO_FLAG_WRITE;
329 return 0;
330 default: /* all other regions are handled via capability chain */
331 {
332 struct vfio_region_info_cap_type cap_type = {
333 .header.id = VFIO_REGION_INFO_CAP_TYPE,
334 .header.version = 1 };
335 int ret;
336
337 if (info->index >=
338 VFIO_CCW_NUM_REGIONS + private->num_regions)
339 return -EINVAL;
340
341 info->index = array_index_nospec(info->index,
342 VFIO_CCW_NUM_REGIONS +
343 private->num_regions);
344
345 i = info->index - VFIO_CCW_NUM_REGIONS;
346
347 info->offset = VFIO_CCW_INDEX_TO_OFFSET(info->index);
348 info->size = private->region[i].size;
349 info->flags = private->region[i].flags;
350
351 cap_type.type = private->region[i].type;
352 cap_type.subtype = private->region[i].subtype;
353
354 ret = vfio_info_add_capability(caps, &cap_type.header,
355 sizeof(cap_type));
356 if (ret)
357 return ret;
358 }
359 }
360 return 0;
361 }
362
vfio_ccw_mdev_get_irq_info(struct vfio_irq_info * info)363 static int vfio_ccw_mdev_get_irq_info(struct vfio_irq_info *info)
364 {
365 switch (info->index) {
366 case VFIO_CCW_IO_IRQ_INDEX:
367 case VFIO_CCW_CRW_IRQ_INDEX:
368 case VFIO_CCW_REQ_IRQ_INDEX:
369 info->count = 1;
370 info->flags = VFIO_IRQ_INFO_EVENTFD;
371 break;
372 default:
373 return -EINVAL;
374 }
375
376 return 0;
377 }
378
vfio_ccw_mdev_set_irqs(struct vfio_ccw_private * private,uint32_t flags,uint32_t index,void __user * data)379 static int vfio_ccw_mdev_set_irqs(struct vfio_ccw_private *private,
380 uint32_t flags,
381 uint32_t index,
382 void __user *data)
383 {
384 struct eventfd_ctx **ctx;
385
386 if (!(flags & VFIO_IRQ_SET_ACTION_TRIGGER))
387 return -EINVAL;
388
389 switch (index) {
390 case VFIO_CCW_IO_IRQ_INDEX:
391 ctx = &private->io_trigger;
392 break;
393 case VFIO_CCW_CRW_IRQ_INDEX:
394 ctx = &private->crw_trigger;
395 break;
396 case VFIO_CCW_REQ_IRQ_INDEX:
397 ctx = &private->req_trigger;
398 break;
399 default:
400 return -EINVAL;
401 }
402
403 switch (flags & VFIO_IRQ_SET_DATA_TYPE_MASK) {
404 case VFIO_IRQ_SET_DATA_NONE:
405 {
406 if (*ctx)
407 eventfd_signal(*ctx);
408 return 0;
409 }
410 case VFIO_IRQ_SET_DATA_BOOL:
411 {
412 uint8_t trigger;
413
414 if (get_user(trigger, (uint8_t __user *)data))
415 return -EFAULT;
416
417 if (trigger && *ctx)
418 eventfd_signal(*ctx);
419 return 0;
420 }
421 case VFIO_IRQ_SET_DATA_EVENTFD:
422 {
423 int32_t fd;
424
425 if (get_user(fd, (int32_t __user *)data))
426 return -EFAULT;
427
428 if (fd == -1) {
429 if (*ctx)
430 eventfd_ctx_put(*ctx);
431 *ctx = NULL;
432 } else if (fd >= 0) {
433 struct eventfd_ctx *efdctx;
434
435 efdctx = eventfd_ctx_fdget(fd);
436 if (IS_ERR(efdctx))
437 return PTR_ERR(efdctx);
438
439 if (*ctx)
440 eventfd_ctx_put(*ctx);
441
442 *ctx = efdctx;
443 } else
444 return -EINVAL;
445
446 return 0;
447 }
448 default:
449 return -EINVAL;
450 }
451 }
452
vfio_ccw_register_dev_region(struct vfio_ccw_private * private,unsigned int subtype,const struct vfio_ccw_regops * ops,size_t size,u32 flags,void * data)453 int vfio_ccw_register_dev_region(struct vfio_ccw_private *private,
454 unsigned int subtype,
455 const struct vfio_ccw_regops *ops,
456 size_t size, u32 flags, void *data)
457 {
458 struct vfio_ccw_region *region;
459
460 region = krealloc(private->region,
461 (private->num_regions + 1) * sizeof(*region),
462 GFP_KERNEL);
463 if (!region)
464 return -ENOMEM;
465
466 private->region = region;
467 private->region[private->num_regions].type = VFIO_REGION_TYPE_CCW;
468 private->region[private->num_regions].subtype = subtype;
469 private->region[private->num_regions].ops = ops;
470 private->region[private->num_regions].size = size;
471 private->region[private->num_regions].flags = flags;
472 private->region[private->num_regions].data = data;
473
474 private->num_regions++;
475
476 return 0;
477 }
478
vfio_ccw_unregister_dev_regions(struct vfio_ccw_private * private)479 void vfio_ccw_unregister_dev_regions(struct vfio_ccw_private *private)
480 {
481 int i;
482
483 for (i = 0; i < private->num_regions; i++)
484 private->region[i].ops->release(private, &private->region[i]);
485 private->num_regions = 0;
486 kfree(private->region);
487 private->region = NULL;
488 }
489
vfio_ccw_mdev_ioctl(struct vfio_device * vdev,unsigned int cmd,unsigned long arg)490 static ssize_t vfio_ccw_mdev_ioctl(struct vfio_device *vdev,
491 unsigned int cmd,
492 unsigned long arg)
493 {
494 struct vfio_ccw_private *private =
495 container_of(vdev, struct vfio_ccw_private, vdev);
496 int ret = 0;
497 unsigned long minsz;
498
499 switch (cmd) {
500 case VFIO_DEVICE_GET_INFO:
501 {
502 struct vfio_device_info info;
503
504 minsz = offsetofend(struct vfio_device_info, num_irqs);
505
506 if (copy_from_user(&info, (void __user *)arg, minsz))
507 return -EFAULT;
508
509 if (info.argsz < minsz)
510 return -EINVAL;
511
512 ret = vfio_ccw_mdev_get_device_info(private, &info);
513 if (ret)
514 return ret;
515
516 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
517 }
518 case VFIO_DEVICE_GET_IRQ_INFO:
519 {
520 struct vfio_irq_info info;
521
522 minsz = offsetofend(struct vfio_irq_info, count);
523
524 if (copy_from_user(&info, (void __user *)arg, minsz))
525 return -EFAULT;
526
527 if (info.argsz < minsz || info.index >= VFIO_CCW_NUM_IRQS)
528 return -EINVAL;
529
530 ret = vfio_ccw_mdev_get_irq_info(&info);
531 if (ret)
532 return ret;
533
534 if (info.count == -1)
535 return -EINVAL;
536
537 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
538 }
539 case VFIO_DEVICE_SET_IRQS:
540 {
541 struct vfio_irq_set hdr;
542 size_t data_size;
543 void __user *data;
544
545 minsz = offsetofend(struct vfio_irq_set, count);
546
547 if (copy_from_user(&hdr, (void __user *)arg, minsz))
548 return -EFAULT;
549
550 ret = vfio_set_irqs_validate_and_prepare(&hdr, 1,
551 VFIO_CCW_NUM_IRQS,
552 &data_size);
553 if (ret)
554 return ret;
555
556 data = (void __user *)(arg + minsz);
557 return vfio_ccw_mdev_set_irqs(private, hdr.flags, hdr.index,
558 data);
559 }
560 case VFIO_DEVICE_RESET:
561 return vfio_ccw_mdev_reset(private);
562 default:
563 return -ENOTTY;
564 }
565 }
566
567 /* Request removal of the device*/
vfio_ccw_mdev_request(struct vfio_device * vdev,unsigned int count)568 static void vfio_ccw_mdev_request(struct vfio_device *vdev, unsigned int count)
569 {
570 struct vfio_ccw_private *private =
571 container_of(vdev, struct vfio_ccw_private, vdev);
572 struct device *dev = vdev->dev;
573
574 if (private->req_trigger) {
575 if (!(count % 10))
576 dev_notice_ratelimited(dev,
577 "Relaying device request to user (#%u)\n",
578 count);
579
580 eventfd_signal(private->req_trigger);
581 } else if (count == 0) {
582 dev_notice(dev,
583 "No device request channel registered, blocked until released by user\n");
584 }
585 }
586
587 static const struct vfio_device_ops vfio_ccw_dev_ops = {
588 .init = vfio_ccw_mdev_init_dev,
589 .release = vfio_ccw_mdev_release_dev,
590 .open_device = vfio_ccw_mdev_open_device,
591 .close_device = vfio_ccw_mdev_close_device,
592 .read = vfio_ccw_mdev_read,
593 .write = vfio_ccw_mdev_write,
594 .ioctl = vfio_ccw_mdev_ioctl,
595 .get_region_info_caps = vfio_ccw_mdev_ioctl_get_region_info,
596 .request = vfio_ccw_mdev_request,
597 .dma_unmap = vfio_ccw_dma_unmap,
598 .bind_iommufd = vfio_iommufd_emulated_bind,
599 .unbind_iommufd = vfio_iommufd_emulated_unbind,
600 .attach_ioas = vfio_iommufd_emulated_attach_ioas,
601 .detach_ioas = vfio_iommufd_emulated_detach_ioas,
602 };
603
604 struct mdev_driver vfio_ccw_mdev_driver = {
605 .device_api = VFIO_DEVICE_API_CCW_STRING,
606 .max_instances = 1,
607 .driver = {
608 .name = "vfio_ccw_mdev",
609 .owner = THIS_MODULE,
610 .mod_name = KBUILD_MODNAME,
611 },
612 .probe = vfio_ccw_mdev_probe,
613 .remove = vfio_ccw_mdev_remove,
614 };
615