1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Physical device callbacks for vfio_ccw
4 *
5 * Copyright IBM Corp. 2017
6 * Copyright Red Hat, Inc. 2019
7 *
8 * Author(s): Dong Jia Shi <bjsdjshi@linux.vnet.ibm.com>
9 * Xiao Feng Ren <renxiaof@linux.vnet.ibm.com>
10 * Cornelia Huck <cohuck@redhat.com>
11 */
12
13 #include <linux/vfio.h>
14 #include <linux/nospec.h>
15 #include <linux/slab.h>
16
17 #include "vfio_ccw_private.h"
18
19 static const struct vfio_device_ops vfio_ccw_dev_ops;
20
vfio_ccw_mdev_reset(struct vfio_ccw_private * private)21 static int vfio_ccw_mdev_reset(struct vfio_ccw_private *private)
22 {
23 /*
24 * If the FSM state is seen as Not Operational after closing
25 * and re-opening the mdev, return an error.
26 */
27 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_CLOSE);
28 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_OPEN);
29 if (private->state == VFIO_CCW_STATE_NOT_OPER)
30 return -EINVAL;
31
32 return 0;
33 }
34
vfio_ccw_dma_unmap(struct vfio_device * vdev,u64 iova,u64 length)35 static void vfio_ccw_dma_unmap(struct vfio_device *vdev, u64 iova, u64 length)
36 {
37 struct vfio_ccw_private *private =
38 container_of(vdev, struct vfio_ccw_private, vdev);
39
40 /* Drivers MUST unpin pages in response to an invalidation. */
41 if (!cp_iova_pinned(&private->cp, iova, length))
42 return;
43
44 vfio_ccw_mdev_reset(private);
45 }
46
vfio_ccw_mdev_init_dev(struct vfio_device * vdev)47 static int vfio_ccw_mdev_init_dev(struct vfio_device *vdev)
48 {
49 struct vfio_ccw_private *private =
50 container_of(vdev, struct vfio_ccw_private, vdev);
51
52 mutex_init(&private->io_mutex);
53 private->state = VFIO_CCW_STATE_STANDBY;
54 INIT_LIST_HEAD(&private->crw);
55 INIT_WORK(&private->io_work, vfio_ccw_sch_io_todo);
56 INIT_WORK(&private->crw_work, vfio_ccw_crw_todo);
57
58 private->cp.guest_cp = kcalloc(CCWCHAIN_LEN_MAX, sizeof(struct ccw1),
59 GFP_KERNEL);
60 if (!private->cp.guest_cp)
61 goto out_free_private;
62
63 private->io_region = kmem_cache_zalloc(vfio_ccw_io_region,
64 GFP_KERNEL | GFP_DMA);
65 if (!private->io_region)
66 goto out_free_cp;
67
68 private->cmd_region = kmem_cache_zalloc(vfio_ccw_cmd_region,
69 GFP_KERNEL | GFP_DMA);
70 if (!private->cmd_region)
71 goto out_free_io;
72
73 private->schib_region = kmem_cache_zalloc(vfio_ccw_schib_region,
74 GFP_KERNEL | GFP_DMA);
75 if (!private->schib_region)
76 goto out_free_cmd;
77
78 private->crw_region = kmem_cache_zalloc(vfio_ccw_crw_region,
79 GFP_KERNEL | GFP_DMA);
80 if (!private->crw_region)
81 goto out_free_schib;
82
83 return 0;
84
85 out_free_schib:
86 kmem_cache_free(vfio_ccw_schib_region, private->schib_region);
87 out_free_cmd:
88 kmem_cache_free(vfio_ccw_cmd_region, private->cmd_region);
89 out_free_io:
90 kmem_cache_free(vfio_ccw_io_region, private->io_region);
91 out_free_cp:
92 kfree(private->cp.guest_cp);
93 out_free_private:
94 mutex_destroy(&private->io_mutex);
95 return -ENOMEM;
96 }
97
vfio_ccw_mdev_probe(struct mdev_device * mdev)98 static int vfio_ccw_mdev_probe(struct mdev_device *mdev)
99 {
100 struct subchannel *sch = to_subchannel(mdev->dev.parent);
101 struct vfio_ccw_parent *parent = dev_get_drvdata(&sch->dev);
102 struct vfio_ccw_private *private;
103 int ret;
104
105 private = vfio_alloc_device(vfio_ccw_private, vdev, &mdev->dev,
106 &vfio_ccw_dev_ops);
107 if (IS_ERR(private))
108 return PTR_ERR(private);
109
110 dev_set_drvdata(&parent->dev, private);
111
112 VFIO_CCW_MSG_EVENT(2, "sch %x.%x.%04x: create\n",
113 sch->schid.cssid,
114 sch->schid.ssid,
115 sch->schid.sch_no);
116
117 ret = vfio_register_emulated_iommu_dev(&private->vdev);
118 if (ret)
119 goto err_put_vdev;
120 dev_set_drvdata(&mdev->dev, private);
121 return 0;
122
123 err_put_vdev:
124 dev_set_drvdata(&parent->dev, NULL);
125 vfio_put_device(&private->vdev);
126 return ret;
127 }
128
vfio_ccw_mdev_release_dev(struct vfio_device * vdev)129 static void vfio_ccw_mdev_release_dev(struct vfio_device *vdev)
130 {
131 struct vfio_ccw_private *private =
132 container_of(vdev, struct vfio_ccw_private, vdev);
133 struct vfio_ccw_crw *crw, *temp;
134
135 list_for_each_entry_safe(crw, temp, &private->crw, next) {
136 list_del(&crw->next);
137 kfree(crw);
138 }
139
140 kmem_cache_free(vfio_ccw_crw_region, private->crw_region);
141 kmem_cache_free(vfio_ccw_schib_region, private->schib_region);
142 kmem_cache_free(vfio_ccw_cmd_region, private->cmd_region);
143 kmem_cache_free(vfio_ccw_io_region, private->io_region);
144 kfree(private->cp.guest_cp);
145 mutex_destroy(&private->io_mutex);
146 }
147
vfio_ccw_mdev_remove(struct mdev_device * mdev)148 static void vfio_ccw_mdev_remove(struct mdev_device *mdev)
149 {
150 struct subchannel *sch = to_subchannel(mdev->dev.parent);
151 struct vfio_ccw_parent *parent = dev_get_drvdata(&sch->dev);
152 struct vfio_ccw_private *private = dev_get_drvdata(&parent->dev);
153
154 VFIO_CCW_MSG_EVENT(2, "sch %x.%x.%04x: remove\n",
155 sch->schid.cssid,
156 sch->schid.ssid,
157 sch->schid.sch_no);
158
159 vfio_unregister_group_dev(&private->vdev);
160
161 dev_set_drvdata(&parent->dev, NULL);
162 vfio_put_device(&private->vdev);
163 }
164
vfio_ccw_mdev_open_device(struct vfio_device * vdev)165 static int vfio_ccw_mdev_open_device(struct vfio_device *vdev)
166 {
167 struct vfio_ccw_private *private =
168 container_of(vdev, struct vfio_ccw_private, vdev);
169 int ret;
170
171 /* Device cannot simply be opened again from this state */
172 if (private->state == VFIO_CCW_STATE_NOT_OPER)
173 return -EINVAL;
174
175 ret = vfio_ccw_register_async_dev_regions(private);
176 if (ret)
177 return ret;
178
179 ret = vfio_ccw_register_schib_dev_regions(private);
180 if (ret)
181 goto out_unregister;
182
183 ret = vfio_ccw_register_crw_dev_regions(private);
184 if (ret)
185 goto out_unregister;
186
187 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_OPEN);
188 if (private->state == VFIO_CCW_STATE_NOT_OPER) {
189 ret = -EINVAL;
190 goto out_unregister;
191 }
192
193 return ret;
194
195 out_unregister:
196 vfio_ccw_unregister_dev_regions(private);
197 return ret;
198 }
199
vfio_ccw_mdev_close_device(struct vfio_device * vdev)200 static void vfio_ccw_mdev_close_device(struct vfio_device *vdev)
201 {
202 struct vfio_ccw_private *private =
203 container_of(vdev, struct vfio_ccw_private, vdev);
204
205 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_CLOSE);
206 vfio_ccw_unregister_dev_regions(private);
207 }
208
vfio_ccw_mdev_read_io_region(struct vfio_ccw_private * private,char __user * buf,size_t count,loff_t * ppos)209 static ssize_t vfio_ccw_mdev_read_io_region(struct vfio_ccw_private *private,
210 char __user *buf, size_t count,
211 loff_t *ppos)
212 {
213 loff_t pos = *ppos & VFIO_CCW_OFFSET_MASK;
214 struct ccw_io_region *region;
215 int ret;
216
217 if (pos + count > sizeof(*region))
218 return -EINVAL;
219
220 mutex_lock(&private->io_mutex);
221 region = private->io_region;
222 if (copy_to_user(buf, (void *)region + pos, count))
223 ret = -EFAULT;
224 else
225 ret = count;
226 mutex_unlock(&private->io_mutex);
227 return ret;
228 }
229
vfio_ccw_mdev_read(struct vfio_device * vdev,char __user * buf,size_t count,loff_t * ppos)230 static ssize_t vfio_ccw_mdev_read(struct vfio_device *vdev,
231 char __user *buf,
232 size_t count,
233 loff_t *ppos)
234 {
235 struct vfio_ccw_private *private =
236 container_of(vdev, struct vfio_ccw_private, vdev);
237 unsigned int index = VFIO_CCW_OFFSET_TO_INDEX(*ppos);
238
239 if (index >= VFIO_CCW_NUM_REGIONS + private->num_regions)
240 return -EINVAL;
241
242 switch (index) {
243 case VFIO_CCW_CONFIG_REGION_INDEX:
244 return vfio_ccw_mdev_read_io_region(private, buf, count, ppos);
245 default:
246 index -= VFIO_CCW_NUM_REGIONS;
247 return private->region[index].ops->read(private, buf, count,
248 ppos);
249 }
250
251 return -EINVAL;
252 }
253
vfio_ccw_mdev_write_io_region(struct vfio_ccw_private * private,const char __user * buf,size_t count,loff_t * ppos)254 static ssize_t vfio_ccw_mdev_write_io_region(struct vfio_ccw_private *private,
255 const char __user *buf,
256 size_t count, loff_t *ppos)
257 {
258 loff_t pos = *ppos & VFIO_CCW_OFFSET_MASK;
259 struct ccw_io_region *region;
260 int ret;
261
262 if (pos + count > sizeof(*region))
263 return -EINVAL;
264
265 if (!mutex_trylock(&private->io_mutex))
266 return -EAGAIN;
267
268 region = private->io_region;
269 if (copy_from_user((void *)region + pos, buf, count)) {
270 ret = -EFAULT;
271 goto out_unlock;
272 }
273
274 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_IO_REQ);
275 ret = (region->ret_code != 0) ? region->ret_code : count;
276
277 out_unlock:
278 mutex_unlock(&private->io_mutex);
279 return ret;
280 }
281
vfio_ccw_mdev_write(struct vfio_device * vdev,const char __user * buf,size_t count,loff_t * ppos)282 static ssize_t vfio_ccw_mdev_write(struct vfio_device *vdev,
283 const char __user *buf,
284 size_t count,
285 loff_t *ppos)
286 {
287 struct vfio_ccw_private *private =
288 container_of(vdev, struct vfio_ccw_private, vdev);
289 unsigned int index = VFIO_CCW_OFFSET_TO_INDEX(*ppos);
290
291 if (index >= VFIO_CCW_NUM_REGIONS + private->num_regions)
292 return -EINVAL;
293
294 switch (index) {
295 case VFIO_CCW_CONFIG_REGION_INDEX:
296 return vfio_ccw_mdev_write_io_region(private, buf, count, ppos);
297 default:
298 index -= VFIO_CCW_NUM_REGIONS;
299 return private->region[index].ops->write(private, buf, count,
300 ppos);
301 }
302
303 return -EINVAL;
304 }
305
vfio_ccw_mdev_get_device_info(struct vfio_ccw_private * private,struct vfio_device_info * info)306 static int vfio_ccw_mdev_get_device_info(struct vfio_ccw_private *private,
307 struct vfio_device_info *info)
308 {
309 info->flags = VFIO_DEVICE_FLAGS_CCW | VFIO_DEVICE_FLAGS_RESET;
310 info->num_regions = VFIO_CCW_NUM_REGIONS + private->num_regions;
311 info->num_irqs = VFIO_CCW_NUM_IRQS;
312
313 return 0;
314 }
315
vfio_ccw_mdev_ioctl_get_region_info(struct vfio_device * vdev,struct vfio_region_info * info,struct vfio_info_cap * caps)316 static int vfio_ccw_mdev_ioctl_get_region_info(struct vfio_device *vdev,
317 struct vfio_region_info *info,
318 struct vfio_info_cap *caps)
319 {
320 struct vfio_ccw_private *private =
321 container_of(vdev, struct vfio_ccw_private, vdev);
322 int i;
323
324 switch (info->index) {
325 case VFIO_CCW_CONFIG_REGION_INDEX:
326 info->offset = 0;
327 info->size = sizeof(struct ccw_io_region);
328 info->flags = VFIO_REGION_INFO_FLAG_READ
329 | VFIO_REGION_INFO_FLAG_WRITE;
330 return 0;
331 default: /* all other regions are handled via capability chain */
332 {
333 struct vfio_region_info_cap_type cap_type = {
334 .header.id = VFIO_REGION_INFO_CAP_TYPE,
335 .header.version = 1 };
336 int ret;
337
338 if (info->index >=
339 VFIO_CCW_NUM_REGIONS + private->num_regions)
340 return -EINVAL;
341
342 info->index = array_index_nospec(info->index,
343 VFIO_CCW_NUM_REGIONS +
344 private->num_regions);
345
346 i = info->index - VFIO_CCW_NUM_REGIONS;
347
348 info->offset = VFIO_CCW_INDEX_TO_OFFSET(info->index);
349 info->size = private->region[i].size;
350 info->flags = private->region[i].flags;
351
352 cap_type.type = private->region[i].type;
353 cap_type.subtype = private->region[i].subtype;
354
355 ret = vfio_info_add_capability(caps, &cap_type.header,
356 sizeof(cap_type));
357 if (ret)
358 return ret;
359 }
360 }
361 return 0;
362 }
363
vfio_ccw_mdev_get_irq_info(struct vfio_irq_info * info)364 static int vfio_ccw_mdev_get_irq_info(struct vfio_irq_info *info)
365 {
366 switch (info->index) {
367 case VFIO_CCW_IO_IRQ_INDEX:
368 case VFIO_CCW_CRW_IRQ_INDEX:
369 case VFIO_CCW_REQ_IRQ_INDEX:
370 info->count = 1;
371 info->flags = VFIO_IRQ_INFO_EVENTFD;
372 break;
373 default:
374 return -EINVAL;
375 }
376
377 return 0;
378 }
379
vfio_ccw_mdev_set_irqs(struct vfio_ccw_private * private,uint32_t flags,uint32_t index,void __user * data)380 static int vfio_ccw_mdev_set_irqs(struct vfio_ccw_private *private,
381 uint32_t flags,
382 uint32_t index,
383 void __user *data)
384 {
385 struct eventfd_ctx **ctx;
386
387 if (!(flags & VFIO_IRQ_SET_ACTION_TRIGGER))
388 return -EINVAL;
389
390 switch (index) {
391 case VFIO_CCW_IO_IRQ_INDEX:
392 ctx = &private->io_trigger;
393 break;
394 case VFIO_CCW_CRW_IRQ_INDEX:
395 ctx = &private->crw_trigger;
396 break;
397 case VFIO_CCW_REQ_IRQ_INDEX:
398 ctx = &private->req_trigger;
399 break;
400 default:
401 return -EINVAL;
402 }
403
404 switch (flags & VFIO_IRQ_SET_DATA_TYPE_MASK) {
405 case VFIO_IRQ_SET_DATA_NONE:
406 {
407 if (*ctx)
408 eventfd_signal(*ctx);
409 return 0;
410 }
411 case VFIO_IRQ_SET_DATA_BOOL:
412 {
413 uint8_t trigger;
414
415 if (get_user(trigger, (uint8_t __user *)data))
416 return -EFAULT;
417
418 if (trigger && *ctx)
419 eventfd_signal(*ctx);
420 return 0;
421 }
422 case VFIO_IRQ_SET_DATA_EVENTFD:
423 {
424 int32_t fd;
425
426 if (get_user(fd, (int32_t __user *)data))
427 return -EFAULT;
428
429 if (fd == -1) {
430 if (*ctx)
431 eventfd_ctx_put(*ctx);
432 *ctx = NULL;
433 } else if (fd >= 0) {
434 struct eventfd_ctx *efdctx;
435
436 efdctx = eventfd_ctx_fdget(fd);
437 if (IS_ERR(efdctx))
438 return PTR_ERR(efdctx);
439
440 if (*ctx)
441 eventfd_ctx_put(*ctx);
442
443 *ctx = efdctx;
444 } else
445 return -EINVAL;
446
447 return 0;
448 }
449 default:
450 return -EINVAL;
451 }
452 }
453
vfio_ccw_register_dev_region(struct vfio_ccw_private * private,unsigned int subtype,const struct vfio_ccw_regops * ops,size_t size,u32 flags,void * data)454 int vfio_ccw_register_dev_region(struct vfio_ccw_private *private,
455 unsigned int subtype,
456 const struct vfio_ccw_regops *ops,
457 size_t size, u32 flags, void *data)
458 {
459 struct vfio_ccw_region *region;
460
461 region = krealloc(private->region,
462 (private->num_regions + 1) * sizeof(*region),
463 GFP_KERNEL);
464 if (!region)
465 return -ENOMEM;
466
467 private->region = region;
468 private->region[private->num_regions].type = VFIO_REGION_TYPE_CCW;
469 private->region[private->num_regions].subtype = subtype;
470 private->region[private->num_regions].ops = ops;
471 private->region[private->num_regions].size = size;
472 private->region[private->num_regions].flags = flags;
473 private->region[private->num_regions].data = data;
474
475 private->num_regions++;
476
477 return 0;
478 }
479
vfio_ccw_unregister_dev_regions(struct vfio_ccw_private * private)480 void vfio_ccw_unregister_dev_regions(struct vfio_ccw_private *private)
481 {
482 int i;
483
484 for (i = 0; i < private->num_regions; i++)
485 private->region[i].ops->release(private, &private->region[i]);
486 private->num_regions = 0;
487 kfree(private->region);
488 private->region = NULL;
489 }
490
vfio_ccw_mdev_ioctl(struct vfio_device * vdev,unsigned int cmd,unsigned long arg)491 static ssize_t vfio_ccw_mdev_ioctl(struct vfio_device *vdev,
492 unsigned int cmd,
493 unsigned long arg)
494 {
495 struct vfio_ccw_private *private =
496 container_of(vdev, struct vfio_ccw_private, vdev);
497 int ret = 0;
498 unsigned long minsz;
499
500 switch (cmd) {
501 case VFIO_DEVICE_GET_INFO:
502 {
503 struct vfio_device_info info;
504
505 minsz = offsetofend(struct vfio_device_info, num_irqs);
506
507 if (copy_from_user(&info, (void __user *)arg, minsz))
508 return -EFAULT;
509
510 if (info.argsz < minsz)
511 return -EINVAL;
512
513 ret = vfio_ccw_mdev_get_device_info(private, &info);
514 if (ret)
515 return ret;
516
517 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
518 }
519 case VFIO_DEVICE_GET_IRQ_INFO:
520 {
521 struct vfio_irq_info info;
522
523 minsz = offsetofend(struct vfio_irq_info, count);
524
525 if (copy_from_user(&info, (void __user *)arg, minsz))
526 return -EFAULT;
527
528 if (info.argsz < minsz || info.index >= VFIO_CCW_NUM_IRQS)
529 return -EINVAL;
530
531 ret = vfio_ccw_mdev_get_irq_info(&info);
532 if (ret)
533 return ret;
534
535 if (info.count == -1)
536 return -EINVAL;
537
538 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
539 }
540 case VFIO_DEVICE_SET_IRQS:
541 {
542 struct vfio_irq_set hdr;
543 size_t data_size;
544 void __user *data;
545
546 minsz = offsetofend(struct vfio_irq_set, count);
547
548 if (copy_from_user(&hdr, (void __user *)arg, minsz))
549 return -EFAULT;
550
551 ret = vfio_set_irqs_validate_and_prepare(&hdr, 1,
552 VFIO_CCW_NUM_IRQS,
553 &data_size);
554 if (ret)
555 return ret;
556
557 data = (void __user *)(arg + minsz);
558 return vfio_ccw_mdev_set_irqs(private, hdr.flags, hdr.index,
559 data);
560 }
561 case VFIO_DEVICE_RESET:
562 return vfio_ccw_mdev_reset(private);
563 default:
564 return -ENOTTY;
565 }
566 }
567
568 /* Request removal of the device*/
vfio_ccw_mdev_request(struct vfio_device * vdev,unsigned int count)569 static void vfio_ccw_mdev_request(struct vfio_device *vdev, unsigned int count)
570 {
571 struct vfio_ccw_private *private =
572 container_of(vdev, struct vfio_ccw_private, vdev);
573 struct device *dev = vdev->dev;
574
575 if (private->req_trigger) {
576 if (!(count % 10))
577 dev_notice_ratelimited(dev,
578 "Relaying device request to user (#%u)\n",
579 count);
580
581 eventfd_signal(private->req_trigger);
582 } else if (count == 0) {
583 dev_notice(dev,
584 "No device request channel registered, blocked until released by user\n");
585 }
586 }
587
588 static const struct vfio_device_ops vfio_ccw_dev_ops = {
589 .init = vfio_ccw_mdev_init_dev,
590 .release = vfio_ccw_mdev_release_dev,
591 .open_device = vfio_ccw_mdev_open_device,
592 .close_device = vfio_ccw_mdev_close_device,
593 .read = vfio_ccw_mdev_read,
594 .write = vfio_ccw_mdev_write,
595 .ioctl = vfio_ccw_mdev_ioctl,
596 .get_region_info_caps = vfio_ccw_mdev_ioctl_get_region_info,
597 .request = vfio_ccw_mdev_request,
598 .dma_unmap = vfio_ccw_dma_unmap,
599 .bind_iommufd = vfio_iommufd_emulated_bind,
600 .unbind_iommufd = vfio_iommufd_emulated_unbind,
601 .attach_ioas = vfio_iommufd_emulated_attach_ioas,
602 .detach_ioas = vfio_iommufd_emulated_detach_ioas,
603 };
604
605 struct mdev_driver vfio_ccw_mdev_driver = {
606 .device_api = VFIO_DEVICE_API_CCW_STRING,
607 .max_instances = 1,
608 .driver = {
609 .name = "vfio_ccw_mdev",
610 .owner = THIS_MODULE,
611 .mod_name = KBUILD_MODNAME,
612 },
613 .probe = vfio_ccw_mdev_probe,
614 .remove = vfio_ccw_mdev_remove,
615 };
616