1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved
4 */
5
6 #include <linux/device.h>
7 #include <linux/module.h>
8 #include <linux/mutex.h>
9 #include <linux/pci.h>
10 #include <linux/pm_runtime.h>
11 #include <linux/types.h>
12 #include <linux/uaccess.h>
13 #include <linux/vfio.h>
14 #include <linux/vfio_pci_core.h>
15 #include <linux/virtio_pci.h>
16 #include <linux/virtio_net.h>
17 #include <linux/virtio_pci_admin.h>
18
19 #include "common.h"
20
virtiovf_pci_open_device(struct vfio_device * core_vdev)21 static int virtiovf_pci_open_device(struct vfio_device *core_vdev)
22 {
23 struct virtiovf_pci_core_device *virtvdev = container_of(core_vdev,
24 struct virtiovf_pci_core_device, core_device.vdev);
25 struct vfio_pci_core_device *vdev = &virtvdev->core_device;
26 int ret;
27
28 ret = vfio_pci_core_enable(vdev);
29 if (ret)
30 return ret;
31
32 #ifdef CONFIG_VIRTIO_VFIO_PCI_ADMIN_LEGACY
33 ret = virtiovf_open_legacy_io(virtvdev);
34 if (ret) {
35 vfio_pci_core_disable(vdev);
36 return ret;
37 }
38 #endif
39
40 virtiovf_open_migration(virtvdev);
41 vfio_pci_core_finish_enable(vdev);
42 return 0;
43 }
44
virtiovf_pci_close_device(struct vfio_device * core_vdev)45 static void virtiovf_pci_close_device(struct vfio_device *core_vdev)
46 {
47 struct virtiovf_pci_core_device *virtvdev = container_of(core_vdev,
48 struct virtiovf_pci_core_device, core_device.vdev);
49
50 virtiovf_close_migration(virtvdev);
51 vfio_pci_core_close_device(core_vdev);
52 }
53
54 #ifdef CONFIG_VIRTIO_VFIO_PCI_ADMIN_LEGACY
virtiovf_pci_init_device(struct vfio_device * core_vdev)55 static int virtiovf_pci_init_device(struct vfio_device *core_vdev)
56 {
57 struct virtiovf_pci_core_device *virtvdev = container_of(core_vdev,
58 struct virtiovf_pci_core_device, core_device.vdev);
59 int ret;
60
61 ret = vfio_pci_core_init_dev(core_vdev);
62 if (ret)
63 return ret;
64
65 /*
66 * The vfio_device_ops.init() callback is set to virtiovf_pci_init_device()
67 * only when legacy I/O is supported. Now, let's initialize it.
68 */
69 return virtiovf_init_legacy_io(virtvdev);
70 }
71 #endif
72
virtiovf_pci_core_release_dev(struct vfio_device * core_vdev)73 static void virtiovf_pci_core_release_dev(struct vfio_device *core_vdev)
74 {
75 #ifdef CONFIG_VIRTIO_VFIO_PCI_ADMIN_LEGACY
76 struct virtiovf_pci_core_device *virtvdev = container_of(core_vdev,
77 struct virtiovf_pci_core_device, core_device.vdev);
78
79 virtiovf_release_legacy_io(virtvdev);
80 #endif
81 vfio_pci_core_release_dev(core_vdev);
82 }
83
84 static const struct vfio_device_ops virtiovf_vfio_pci_lm_ops = {
85 .name = "virtio-vfio-pci-lm",
86 .init = vfio_pci_core_init_dev,
87 .release = virtiovf_pci_core_release_dev,
88 .open_device = virtiovf_pci_open_device,
89 .close_device = virtiovf_pci_close_device,
90 .ioctl = vfio_pci_core_ioctl,
91 .get_region_info_caps = vfio_pci_ioctl_get_region_info,
92 .device_feature = vfio_pci_core_ioctl_feature,
93 .read = vfio_pci_core_read,
94 .write = vfio_pci_core_write,
95 .mmap = vfio_pci_core_mmap,
96 .request = vfio_pci_core_request,
97 .match = vfio_pci_core_match,
98 .match_token_uuid = vfio_pci_core_match_token_uuid,
99 .bind_iommufd = vfio_iommufd_physical_bind,
100 .unbind_iommufd = vfio_iommufd_physical_unbind,
101 .attach_ioas = vfio_iommufd_physical_attach_ioas,
102 .detach_ioas = vfio_iommufd_physical_detach_ioas,
103 };
104
105 #ifdef CONFIG_VIRTIO_VFIO_PCI_ADMIN_LEGACY
106 static const struct vfio_device_ops virtiovf_vfio_pci_tran_lm_ops = {
107 .name = "virtio-vfio-pci-trans-lm",
108 .init = virtiovf_pci_init_device,
109 .release = virtiovf_pci_core_release_dev,
110 .open_device = virtiovf_pci_open_device,
111 .close_device = virtiovf_pci_close_device,
112 .ioctl = vfio_pci_core_ioctl,
113 .get_region_info_caps = virtiovf_pci_ioctl_get_region_info,
114 .device_feature = vfio_pci_core_ioctl_feature,
115 .read = virtiovf_pci_core_read,
116 .write = virtiovf_pci_core_write,
117 .mmap = vfio_pci_core_mmap,
118 .request = vfio_pci_core_request,
119 .match = vfio_pci_core_match,
120 .match_token_uuid = vfio_pci_core_match_token_uuid,
121 .bind_iommufd = vfio_iommufd_physical_bind,
122 .unbind_iommufd = vfio_iommufd_physical_unbind,
123 .attach_ioas = vfio_iommufd_physical_attach_ioas,
124 .detach_ioas = vfio_iommufd_physical_detach_ioas,
125 };
126 #endif
127
128 static const struct vfio_device_ops virtiovf_vfio_pci_ops = {
129 .name = "virtio-vfio-pci",
130 .init = vfio_pci_core_init_dev,
131 .release = vfio_pci_core_release_dev,
132 .open_device = virtiovf_pci_open_device,
133 .close_device = vfio_pci_core_close_device,
134 .ioctl = vfio_pci_core_ioctl,
135 .get_region_info_caps = vfio_pci_ioctl_get_region_info,
136 .device_feature = vfio_pci_core_ioctl_feature,
137 .read = vfio_pci_core_read,
138 .write = vfio_pci_core_write,
139 .mmap = vfio_pci_core_mmap,
140 .request = vfio_pci_core_request,
141 .match = vfio_pci_core_match,
142 .match_token_uuid = vfio_pci_core_match_token_uuid,
143 .bind_iommufd = vfio_iommufd_physical_bind,
144 .unbind_iommufd = vfio_iommufd_physical_unbind,
145 .attach_ioas = vfio_iommufd_physical_attach_ioas,
146 .detach_ioas = vfio_iommufd_physical_detach_ioas,
147 };
148
virtiovf_pci_probe(struct pci_dev * pdev,const struct pci_device_id * id)149 static int virtiovf_pci_probe(struct pci_dev *pdev,
150 const struct pci_device_id *id)
151 {
152 const struct vfio_device_ops *ops = &virtiovf_vfio_pci_ops;
153 struct virtiovf_pci_core_device *virtvdev;
154 bool sup_legacy_io = false;
155 bool sup_lm = false;
156 int ret;
157
158 if (pdev->is_virtfn) {
159 #ifdef CONFIG_VIRTIO_VFIO_PCI_ADMIN_LEGACY
160 sup_legacy_io = virtiovf_support_legacy_io(pdev);
161 if (sup_legacy_io)
162 ops = &virtiovf_vfio_pci_tran_lm_ops;
163 #endif
164 sup_lm = virtio_pci_admin_has_dev_parts(pdev);
165 if (sup_lm && !sup_legacy_io)
166 ops = &virtiovf_vfio_pci_lm_ops;
167 }
168
169 virtvdev = vfio_alloc_device(virtiovf_pci_core_device, core_device.vdev,
170 &pdev->dev, ops);
171 if (IS_ERR(virtvdev))
172 return PTR_ERR(virtvdev);
173
174 if (sup_lm)
175 virtiovf_set_migratable(virtvdev);
176
177 dev_set_drvdata(&pdev->dev, &virtvdev->core_device);
178 ret = vfio_pci_core_register_device(&virtvdev->core_device);
179 if (ret)
180 goto out;
181 return 0;
182 out:
183 vfio_put_device(&virtvdev->core_device.vdev);
184 return ret;
185 }
186
virtiovf_pci_remove(struct pci_dev * pdev)187 static void virtiovf_pci_remove(struct pci_dev *pdev)
188 {
189 struct virtiovf_pci_core_device *virtvdev = dev_get_drvdata(&pdev->dev);
190
191 vfio_pci_core_unregister_device(&virtvdev->core_device);
192 vfio_put_device(&virtvdev->core_device.vdev);
193 }
194
195 static const struct pci_device_id virtiovf_pci_table[] = {
196 /* Only virtio-net and virtio-block are supported/tested so far */
197 { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_REDHAT_QUMRANET, 0x1041) },
198 { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_REDHAT_QUMRANET, 0x1042) },
199 {}
200 };
201
202 MODULE_DEVICE_TABLE(pci, virtiovf_pci_table);
203
virtiovf_pci_aer_reset_done(struct pci_dev * pdev)204 static void virtiovf_pci_aer_reset_done(struct pci_dev *pdev)
205 {
206 #ifdef CONFIG_VIRTIO_VFIO_PCI_ADMIN_LEGACY
207 virtiovf_legacy_io_reset_done(pdev);
208 #endif
209 virtiovf_migration_reset_done(pdev);
210 }
211
212 static const struct pci_error_handlers virtiovf_err_handlers = {
213 .reset_done = virtiovf_pci_aer_reset_done,
214 .error_detected = vfio_pci_core_aer_err_detected,
215 };
216
217 static struct pci_driver virtiovf_pci_driver = {
218 .name = KBUILD_MODNAME,
219 .id_table = virtiovf_pci_table,
220 .probe = virtiovf_pci_probe,
221 .remove = virtiovf_pci_remove,
222 .err_handler = &virtiovf_err_handlers,
223 .driver_managed_dma = true,
224 };
225
226 module_pci_driver(virtiovf_pci_driver);
227
228 MODULE_LICENSE("GPL");
229 MODULE_AUTHOR("Yishai Hadas <yishaih@nvidia.com>");
230 MODULE_DESCRIPTION(
231 "VIRTIO VFIO PCI - User Level meta-driver for VIRTIO NET and BLOCK devices");
232