1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2011 NetApp, Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 #include <sys/param.h>
30 #include <sys/bus.h>
31 #include <sys/eventhandler.h>
32 #include <sys/sysctl.h>
33 #include <sys/systm.h>
34
35 #include <dev/pci/pcivar.h>
36 #include <dev/pci/pcireg.h>
37
38 #include <machine/cpu.h>
39 #include <machine/md_var.h>
40
41 #include "vmm_util.h"
42 #include "vmm_mem.h"
43 #include "iommu.h"
44
45 SYSCTL_DECL(_hw_vmm);
46 SYSCTL_NODE(_hw_vmm, OID_AUTO, iommu, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
47 "bhyve iommu parameters");
48
49 static int iommu_avail;
50 SYSCTL_INT(_hw_vmm_iommu, OID_AUTO, initialized, CTLFLAG_RD, &iommu_avail,
51 0, "bhyve iommu initialized?");
52
53 static int iommu_enable = 1;
54 SYSCTL_INT(_hw_vmm_iommu, OID_AUTO, enable, CTLFLAG_RDTUN, &iommu_enable, 0,
55 "Enable use of I/O MMU (required for PCI passthrough).");
56
57 static const struct iommu_ops *ops;
58 static void *host_domain;
59 static eventhandler_tag add_tag, delete_tag;
60
61 static void iommu_cleanup_int(bool iommu_disable);
62
63 static __inline int
IOMMU_INIT(void)64 IOMMU_INIT(void)
65 {
66 if (ops != NULL)
67 return ((*ops->init)());
68 else
69 return (ENXIO);
70 }
71
72 static __inline void
IOMMU_CLEANUP(void)73 IOMMU_CLEANUP(void)
74 {
75 if (ops != NULL && iommu_avail)
76 (*ops->cleanup)();
77 }
78
79 static __inline void *
IOMMU_CREATE_DOMAIN(vm_paddr_t maxaddr)80 IOMMU_CREATE_DOMAIN(vm_paddr_t maxaddr)
81 {
82
83 if (ops != NULL && iommu_avail)
84 return ((*ops->create_domain)(maxaddr));
85 else
86 return (NULL);
87 }
88
89 static __inline void
IOMMU_DESTROY_DOMAIN(void * dom)90 IOMMU_DESTROY_DOMAIN(void *dom)
91 {
92
93 if (ops != NULL && iommu_avail)
94 (*ops->destroy_domain)(dom);
95 }
96
97 static __inline int
IOMMU_CREATE_MAPPING(void * domain,vm_paddr_t gpa,vm_paddr_t hpa,uint64_t len,uint64_t * res_len)98 IOMMU_CREATE_MAPPING(void *domain, vm_paddr_t gpa, vm_paddr_t hpa,
99 uint64_t len, uint64_t *res_len)
100 {
101
102 if (ops != NULL && iommu_avail)
103 return ((*ops->create_mapping)(domain, gpa, hpa, len, res_len));
104 return (EOPNOTSUPP);
105 }
106
107 static __inline uint64_t
IOMMU_REMOVE_MAPPING(void * domain,vm_paddr_t gpa,uint64_t len,uint64_t * res_len)108 IOMMU_REMOVE_MAPPING(void *domain, vm_paddr_t gpa, uint64_t len,
109 uint64_t *res_len)
110 {
111
112 if (ops != NULL && iommu_avail)
113 return ((*ops->remove_mapping)(domain, gpa, len, res_len));
114 return (EOPNOTSUPP);
115 }
116
117 static __inline int
IOMMU_ADD_DEVICE(void * domain,device_t dev,uint16_t rid)118 IOMMU_ADD_DEVICE(void *domain, device_t dev, uint16_t rid)
119 {
120
121 if (ops != NULL && iommu_avail)
122 return ((*ops->add_device)(domain, dev, rid));
123 return (EOPNOTSUPP);
124 }
125
126 static __inline int
IOMMU_REMOVE_DEVICE(void * domain,device_t dev,uint16_t rid)127 IOMMU_REMOVE_DEVICE(void *domain, device_t dev, uint16_t rid)
128 {
129
130 if (ops != NULL && iommu_avail)
131 return ((*ops->remove_device)(domain, dev, rid));
132 return (0); /* To allow ppt_attach() to succeed. */
133 }
134
135 static __inline int
IOMMU_INVALIDATE_TLB(void * domain)136 IOMMU_INVALIDATE_TLB(void *domain)
137 {
138
139 if (ops != NULL && iommu_avail)
140 return ((*ops->invalidate_tlb)(domain));
141 return (0);
142 }
143
144 static __inline void
IOMMU_ENABLE(void)145 IOMMU_ENABLE(void)
146 {
147
148 if (ops != NULL && iommu_avail)
149 (*ops->enable)();
150 }
151
152 static __inline void
IOMMU_DISABLE(void)153 IOMMU_DISABLE(void)
154 {
155
156 if (ops != NULL && iommu_avail)
157 (*ops->disable)();
158 }
159
160 static void
iommu_pci_add(void * arg,device_t dev)161 iommu_pci_add(void *arg, device_t dev)
162 {
163
164 /* Add new devices to the host domain. */
165 iommu_add_device(host_domain, dev, pci_get_rid(dev));
166 }
167
168 static void
iommu_pci_delete(void * arg,device_t dev)169 iommu_pci_delete(void *arg, device_t dev)
170 {
171
172 iommu_remove_device(host_domain, dev, pci_get_rid(dev));
173 }
174
175 static void
iommu_init(void)176 iommu_init(void)
177 {
178 int error, bus, slot, func;
179 vm_paddr_t maxaddr;
180 devclass_t dc;
181 device_t dev;
182
183 if (!iommu_enable)
184 return;
185
186 if (vmm_is_intel())
187 ops = &iommu_ops_intel;
188 else if (vmm_is_svm())
189 ops = &iommu_ops_amd;
190 else
191 ops = NULL;
192
193 error = IOMMU_INIT();
194 if (error)
195 return;
196
197 iommu_avail = 1;
198
199 /*
200 * Create a domain for the devices owned by the host
201 */
202 maxaddr = vmm_mem_maxaddr();
203 host_domain = IOMMU_CREATE_DOMAIN(maxaddr);
204 if (host_domain == NULL) {
205 printf("iommu_init: unable to create a host domain");
206 IOMMU_CLEANUP();
207 ops = NULL;
208 iommu_avail = 0;
209 return;
210 }
211
212 /*
213 * Create 1:1 mappings from '0' to 'maxaddr' for devices assigned to
214 * the host
215 */
216 iommu_create_mapping(host_domain, 0, 0, maxaddr);
217
218 add_tag = EVENTHANDLER_REGISTER(pci_add_device, iommu_pci_add, NULL, 0);
219 delete_tag = EVENTHANDLER_REGISTER(pci_delete_device, iommu_pci_delete,
220 NULL, 0);
221 dc = devclass_find("ppt");
222 for (bus = 0; bus <= PCI_BUSMAX; bus++) {
223 for (slot = 0; slot <= PCI_SLOTMAX; slot++) {
224 for (func = 0; func <= PCI_FUNCMAX; func++) {
225 dev = pci_find_dbsf(0, bus, slot, func);
226 if (dev == NULL)
227 continue;
228
229 /* Skip passthrough devices. */
230 if (dc != NULL &&
231 device_get_devclass(dev) == dc)
232 continue;
233
234 /*
235 * Everything else belongs to the host
236 * domain.
237 */
238 error = iommu_add_device(host_domain, dev,
239 pci_get_rid(dev));
240 if (error != 0) {
241 iommu_cleanup_int(false);
242 return;
243 }
244 }
245 }
246 }
247 IOMMU_ENABLE();
248 }
249
250 static void
iommu_cleanup_int(bool iommu_disable)251 iommu_cleanup_int(bool iommu_disable)
252 {
253
254 if (add_tag != NULL) {
255 EVENTHANDLER_DEREGISTER(pci_add_device, add_tag);
256 add_tag = NULL;
257 }
258 if (delete_tag != NULL) {
259 EVENTHANDLER_DEREGISTER(pci_delete_device, delete_tag);
260 delete_tag = NULL;
261 }
262 if (iommu_disable)
263 IOMMU_DISABLE();
264 IOMMU_DESTROY_DOMAIN(host_domain);
265 host_domain = NULL;
266 IOMMU_CLEANUP();
267 }
268
269 void
iommu_cleanup(void)270 iommu_cleanup(void)
271 {
272 iommu_cleanup_int(true);
273 }
274
275 void *
iommu_create_domain(vm_paddr_t maxaddr)276 iommu_create_domain(vm_paddr_t maxaddr)
277 {
278 static volatile int iommu_initted;
279
280 if (iommu_initted < 2) {
281 if (atomic_cmpset_int(&iommu_initted, 0, 1)) {
282 iommu_init();
283 atomic_store_rel_int(&iommu_initted, 2);
284 } else
285 while (iommu_initted == 1)
286 cpu_spinwait();
287 }
288 return (IOMMU_CREATE_DOMAIN(maxaddr));
289 }
290
291 void
iommu_destroy_domain(void * dom)292 iommu_destroy_domain(void *dom)
293 {
294
295 IOMMU_DESTROY_DOMAIN(dom);
296 }
297
298 int
iommu_create_mapping(void * dom,vm_paddr_t gpa,vm_paddr_t hpa,size_t len)299 iommu_create_mapping(void *dom, vm_paddr_t gpa, vm_paddr_t hpa, size_t len)
300 {
301 uint64_t mapped, remaining;
302 int error;
303
304 for (remaining = len; remaining > 0; gpa += mapped, hpa += mapped,
305 remaining -= mapped) {
306 error = IOMMU_CREATE_MAPPING(dom, gpa, hpa, remaining,
307 &mapped);
308 if (error != 0) {
309 /* XXXKIB rollback */
310 return (error);
311 }
312 }
313 return (0);
314 }
315
316 int
iommu_remove_mapping(void * dom,vm_paddr_t gpa,size_t len)317 iommu_remove_mapping(void *dom, vm_paddr_t gpa, size_t len)
318 {
319 uint64_t unmapped, remaining;
320 int error;
321
322 for (remaining = len; remaining > 0; gpa += unmapped,
323 remaining -= unmapped) {
324 error = IOMMU_REMOVE_MAPPING(dom, gpa, remaining, &unmapped);
325 if (error != 0) {
326 /* XXXKIB ? */
327 return (error);
328 }
329 }
330 return (0);
331 }
332
333 void *
iommu_host_domain(void)334 iommu_host_domain(void)
335 {
336
337 return (host_domain);
338 }
339
340 int
iommu_add_device(void * dom,device_t dev,uint16_t rid)341 iommu_add_device(void *dom, device_t dev, uint16_t rid)
342 {
343
344 return (IOMMU_ADD_DEVICE(dom, dev, rid));
345 }
346
347 int
iommu_remove_device(void * dom,device_t dev,uint16_t rid)348 iommu_remove_device(void *dom, device_t dev, uint16_t rid)
349 {
350
351 return (IOMMU_REMOVE_DEVICE(dom, dev, rid));
352 }
353
354 int
iommu_invalidate_tlb(void * domain)355 iommu_invalidate_tlb(void *domain)
356 {
357
358 return (IOMMU_INVALIDATE_TLB(domain));
359 }
360