1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2011 NetApp, Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 #include <sys/param.h>
30 #include <sys/bus.h>
31 #include <sys/eventhandler.h>
32 #include <sys/sysctl.h>
33 #include <sys/systm.h>
34
35 #include <dev/pci/pcivar.h>
36 #include <dev/pci/pcireg.h>
37
38 #include <machine/cpu.h>
39 #include <machine/md_var.h>
40
41 #include "vmm_util.h"
42 #include "vmm_mem.h"
43 #include "iommu.h"
44
45 SYSCTL_DECL(_hw_vmm);
46 SYSCTL_NODE(_hw_vmm, OID_AUTO, iommu, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
47 "bhyve iommu parameters");
48
49 static int iommu_avail;
50 SYSCTL_INT(_hw_vmm_iommu, OID_AUTO, initialized, CTLFLAG_RD, &iommu_avail,
51 0, "bhyve iommu initialized?");
52
53 static int iommu_enable = 1;
54 SYSCTL_INT(_hw_vmm_iommu, OID_AUTO, enable, CTLFLAG_RDTUN, &iommu_enable, 0,
55 "Enable use of I/O MMU (required for PCI passthrough).");
56
57 static const struct iommu_ops *ops;
58 static void *host_domain;
59 static eventhandler_tag add_tag, delete_tag;
60
61 static __inline int
IOMMU_INIT(void)62 IOMMU_INIT(void)
63 {
64 if (ops != NULL)
65 return ((*ops->init)());
66 else
67 return (ENXIO);
68 }
69
70 static __inline void
IOMMU_CLEANUP(void)71 IOMMU_CLEANUP(void)
72 {
73 if (ops != NULL && iommu_avail)
74 (*ops->cleanup)();
75 }
76
77 static __inline void *
IOMMU_CREATE_DOMAIN(vm_paddr_t maxaddr)78 IOMMU_CREATE_DOMAIN(vm_paddr_t maxaddr)
79 {
80
81 if (ops != NULL && iommu_avail)
82 return ((*ops->create_domain)(maxaddr));
83 else
84 return (NULL);
85 }
86
87 static __inline void
IOMMU_DESTROY_DOMAIN(void * dom)88 IOMMU_DESTROY_DOMAIN(void *dom)
89 {
90
91 if (ops != NULL && iommu_avail)
92 (*ops->destroy_domain)(dom);
93 }
94
95 static __inline uint64_t
IOMMU_CREATE_MAPPING(void * domain,vm_paddr_t gpa,vm_paddr_t hpa,uint64_t len)96 IOMMU_CREATE_MAPPING(void *domain, vm_paddr_t gpa, vm_paddr_t hpa, uint64_t len)
97 {
98
99 if (ops != NULL && iommu_avail)
100 return ((*ops->create_mapping)(domain, gpa, hpa, len));
101 else
102 return (len); /* XXX */
103 }
104
105 static __inline uint64_t
IOMMU_REMOVE_MAPPING(void * domain,vm_paddr_t gpa,uint64_t len)106 IOMMU_REMOVE_MAPPING(void *domain, vm_paddr_t gpa, uint64_t len)
107 {
108
109 if (ops != NULL && iommu_avail)
110 return ((*ops->remove_mapping)(domain, gpa, len));
111 else
112 return (len); /* XXX */
113 }
114
115 static __inline void
IOMMU_ADD_DEVICE(void * domain,uint16_t rid)116 IOMMU_ADD_DEVICE(void *domain, uint16_t rid)
117 {
118
119 if (ops != NULL && iommu_avail)
120 (*ops->add_device)(domain, rid);
121 }
122
123 static __inline void
IOMMU_REMOVE_DEVICE(void * domain,uint16_t rid)124 IOMMU_REMOVE_DEVICE(void *domain, uint16_t rid)
125 {
126
127 if (ops != NULL && iommu_avail)
128 (*ops->remove_device)(domain, rid);
129 }
130
131 static __inline void
IOMMU_INVALIDATE_TLB(void * domain)132 IOMMU_INVALIDATE_TLB(void *domain)
133 {
134
135 if (ops != NULL && iommu_avail)
136 (*ops->invalidate_tlb)(domain);
137 }
138
139 static __inline void
IOMMU_ENABLE(void)140 IOMMU_ENABLE(void)
141 {
142
143 if (ops != NULL && iommu_avail)
144 (*ops->enable)();
145 }
146
147 static __inline void
IOMMU_DISABLE(void)148 IOMMU_DISABLE(void)
149 {
150
151 if (ops != NULL && iommu_avail)
152 (*ops->disable)();
153 }
154
155 static void
iommu_pci_add(void * arg,device_t dev)156 iommu_pci_add(void *arg, device_t dev)
157 {
158
159 /* Add new devices to the host domain. */
160 iommu_add_device(host_domain, pci_get_rid(dev));
161 }
162
163 static void
iommu_pci_delete(void * arg,device_t dev)164 iommu_pci_delete(void *arg, device_t dev)
165 {
166
167 iommu_remove_device(host_domain, pci_get_rid(dev));
168 }
169
170 static void
iommu_init(void)171 iommu_init(void)
172 {
173 int error, bus, slot, func;
174 vm_paddr_t maxaddr;
175 devclass_t dc;
176 device_t dev;
177
178 if (!iommu_enable)
179 return;
180
181 if (vmm_is_intel())
182 ops = &iommu_ops_intel;
183 else if (vmm_is_svm())
184 ops = &iommu_ops_amd;
185 else
186 ops = NULL;
187
188 error = IOMMU_INIT();
189 if (error)
190 return;
191
192 iommu_avail = 1;
193
194 /*
195 * Create a domain for the devices owned by the host
196 */
197 maxaddr = vmm_mem_maxaddr();
198 host_domain = IOMMU_CREATE_DOMAIN(maxaddr);
199 if (host_domain == NULL) {
200 printf("iommu_init: unable to create a host domain");
201 IOMMU_CLEANUP();
202 ops = NULL;
203 iommu_avail = 0;
204 return;
205 }
206
207 /*
208 * Create 1:1 mappings from '0' to 'maxaddr' for devices assigned to
209 * the host
210 */
211 iommu_create_mapping(host_domain, 0, 0, maxaddr);
212
213 add_tag = EVENTHANDLER_REGISTER(pci_add_device, iommu_pci_add, NULL, 0);
214 delete_tag = EVENTHANDLER_REGISTER(pci_delete_device, iommu_pci_delete,
215 NULL, 0);
216 dc = devclass_find("ppt");
217 for (bus = 0; bus <= PCI_BUSMAX; bus++) {
218 for (slot = 0; slot <= PCI_SLOTMAX; slot++) {
219 for (func = 0; func <= PCI_FUNCMAX; func++) {
220 dev = pci_find_dbsf(0, bus, slot, func);
221 if (dev == NULL)
222 continue;
223
224 /* Skip passthrough devices. */
225 if (dc != NULL &&
226 device_get_devclass(dev) == dc)
227 continue;
228
229 /*
230 * Everything else belongs to the host
231 * domain.
232 */
233 iommu_add_device(host_domain,
234 pci_get_rid(dev));
235 }
236 }
237 }
238 IOMMU_ENABLE();
239
240 }
241
242 void
iommu_cleanup(void)243 iommu_cleanup(void)
244 {
245
246 if (add_tag != NULL) {
247 EVENTHANDLER_DEREGISTER(pci_add_device, add_tag);
248 add_tag = NULL;
249 }
250 if (delete_tag != NULL) {
251 EVENTHANDLER_DEREGISTER(pci_delete_device, delete_tag);
252 delete_tag = NULL;
253 }
254 IOMMU_DISABLE();
255 IOMMU_DESTROY_DOMAIN(host_domain);
256 host_domain = NULL;
257 IOMMU_CLEANUP();
258 }
259
260 void *
iommu_create_domain(vm_paddr_t maxaddr)261 iommu_create_domain(vm_paddr_t maxaddr)
262 {
263 static volatile int iommu_initted;
264
265 if (iommu_initted < 2) {
266 if (atomic_cmpset_int(&iommu_initted, 0, 1)) {
267 iommu_init();
268 atomic_store_rel_int(&iommu_initted, 2);
269 } else
270 while (iommu_initted == 1)
271 cpu_spinwait();
272 }
273 return (IOMMU_CREATE_DOMAIN(maxaddr));
274 }
275
276 void
iommu_destroy_domain(void * dom)277 iommu_destroy_domain(void *dom)
278 {
279
280 IOMMU_DESTROY_DOMAIN(dom);
281 }
282
283 void
iommu_create_mapping(void * dom,vm_paddr_t gpa,vm_paddr_t hpa,size_t len)284 iommu_create_mapping(void *dom, vm_paddr_t gpa, vm_paddr_t hpa, size_t len)
285 {
286 uint64_t mapped, remaining;
287
288 remaining = len;
289
290 while (remaining > 0) {
291 mapped = IOMMU_CREATE_MAPPING(dom, gpa, hpa, remaining);
292 gpa += mapped;
293 hpa += mapped;
294 remaining -= mapped;
295 }
296 }
297
298 void
iommu_remove_mapping(void * dom,vm_paddr_t gpa,size_t len)299 iommu_remove_mapping(void *dom, vm_paddr_t gpa, size_t len)
300 {
301 uint64_t unmapped, remaining;
302
303 remaining = len;
304
305 while (remaining > 0) {
306 unmapped = IOMMU_REMOVE_MAPPING(dom, gpa, remaining);
307 gpa += unmapped;
308 remaining -= unmapped;
309 }
310 }
311
312 void *
iommu_host_domain(void)313 iommu_host_domain(void)
314 {
315
316 return (host_domain);
317 }
318
319 void
iommu_add_device(void * dom,uint16_t rid)320 iommu_add_device(void *dom, uint16_t rid)
321 {
322
323 IOMMU_ADD_DEVICE(dom, rid);
324 }
325
326 void
iommu_remove_device(void * dom,uint16_t rid)327 iommu_remove_device(void *dom, uint16_t rid)
328 {
329
330 IOMMU_REMOVE_DEVICE(dom, rid);
331 }
332
333 void
iommu_invalidate_tlb(void * domain)334 iommu_invalidate_tlb(void *domain)
335 {
336
337 IOMMU_INVALIDATE_TLB(domain);
338 }
339