1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2011 NetApp, Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28 /*
29 * This file and its contents are supplied under the terms of the
30 * Common Development and Distribution License ("CDDL"), version 1.0.
31 * You may only use this file in accordance with the terms of version
32 * 1.0 of the CDDL.
33 *
34 * A full copy of the text of the CDDL should have accompanied this
35 * source. A copy of the CDDL is also available via the Internet at
36 * http://www.illumos.org/license/CDDL.
37 */
38 /* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */
39
40 /*
41 * Copyright 2022 Oxide Computer Company
42 */
43
44 #include <sys/cdefs.h>
45
46 #include <sys/param.h>
47 #include <sys/bus.h>
48 #include <sys/eventhandler.h>
49 #include <sys/sysctl.h>
50 #include <sys/systm.h>
51
52 #include <dev/pci/pcivar.h>
53 #include <dev/pci/pcireg.h>
54
55 #include <machine/cpu.h>
56 #include <machine/md_var.h>
57
58 #include <sys/ddi.h>
59 #include <sys/sunddi.h>
60 #include <sys/pci.h>
61
62 #include "vmm_util.h"
63 #include "iommu.h"
64
65
66 static kmutex_t iommu_lock;
67
68 static uint_t iommu_refcnt;
69 ddi_modhandle_t iommu_modhdl;
70 static const struct iommu_ops *ops;
71 static void *host_domain;
72
73 static int
iommu_find_device(dev_info_t * dip,void * arg)74 iommu_find_device(dev_info_t *dip, void *arg)
75 {
76 boolean_t add = (boolean_t)(uintptr_t)arg;
77
78 if (pcie_is_pci_device(dip)) {
79 if (add)
80 iommu_add_device(host_domain, pci_get_rid(dip));
81 else
82 iommu_remove_device(host_domain, pci_get_rid(dip));
83 }
84
85 return (DDI_WALK_CONTINUE);
86 }
87
88 static vm_paddr_t
vmm_mem_maxaddr(void)89 vmm_mem_maxaddr(void)
90 {
91 return (ptoa(physmax + 1));
92 }
93
94 static int
iommu_init(void)95 iommu_init(void)
96 {
97 const char *mod_name;
98 int error = 0;
99
100 ASSERT(MUTEX_HELD(&iommu_lock));
101
102 if (vmm_is_intel()) {
103 mod_name = "misc/vmm_vtd";
104 } else if (vmm_is_svm()) {
105 /* Use the expected name for if/when this is ported */
106 mod_name = "misc/vmm_amdvi";
107 } else {
108 return (ENXIO);
109 }
110
111 /* Load the backend driver */
112 iommu_modhdl = ddi_modopen(mod_name, KRTLD_MODE_FIRST, &error);
113 if (iommu_modhdl == NULL) {
114 return (error);
115 }
116
117 /* Locate the iommu_ops struct */
118 ops = ddi_modsym(iommu_modhdl, IOMMU_OPS_SYM_NAME, &error);
119 if (ops == NULL) {
120 goto bail;
121 }
122
123 /* Initialize the backend */
124 error = ops->init();
125 if (error != 0) {
126 goto bail;
127 }
128
129 /* Create a domain for the devices owned by the host */
130 const vm_paddr_t maxaddr = vmm_mem_maxaddr();
131 host_domain = ops->create_domain(maxaddr);
132 if (host_domain == NULL) {
133 goto bail;
134 }
135
136 /* ... and populate it with 1:1 mappings for all of physical mem */
137 iommu_create_mapping(host_domain, 0, 0, maxaddr);
138
139 ddi_walk_devs(ddi_root_node(), iommu_find_device, (void *)B_TRUE);
140 ops->enable();
141
142 return (0);
143
144 bail:
145 if (ops != NULL) {
146 ops->cleanup();
147 ops = NULL;
148 }
149 if (iommu_modhdl != NULL) {
150 (void) ddi_modclose(iommu_modhdl);
151 iommu_modhdl = NULL;
152 }
153 return (error);
154 }
155
156 static void
iommu_cleanup(void)157 iommu_cleanup(void)
158 {
159 ASSERT(MUTEX_HELD(&iommu_lock));
160 ASSERT3P(ops, !=, NULL);
161 ASSERT0(iommu_refcnt);
162
163 ops->disable();
164 ddi_walk_devs(ddi_root_node(), iommu_find_device, (void *)B_FALSE);
165
166 ops->destroy_domain(host_domain);
167 host_domain = NULL;
168
169 ops->cleanup();
170 ops = NULL;
171
172 (void) ddi_modclose(iommu_modhdl);
173 iommu_modhdl = NULL;
174 }
175
176 static bool
iommu_ref(void)177 iommu_ref(void)
178 {
179 mutex_enter(&iommu_lock);
180 if (ops == NULL) {
181 int err = iommu_init();
182
183 if (err != 0) {
184 VERIFY3P(ops, ==, NULL);
185 mutex_exit(&iommu_lock);
186 return (false);
187 }
188 VERIFY3P(ops, !=, NULL);
189 }
190 iommu_refcnt++;
191 VERIFY3U(iommu_refcnt, <, UINT_MAX);
192 mutex_exit(&iommu_lock);
193
194 return (true);
195 }
196
197 static void
iommu_unref(void)198 iommu_unref(void)
199 {
200 mutex_enter(&iommu_lock);
201 VERIFY3U(iommu_refcnt, >, 0);
202 iommu_refcnt--;
203 if (iommu_refcnt == 0) {
204 iommu_cleanup();
205 VERIFY3P(ops, ==, NULL);
206 }
207 mutex_exit(&iommu_lock);
208 }
209
210 void *
iommu_create_domain(vm_paddr_t maxaddr)211 iommu_create_domain(vm_paddr_t maxaddr)
212 {
213 if (iommu_ref()) {
214 return (ops->create_domain(maxaddr));
215 } else {
216 return (NULL);
217 }
218 }
219
220 void
iommu_destroy_domain(void * domain)221 iommu_destroy_domain(void *domain)
222 {
223 ASSERT3P(domain, !=, NULL);
224
225 ops->destroy_domain(domain);
226 iommu_unref();
227 }
228
229 void
iommu_create_mapping(void * domain,vm_paddr_t gpa,vm_paddr_t hpa,size_t len)230 iommu_create_mapping(void *domain, vm_paddr_t gpa, vm_paddr_t hpa, size_t len)
231 {
232 uint64_t remaining = len;
233
234 ASSERT3P(domain, !=, NULL);
235
236 while (remaining > 0) {
237 uint64_t mapped;
238
239 mapped = ops->create_mapping(domain, gpa, hpa, remaining);
240 gpa += mapped;
241 hpa += mapped;
242 remaining -= mapped;
243 }
244 }
245
246 void
iommu_remove_mapping(void * domain,vm_paddr_t gpa,size_t len)247 iommu_remove_mapping(void *domain, vm_paddr_t gpa, size_t len)
248 {
249 uint64_t remaining = len;
250
251 ASSERT3P(domain, !=, NULL);
252
253 while (remaining > 0) {
254 uint64_t unmapped;
255
256 unmapped = ops->remove_mapping(domain, gpa, remaining);
257 gpa += unmapped;
258 remaining -= unmapped;
259 }
260 }
261
262 void *
iommu_host_domain(void)263 iommu_host_domain(void)
264 {
265 return (host_domain);
266 }
267
268 void
iommu_add_device(void * domain,uint16_t rid)269 iommu_add_device(void *domain, uint16_t rid)
270 {
271 ASSERT3P(domain, !=, NULL);
272
273 ops->add_device(domain, rid);
274 }
275
276 void
iommu_remove_device(void * domain,uint16_t rid)277 iommu_remove_device(void *domain, uint16_t rid)
278 {
279 ASSERT3P(domain, !=, NULL);
280
281 ops->remove_device(domain, rid);
282 }
283
284 void
iommu_invalidate_tlb(void * domain)285 iommu_invalidate_tlb(void *domain)
286 {
287 ASSERT3P(domain, !=, NULL);
288
289 ops->invalidate_tlb(domain);
290 }
291