1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2011 NetApp, Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28 /*
29 * This file and its contents are supplied under the terms of the
30 * Common Development and Distribution License ("CDDL"), version 1.0.
31 * You may only use this file in accordance with the terms of version
32 * 1.0 of the CDDL.
33 *
34 * A full copy of the text of the CDDL should have accompanied this
35 * source. A copy of the CDDL is also available via the Internet at
36 * http://www.illumos.org/license/CDDL.
37 *
38 * Copyright 2022 Oxide Computer Company
39 */
40
41 #include <sys/cdefs.h>
42
43 #include <sys/param.h>
44 #include <sys/bus.h>
45 #include <sys/eventhandler.h>
46 #include <sys/sysctl.h>
47 #include <sys/systm.h>
48
49 #include <dev/pci/pcivar.h>
50 #include <dev/pci/pcireg.h>
51
52 #include <machine/cpu.h>
53 #include <machine/md_var.h>
54
55 #include <sys/ddi.h>
56 #include <sys/sunddi.h>
57 #include <sys/pci.h>
58
59 #include "vmm_util.h"
60 #include "iommu.h"
61
62
63 static kmutex_t iommu_lock;
64
65 static uint_t iommu_refcnt;
66 ddi_modhandle_t iommu_modhdl;
67 static const struct iommu_ops *ops;
68 static void *host_domain;
69
70 static int
iommu_find_device(dev_info_t * dip,void * arg)71 iommu_find_device(dev_info_t *dip, void *arg)
72 {
73 boolean_t add = (boolean_t)(uintptr_t)arg;
74
75 if (pcie_is_pci_device(dip)) {
76 if (add)
77 iommu_add_device(host_domain, pci_get_rid(dip));
78 else
79 iommu_remove_device(host_domain, pci_get_rid(dip));
80 }
81
82 return (DDI_WALK_CONTINUE);
83 }
84
85 static vm_paddr_t
vmm_mem_maxaddr(void)86 vmm_mem_maxaddr(void)
87 {
88 return (ptoa(physmax + 1));
89 }
90
91 static int
iommu_init(void)92 iommu_init(void)
93 {
94 const char *mod_name;
95 int error = 0;
96
97 ASSERT(MUTEX_HELD(&iommu_lock));
98
99 if (vmm_is_intel()) {
100 mod_name = "misc/vmm_vtd";
101 } else if (vmm_is_svm()) {
102 /* Use the expected name for if/when this is ported */
103 mod_name = "misc/vmm_amdvi";
104 } else {
105 return (ENXIO);
106 }
107
108 /* Load the backend driver */
109 iommu_modhdl = ddi_modopen(mod_name, KRTLD_MODE_FIRST, &error);
110 if (iommu_modhdl == NULL) {
111 return (error);
112 }
113
114 /* Locate the iommu_ops struct */
115 ops = ddi_modsym(iommu_modhdl, IOMMU_OPS_SYM_NAME, &error);
116 if (ops == NULL) {
117 goto bail;
118 }
119
120 /* Initialize the backend */
121 error = ops->init();
122 if (error != 0) {
123 goto bail;
124 }
125
126 /* Create a domain for the devices owned by the host */
127 const vm_paddr_t maxaddr = vmm_mem_maxaddr();
128 host_domain = ops->create_domain(maxaddr);
129 if (host_domain == NULL) {
130 goto bail;
131 }
132
133 /* ... and populate it with 1:1 mappings for all of physical mem */
134 iommu_create_mapping(host_domain, 0, 0, maxaddr);
135
136 ddi_walk_devs(ddi_root_node(), iommu_find_device, (void *)B_TRUE);
137 ops->enable();
138
139 return (0);
140
141 bail:
142 if (ops != NULL) {
143 ops->cleanup();
144 ops = NULL;
145 }
146 if (iommu_modhdl != NULL) {
147 (void) ddi_modclose(iommu_modhdl);
148 iommu_modhdl = NULL;
149 }
150 return (error);
151 }
152
153 static void
iommu_cleanup(void)154 iommu_cleanup(void)
155 {
156 ASSERT(MUTEX_HELD(&iommu_lock));
157 ASSERT3P(ops, !=, NULL);
158 ASSERT0(iommu_refcnt);
159
160 ops->disable();
161 ddi_walk_devs(ddi_root_node(), iommu_find_device, (void *)B_FALSE);
162
163 ops->destroy_domain(host_domain);
164 host_domain = NULL;
165
166 ops->cleanup();
167 ops = NULL;
168
169 (void) ddi_modclose(iommu_modhdl);
170 iommu_modhdl = NULL;
171 }
172
173 static bool
iommu_ref(void)174 iommu_ref(void)
175 {
176 mutex_enter(&iommu_lock);
177 if (ops == NULL) {
178 int err = iommu_init();
179
180 if (err != 0) {
181 VERIFY3P(ops, ==, NULL);
182 mutex_exit(&iommu_lock);
183 return (false);
184 }
185 VERIFY3P(ops, !=, NULL);
186 }
187 iommu_refcnt++;
188 VERIFY3U(iommu_refcnt, <, UINT_MAX);
189 mutex_exit(&iommu_lock);
190
191 return (true);
192 }
193
194 static void
iommu_unref(void)195 iommu_unref(void)
196 {
197 mutex_enter(&iommu_lock);
198 VERIFY3U(iommu_refcnt, >, 0);
199 iommu_refcnt--;
200 if (iommu_refcnt == 0) {
201 iommu_cleanup();
202 VERIFY3P(ops, ==, NULL);
203 }
204 mutex_exit(&iommu_lock);
205 }
206
207 void *
iommu_create_domain(vm_paddr_t maxaddr)208 iommu_create_domain(vm_paddr_t maxaddr)
209 {
210 if (iommu_ref()) {
211 return (ops->create_domain(maxaddr));
212 } else {
213 return (NULL);
214 }
215 }
216
217 void
iommu_destroy_domain(void * domain)218 iommu_destroy_domain(void *domain)
219 {
220 ASSERT3P(domain, !=, NULL);
221
222 ops->destroy_domain(domain);
223 iommu_unref();
224 }
225
226 void
iommu_create_mapping(void * domain,vm_paddr_t gpa,vm_paddr_t hpa,size_t len)227 iommu_create_mapping(void *domain, vm_paddr_t gpa, vm_paddr_t hpa, size_t len)
228 {
229 uint64_t remaining = len;
230
231 ASSERT3P(domain, !=, NULL);
232
233 while (remaining > 0) {
234 uint64_t mapped;
235
236 mapped = ops->create_mapping(domain, gpa, hpa, remaining);
237 gpa += mapped;
238 hpa += mapped;
239 remaining -= mapped;
240 }
241 }
242
243 void
iommu_remove_mapping(void * domain,vm_paddr_t gpa,size_t len)244 iommu_remove_mapping(void *domain, vm_paddr_t gpa, size_t len)
245 {
246 uint64_t remaining = len;
247
248 ASSERT3P(domain, !=, NULL);
249
250 while (remaining > 0) {
251 uint64_t unmapped;
252
253 unmapped = ops->remove_mapping(domain, gpa, remaining);
254 gpa += unmapped;
255 remaining -= unmapped;
256 }
257 }
258
259 void *
iommu_host_domain(void)260 iommu_host_domain(void)
261 {
262 return (host_domain);
263 }
264
265 void
iommu_add_device(void * domain,uint16_t rid)266 iommu_add_device(void *domain, uint16_t rid)
267 {
268 ASSERT3P(domain, !=, NULL);
269
270 ops->add_device(domain, rid);
271 }
272
273 void
iommu_remove_device(void * domain,uint16_t rid)274 iommu_remove_device(void *domain, uint16_t rid)
275 {
276 ASSERT3P(domain, !=, NULL);
277
278 ops->remove_device(domain, rid);
279 }
280
281 void
iommu_invalidate_tlb(void * domain)282 iommu_invalidate_tlb(void *domain)
283 {
284 ASSERT3P(domain, !=, NULL);
285
286 ops->invalidate_tlb(domain);
287 }
288