xref: /illumos-gate/usr/src/uts/intel/io/vmm/io/iommu.c (revision 32640292339b07090f10ce34d455f98711077343)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
3  *
4  * Copyright (c) 2011 NetApp, Inc.
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  */
28 /*
29  * This file and its contents are supplied under the terms of the
30  * Common Development and Distribution License ("CDDL"), version 1.0.
31  * You may only use this file in accordance with the terms of version
32  * 1.0 of the CDDL.
33  *
34  * A full copy of the text of the CDDL should have accompanied this
35  * source.  A copy of the CDDL is also available via the Internet at
36  * http://www.illumos.org/license/CDDL.
37  *
38  * Copyright 2022 Oxide Computer Company
39  */
40 
41 #include <sys/cdefs.h>
42 
43 #include <sys/param.h>
44 #include <sys/bus.h>
45 #include <sys/eventhandler.h>
46 #include <sys/sysctl.h>
47 #include <sys/systm.h>
48 
49 #include <dev/pci/pcivar.h>
50 #include <dev/pci/pcireg.h>
51 
52 #include <machine/cpu.h>
53 #include <machine/md_var.h>
54 
55 #include <sys/ddi.h>
56 #include <sys/sunddi.h>
57 #include <sys/pci.h>
58 
59 #include "vmm_util.h"
60 #include "iommu.h"
61 
62 
63 static kmutex_t iommu_lock;
64 
65 static uint_t iommu_refcnt;
66 ddi_modhandle_t iommu_modhdl;
67 static const struct iommu_ops *ops;
68 static void *host_domain;
69 
70 static int
iommu_find_device(dev_info_t * dip,void * arg)71 iommu_find_device(dev_info_t *dip, void *arg)
72 {
73 	boolean_t add = (boolean_t)(uintptr_t)arg;
74 
75 	if (pcie_is_pci_device(dip)) {
76 		if (add)
77 			iommu_add_device(host_domain, pci_get_rid(dip));
78 		else
79 			iommu_remove_device(host_domain, pci_get_rid(dip));
80 	}
81 
82 	return (DDI_WALK_CONTINUE);
83 }
84 
85 static vm_paddr_t
vmm_mem_maxaddr(void)86 vmm_mem_maxaddr(void)
87 {
88 	return (ptoa(physmax + 1));
89 }
90 
91 static int
iommu_init(void)92 iommu_init(void)
93 {
94 	const char *mod_name;
95 	int error = 0;
96 
97 	ASSERT(MUTEX_HELD(&iommu_lock));
98 
99 	if (vmm_is_intel()) {
100 		mod_name = "misc/vmm_vtd";
101 	} else if (vmm_is_svm()) {
102 		/* Use the expected name for if/when this is ported */
103 		mod_name = "misc/vmm_amdvi";
104 	} else {
105 		return (ENXIO);
106 	}
107 
108 	/* Load the backend driver */
109 	iommu_modhdl = ddi_modopen(mod_name, KRTLD_MODE_FIRST, &error);
110 	if (iommu_modhdl == NULL) {
111 		return (error);
112 	}
113 
114 	/* Locate the iommu_ops struct */
115 	ops = ddi_modsym(iommu_modhdl, IOMMU_OPS_SYM_NAME, &error);
116 	if (ops == NULL) {
117 		goto bail;
118 	}
119 
120 	/* Initialize the backend */
121 	error = ops->init();
122 	if (error != 0) {
123 		goto bail;
124 	}
125 
126 	/* Create a domain for the devices owned by the host */
127 	const vm_paddr_t maxaddr = vmm_mem_maxaddr();
128 	host_domain = ops->create_domain(maxaddr);
129 	if (host_domain == NULL) {
130 		goto bail;
131 	}
132 
133 	/* ... and populate it with 1:1 mappings for all of physical mem */
134 	iommu_create_mapping(host_domain, 0, 0, maxaddr);
135 
136 	ddi_walk_devs(ddi_root_node(), iommu_find_device, (void *)B_TRUE);
137 	ops->enable();
138 
139 	return (0);
140 
141 bail:
142 	if (ops != NULL) {
143 		ops->cleanup();
144 		ops = NULL;
145 	}
146 	if (iommu_modhdl != NULL) {
147 		(void) ddi_modclose(iommu_modhdl);
148 		iommu_modhdl = NULL;
149 	}
150 	return (error);
151 }
152 
153 static void
iommu_cleanup(void)154 iommu_cleanup(void)
155 {
156 	ASSERT(MUTEX_HELD(&iommu_lock));
157 	ASSERT3P(ops, !=, NULL);
158 	ASSERT0(iommu_refcnt);
159 
160 	ops->disable();
161 	ddi_walk_devs(ddi_root_node(), iommu_find_device, (void *)B_FALSE);
162 
163 	ops->destroy_domain(host_domain);
164 	host_domain = NULL;
165 
166 	ops->cleanup();
167 	ops = NULL;
168 
169 	(void) ddi_modclose(iommu_modhdl);
170 	iommu_modhdl = NULL;
171 }
172 
173 static bool
iommu_ref(void)174 iommu_ref(void)
175 {
176 	mutex_enter(&iommu_lock);
177 	if (ops == NULL) {
178 		int err = iommu_init();
179 
180 		if (err != 0) {
181 			VERIFY3P(ops, ==, NULL);
182 			mutex_exit(&iommu_lock);
183 			return (false);
184 		}
185 		VERIFY3P(ops, !=, NULL);
186 	}
187 	iommu_refcnt++;
188 	VERIFY3U(iommu_refcnt, <, UINT_MAX);
189 	mutex_exit(&iommu_lock);
190 
191 	return (true);
192 }
193 
194 static void
iommu_unref(void)195 iommu_unref(void)
196 {
197 	mutex_enter(&iommu_lock);
198 	VERIFY3U(iommu_refcnt, >, 0);
199 	iommu_refcnt--;
200 	if (iommu_refcnt == 0) {
201 		iommu_cleanup();
202 		VERIFY3P(ops, ==, NULL);
203 	}
204 	mutex_exit(&iommu_lock);
205 }
206 
207 void *
iommu_create_domain(vm_paddr_t maxaddr)208 iommu_create_domain(vm_paddr_t maxaddr)
209 {
210 	if (iommu_ref()) {
211 		return (ops->create_domain(maxaddr));
212 	} else {
213 		return (NULL);
214 	}
215 }
216 
217 void
iommu_destroy_domain(void * domain)218 iommu_destroy_domain(void *domain)
219 {
220 	ASSERT3P(domain, !=, NULL);
221 
222 	ops->destroy_domain(domain);
223 	iommu_unref();
224 }
225 
226 void
iommu_create_mapping(void * domain,vm_paddr_t gpa,vm_paddr_t hpa,size_t len)227 iommu_create_mapping(void *domain, vm_paddr_t gpa, vm_paddr_t hpa, size_t len)
228 {
229 	uint64_t remaining = len;
230 
231 	ASSERT3P(domain, !=, NULL);
232 
233 	while (remaining > 0) {
234 		uint64_t mapped;
235 
236 		mapped = ops->create_mapping(domain, gpa, hpa, remaining);
237 		gpa += mapped;
238 		hpa += mapped;
239 		remaining -= mapped;
240 	}
241 }
242 
243 void
iommu_remove_mapping(void * domain,vm_paddr_t gpa,size_t len)244 iommu_remove_mapping(void *domain, vm_paddr_t gpa, size_t len)
245 {
246 	uint64_t remaining = len;
247 
248 	ASSERT3P(domain, !=, NULL);
249 
250 	while (remaining > 0) {
251 		uint64_t unmapped;
252 
253 		unmapped = ops->remove_mapping(domain, gpa, remaining);
254 		gpa += unmapped;
255 		remaining -= unmapped;
256 	}
257 }
258 
259 void *
iommu_host_domain(void)260 iommu_host_domain(void)
261 {
262 	return (host_domain);
263 }
264 
265 void
iommu_add_device(void * domain,uint16_t rid)266 iommu_add_device(void *domain, uint16_t rid)
267 {
268 	ASSERT3P(domain, !=, NULL);
269 
270 	ops->add_device(domain, rid);
271 }
272 
273 void
iommu_remove_device(void * domain,uint16_t rid)274 iommu_remove_device(void *domain, uint16_t rid)
275 {
276 	ASSERT3P(domain, !=, NULL);
277 
278 	ops->remove_device(domain, rid);
279 }
280 
281 void
iommu_invalidate_tlb(void * domain)282 iommu_invalidate_tlb(void *domain)
283 {
284 	ASSERT3P(domain, !=, NULL);
285 
286 	ops->invalidate_tlb(domain);
287 }
288