1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2011 NetApp, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * $FreeBSD$ 29 */ 30 /* 31 * This file and its contents are supplied under the terms of the 32 * Common Development and Distribution License ("CDDL"), version 1.0. 33 * You may only use this file in accordance with the terms of version 34 * 1.0 of the CDDL. 35 * 36 * A full copy of the text of the CDDL should have accompanied this 37 * source. A copy of the CDDL is also available via the Internet at 38 * http://www.illumos.org/license/CDDL. 39 * 40 * Copyright 2022 Oxide Computer Company 41 */ 42 43 #include <sys/cdefs.h> 44 __FBSDID("$FreeBSD$"); 45 46 #include <sys/param.h> 47 #include <sys/bus.h> 48 #include <sys/eventhandler.h> 49 #include <sys/sysctl.h> 50 #include <sys/systm.h> 51 52 #include <dev/pci/pcivar.h> 53 #include <dev/pci/pcireg.h> 54 55 #include <machine/cpu.h> 56 #include <machine/md_var.h> 57 58 #include <sys/ddi.h> 59 #include <sys/sunddi.h> 60 #include <sys/pci.h> 61 62 #include "vmm_util.h" 63 #include "iommu.h" 64 65 66 static kmutex_t iommu_lock; 67 68 static uint_t iommu_refcnt; 69 ddi_modhandle_t iommu_modhdl; 70 static const struct iommu_ops *ops; 71 static void *host_domain; 72 73 static volatile uint_t iommu_initted; 74 75 static int 76 iommu_find_device(dev_info_t *dip, void *arg) 77 { 78 boolean_t add = (boolean_t)arg; 79 80 if (pcie_is_pci_device(dip)) { 81 if (add) 82 iommu_add_device(host_domain, pci_get_rid(dip)); 83 else 84 iommu_remove_device(host_domain, pci_get_rid(dip)); 85 } 86 87 return (DDI_WALK_CONTINUE); 88 } 89 90 static vm_paddr_t 91 vmm_mem_maxaddr(void) 92 { 93 return (ptoa(physmax + 1)); 94 } 95 96 static int 97 iommu_init(void) 98 { 99 const char *mod_name; 100 int error = 0; 101 102 ASSERT(MUTEX_HELD(&iommu_lock)); 103 104 if (vmm_is_intel()) { 105 mod_name = "misc/vmm_vtd"; 106 } else if (vmm_is_svm()) { 107 /* Use the expected name for if/when this is ported */ 108 mod_name = "misc/vmm_amdvi"; 109 } else { 110 return (ENXIO); 111 } 112 113 /* Load the backend driver */ 114 iommu_modhdl = ddi_modopen(mod_name, KRTLD_MODE_FIRST, &error); 115 if (iommu_modhdl == NULL) { 116 return (error); 117 } 118 119 /* Locate the iommu_ops struct */ 120 ops = ddi_modsym(iommu_modhdl, IOMMU_OPS_SYM_NAME, &error); 121 if (ops == NULL) { 122 goto bail; 123 } 124 125 /* Initialize the backend */ 126 error = ops->init(); 127 if (error != 0) { 128 goto bail; 129 } 130 131 /* Create a domain for the devices owned by the host */ 132 const vm_paddr_t maxaddr = vmm_mem_maxaddr(); 133 host_domain = ops->create_domain(maxaddr); 134 if (host_domain == NULL) { 135 goto bail; 136 } 137 138 /* ... and populate it with 1:1 mappings for all of physical mem */ 139 iommu_create_mapping(host_domain, 0, 0, maxaddr); 140 141 ddi_walk_devs(ddi_root_node(), iommu_find_device, (void *)B_TRUE); 142 ops->enable(); 143 144 return (0); 145 146 bail: 147 if (ops != NULL) { 148 ops->cleanup(); 149 ops = NULL; 150 } 151 if (iommu_modhdl != NULL) { 152 (void) ddi_modclose(iommu_modhdl); 153 iommu_modhdl = NULL; 154 } 155 return (error); 156 } 157 158 static void 159 iommu_cleanup(void) 160 { 161 ASSERT(MUTEX_HELD(&iommu_lock)); 162 ASSERT3P(ops, !=, NULL); 163 ASSERT0(iommu_refcnt); 164 165 ops->disable(); 166 ddi_walk_devs(ddi_root_node(), iommu_find_device, (void *)B_FALSE); 167 168 ops->destroy_domain(host_domain); 169 host_domain = NULL; 170 171 ops->cleanup(); 172 ops = NULL; 173 174 (void) ddi_modclose(iommu_modhdl); 175 iommu_modhdl = NULL; 176 } 177 178 static bool 179 iommu_ref(void) 180 { 181 mutex_enter(&iommu_lock); 182 if (ops == NULL) { 183 int err = iommu_init(); 184 185 if (err != 0) { 186 VERIFY3P(ops, ==, NULL); 187 mutex_exit(&iommu_lock); 188 return (false); 189 } 190 VERIFY3P(ops, !=, NULL); 191 } 192 iommu_refcnt++; 193 VERIFY3U(iommu_refcnt, <, UINT_MAX); 194 mutex_exit(&iommu_lock); 195 196 return (true); 197 } 198 199 static void 200 iommu_unref(void) 201 { 202 mutex_enter(&iommu_lock); 203 VERIFY3U(iommu_refcnt, >, 0); 204 iommu_refcnt--; 205 if (iommu_refcnt == 0) { 206 iommu_cleanup(); 207 VERIFY3P(ops, ==, NULL); 208 } 209 mutex_exit(&iommu_lock); 210 } 211 212 void * 213 iommu_create_domain(vm_paddr_t maxaddr) 214 { 215 if (iommu_ref()) { 216 return (ops->create_domain(maxaddr)); 217 } else { 218 return (NULL); 219 } 220 } 221 222 void 223 iommu_destroy_domain(void *domain) 224 { 225 ASSERT3P(domain, !=, NULL); 226 227 ops->destroy_domain(domain); 228 iommu_unref(); 229 } 230 231 void 232 iommu_create_mapping(void *domain, vm_paddr_t gpa, vm_paddr_t hpa, size_t len) 233 { 234 uint64_t remaining = len; 235 236 ASSERT3P(domain, !=, NULL); 237 238 while (remaining > 0) { 239 uint64_t mapped; 240 241 mapped = ops->create_mapping(domain, gpa, hpa, remaining); 242 gpa += mapped; 243 hpa += mapped; 244 remaining -= mapped; 245 } 246 } 247 248 void 249 iommu_remove_mapping(void *domain, vm_paddr_t gpa, size_t len) 250 { 251 uint64_t remaining = len; 252 253 ASSERT3P(domain, !=, NULL); 254 255 while (remaining > 0) { 256 uint64_t unmapped; 257 258 unmapped = ops->remove_mapping(domain, gpa, remaining); 259 gpa += unmapped; 260 remaining -= unmapped; 261 } 262 } 263 264 void * 265 iommu_host_domain(void) 266 { 267 return (host_domain); 268 } 269 270 void 271 iommu_add_device(void *domain, uint16_t rid) 272 { 273 ASSERT3P(domain, !=, NULL); 274 275 ops->add_device(domain, rid); 276 } 277 278 void 279 iommu_remove_device(void *domain, uint16_t rid) 280 { 281 ASSERT3P(domain, !=, NULL); 282 283 ops->remove_device(domain, rid); 284 } 285 286 void 287 iommu_invalidate_tlb(void *domain) 288 { 289 ASSERT3P(domain, !=, NULL); 290 291 ops->invalidate_tlb(domain); 292 } 293