1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2011 NetApp, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 /* 29 * This file and its contents are supplied under the terms of the 30 * Common Development and Distribution License ("CDDL"), version 1.0. 31 * You may only use this file in accordance with the terms of version 32 * 1.0 of the CDDL. 33 * 34 * A full copy of the text of the CDDL should have accompanied this 35 * source. A copy of the CDDL is also available via the Internet at 36 * http://www.illumos.org/license/CDDL. 37 * 38 * Copyright 2022 Oxide Computer Company 39 */ 40 41 #include <sys/cdefs.h> 42 43 #include <sys/param.h> 44 #include <sys/bus.h> 45 #include <sys/eventhandler.h> 46 #include <sys/sysctl.h> 47 #include <sys/systm.h> 48 49 #include <dev/pci/pcivar.h> 50 #include <dev/pci/pcireg.h> 51 52 #include <machine/cpu.h> 53 #include <machine/md_var.h> 54 55 #include <sys/ddi.h> 56 #include <sys/sunddi.h> 57 #include <sys/pci.h> 58 59 #include "vmm_util.h" 60 #include "iommu.h" 61 62 63 static kmutex_t iommu_lock; 64 65 static uint_t iommu_refcnt; 66 ddi_modhandle_t iommu_modhdl; 67 static const struct iommu_ops *ops; 68 static void *host_domain; 69 70 static int 71 iommu_find_device(dev_info_t *dip, void *arg) 72 { 73 boolean_t add = (boolean_t)(uintptr_t)arg; 74 75 if (pcie_is_pci_device(dip)) { 76 if (add) 77 iommu_add_device(host_domain, pci_get_rid(dip)); 78 else 79 iommu_remove_device(host_domain, pci_get_rid(dip)); 80 } 81 82 return (DDI_WALK_CONTINUE); 83 } 84 85 static vm_paddr_t 86 vmm_mem_maxaddr(void) 87 { 88 return (ptoa(physmax + 1)); 89 } 90 91 static int 92 iommu_init(void) 93 { 94 const char *mod_name; 95 int error = 0; 96 97 ASSERT(MUTEX_HELD(&iommu_lock)); 98 99 if (vmm_is_intel()) { 100 mod_name = "misc/vmm_vtd"; 101 } else if (vmm_is_svm()) { 102 /* Use the expected name for if/when this is ported */ 103 mod_name = "misc/vmm_amdvi"; 104 } else { 105 return (ENXIO); 106 } 107 108 /* Load the backend driver */ 109 iommu_modhdl = ddi_modopen(mod_name, KRTLD_MODE_FIRST, &error); 110 if (iommu_modhdl == NULL) { 111 return (error); 112 } 113 114 /* Locate the iommu_ops struct */ 115 ops = ddi_modsym(iommu_modhdl, IOMMU_OPS_SYM_NAME, &error); 116 if (ops == NULL) { 117 goto bail; 118 } 119 120 /* Initialize the backend */ 121 error = ops->init(); 122 if (error != 0) { 123 goto bail; 124 } 125 126 /* Create a domain for the devices owned by the host */ 127 const vm_paddr_t maxaddr = vmm_mem_maxaddr(); 128 host_domain = ops->create_domain(maxaddr); 129 if (host_domain == NULL) { 130 goto bail; 131 } 132 133 /* ... and populate it with 1:1 mappings for all of physical mem */ 134 iommu_create_mapping(host_domain, 0, 0, maxaddr); 135 136 ddi_walk_devs(ddi_root_node(), iommu_find_device, (void *)B_TRUE); 137 ops->enable(); 138 139 return (0); 140 141 bail: 142 if (ops != NULL) { 143 ops->cleanup(); 144 ops = NULL; 145 } 146 if (iommu_modhdl != NULL) { 147 (void) ddi_modclose(iommu_modhdl); 148 iommu_modhdl = NULL; 149 } 150 return (error); 151 } 152 153 static void 154 iommu_cleanup(void) 155 { 156 ASSERT(MUTEX_HELD(&iommu_lock)); 157 ASSERT3P(ops, !=, NULL); 158 ASSERT0(iommu_refcnt); 159 160 ops->disable(); 161 ddi_walk_devs(ddi_root_node(), iommu_find_device, (void *)B_FALSE); 162 163 ops->destroy_domain(host_domain); 164 host_domain = NULL; 165 166 ops->cleanup(); 167 ops = NULL; 168 169 (void) ddi_modclose(iommu_modhdl); 170 iommu_modhdl = NULL; 171 } 172 173 static bool 174 iommu_ref(void) 175 { 176 mutex_enter(&iommu_lock); 177 if (ops == NULL) { 178 int err = iommu_init(); 179 180 if (err != 0) { 181 VERIFY3P(ops, ==, NULL); 182 mutex_exit(&iommu_lock); 183 return (false); 184 } 185 VERIFY3P(ops, !=, NULL); 186 } 187 iommu_refcnt++; 188 VERIFY3U(iommu_refcnt, <, UINT_MAX); 189 mutex_exit(&iommu_lock); 190 191 return (true); 192 } 193 194 static void 195 iommu_unref(void) 196 { 197 mutex_enter(&iommu_lock); 198 VERIFY3U(iommu_refcnt, >, 0); 199 iommu_refcnt--; 200 if (iommu_refcnt == 0) { 201 iommu_cleanup(); 202 VERIFY3P(ops, ==, NULL); 203 } 204 mutex_exit(&iommu_lock); 205 } 206 207 void * 208 iommu_create_domain(vm_paddr_t maxaddr) 209 { 210 if (iommu_ref()) { 211 return (ops->create_domain(maxaddr)); 212 } else { 213 return (NULL); 214 } 215 } 216 217 void 218 iommu_destroy_domain(void *domain) 219 { 220 ASSERT3P(domain, !=, NULL); 221 222 ops->destroy_domain(domain); 223 iommu_unref(); 224 } 225 226 void 227 iommu_create_mapping(void *domain, vm_paddr_t gpa, vm_paddr_t hpa, size_t len) 228 { 229 uint64_t remaining = len; 230 231 ASSERT3P(domain, !=, NULL); 232 233 while (remaining > 0) { 234 uint64_t mapped; 235 236 mapped = ops->create_mapping(domain, gpa, hpa, remaining); 237 gpa += mapped; 238 hpa += mapped; 239 remaining -= mapped; 240 } 241 } 242 243 void 244 iommu_remove_mapping(void *domain, vm_paddr_t gpa, size_t len) 245 { 246 uint64_t remaining = len; 247 248 ASSERT3P(domain, !=, NULL); 249 250 while (remaining > 0) { 251 uint64_t unmapped; 252 253 unmapped = ops->remove_mapping(domain, gpa, remaining); 254 gpa += unmapped; 255 remaining -= unmapped; 256 } 257 } 258 259 void * 260 iommu_host_domain(void) 261 { 262 return (host_domain); 263 } 264 265 void 266 iommu_add_device(void *domain, uint16_t rid) 267 { 268 ASSERT3P(domain, !=, NULL); 269 270 ops->add_device(domain, rid); 271 } 272 273 void 274 iommu_remove_device(void *domain, uint16_t rid) 275 { 276 ASSERT3P(domain, !=, NULL); 277 278 ops->remove_device(domain, rid); 279 } 280 281 void 282 iommu_invalidate_tlb(void *domain) 283 { 284 ASSERT3P(domain, !=, NULL); 285 286 ops->invalidate_tlb(domain); 287 } 288