1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2011 NetApp, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <sys/param.h> 33 #include <sys/kernel.h> 34 #include <sys/systm.h> 35 #include <sys/malloc.h> 36 37 #include <vm/vm.h> 38 #include <vm/pmap.h> 39 40 #include <dev/pci/pcireg.h> 41 42 #include <machine/vmparam.h> 43 #include <contrib/dev/acpica/include/acpi.h> 44 45 #include "io/iommu.h" 46 47 /* 48 * Documented in the "Intel Virtualization Technology for Directed I/O", 49 * Architecture Spec, September 2008. 50 */ 51 52 #define VTD_DRHD_INCLUDE_PCI_ALL(Flags) (((Flags) >> 0) & 0x1) 53 54 /* Section 10.4 "Register Descriptions" */ 55 struct vtdmap { 56 volatile uint32_t version; 57 volatile uint32_t res0; 58 volatile uint64_t cap; 59 volatile uint64_t ext_cap; 60 volatile uint32_t gcr; 61 volatile uint32_t gsr; 62 volatile uint64_t rta; 63 volatile uint64_t ccr; 64 }; 65 66 #define VTD_CAP_SAGAW(cap) (((cap) >> 8) & 0x1F) 67 #define VTD_CAP_ND(cap) ((cap) & 0x7) 68 #define VTD_CAP_CM(cap) (((cap) >> 7) & 0x1) 69 #define VTD_CAP_SPS(cap) (((cap) >> 34) & 0xF) 70 #define VTD_CAP_RWBF(cap) (((cap) >> 4) & 0x1) 71 72 #define VTD_ECAP_DI(ecap) (((ecap) >> 2) & 0x1) 73 #define VTD_ECAP_COHERENCY(ecap) ((ecap) & 0x1) 74 #define VTD_ECAP_IRO(ecap) (((ecap) >> 8) & 0x3FF) 75 76 #define VTD_GCR_WBF (1 << 27) 77 #define VTD_GCR_SRTP (1 << 30) 78 #define VTD_GCR_TE (1U << 31) 79 80 #define VTD_GSR_WBFS (1 << 27) 81 #define VTD_GSR_RTPS (1 << 30) 82 #define VTD_GSR_TES (1U << 31) 83 84 #define VTD_CCR_ICC (1UL << 63) /* invalidate context cache */ 85 #define VTD_CCR_CIRG_GLOBAL (1UL << 61) /* global invalidation */ 86 87 #define VTD_IIR_IVT (1UL << 63) /* invalidation IOTLB */ 88 #define VTD_IIR_IIRG_GLOBAL (1ULL << 60) /* global IOTLB invalidation */ 89 #define VTD_IIR_IIRG_DOMAIN (2ULL << 60) /* domain IOTLB invalidation */ 90 #define VTD_IIR_IIRG_PAGE (3ULL << 60) /* page IOTLB invalidation */ 91 #define VTD_IIR_DRAIN_READS (1ULL << 49) /* drain pending DMA reads */ 92 #define VTD_IIR_DRAIN_WRITES (1ULL << 48) /* drain pending DMA writes */ 93 #define VTD_IIR_DOMAIN_P 32 94 95 #define VTD_ROOT_PRESENT 0x1 96 #define VTD_CTX_PRESENT 0x1 97 #define VTD_CTX_TT_ALL (1UL << 2) 98 99 #define VTD_PTE_RD (1UL << 0) 100 #define VTD_PTE_WR (1UL << 1) 101 #define VTD_PTE_SUPERPAGE (1UL << 7) 102 #define VTD_PTE_ADDR_M (0x000FFFFFFFFFF000UL) 103 104 #define VTD_RID2IDX(rid) (((rid) & 0xff) * 2) 105 106 struct domain { 107 uint64_t *ptp; /* first level page table page */ 108 int pt_levels; /* number of page table levels */ 109 int addrwidth; /* 'AW' field in context entry */ 110 int spsmask; /* supported super page sizes */ 111 u_int id; /* domain id */ 112 vm_paddr_t maxaddr; /* highest address to be mapped */ 113 SLIST_ENTRY(domain) next; 114 }; 115 116 static SLIST_HEAD(, domain) domhead; 117 118 #define DRHD_MAX_UNITS 16 119 static ACPI_DMAR_HARDWARE_UNIT *drhds[DRHD_MAX_UNITS]; 120 static int drhd_num; 121 static struct vtdmap *vtdmaps[DRHD_MAX_UNITS]; 122 static int max_domains; 123 typedef int (*drhd_ident_func_t)(void); 124 125 static uint64_t root_table[PAGE_SIZE / sizeof(uint64_t)] __aligned(4096); 126 static uint64_t ctx_tables[256][PAGE_SIZE / sizeof(uint64_t)] __aligned(4096); 127 128 static MALLOC_DEFINE(M_VTD, "vtd", "vtd"); 129 130 static int 131 vtd_max_domains(struct vtdmap *vtdmap) 132 { 133 int nd; 134 135 nd = VTD_CAP_ND(vtdmap->cap); 136 137 switch (nd) { 138 case 0: 139 return (16); 140 case 1: 141 return (64); 142 case 2: 143 return (256); 144 case 3: 145 return (1024); 146 case 4: 147 return (4 * 1024); 148 case 5: 149 return (16 * 1024); 150 case 6: 151 return (64 * 1024); 152 default: 153 panic("vtd_max_domains: invalid value of nd (0x%0x)", nd); 154 } 155 } 156 157 static u_int 158 domain_id(void) 159 { 160 u_int id; 161 struct domain *dom; 162 163 /* Skip domain id 0 - it is reserved when Caching Mode field is set */ 164 for (id = 1; id < max_domains; id++) { 165 SLIST_FOREACH(dom, &domhead, next) { 166 if (dom->id == id) 167 break; 168 } 169 if (dom == NULL) 170 break; /* found it */ 171 } 172 173 if (id >= max_domains) 174 panic("domain ids exhausted"); 175 176 return (id); 177 } 178 179 static struct vtdmap * 180 vtd_device_scope(uint16_t rid) 181 { 182 int i, remaining, pathremaining; 183 char *end, *pathend; 184 struct vtdmap *vtdmap; 185 ACPI_DMAR_HARDWARE_UNIT *drhd; 186 ACPI_DMAR_DEVICE_SCOPE *device_scope; 187 ACPI_DMAR_PCI_PATH *path; 188 189 for (i = 0; i < drhd_num; i++) { 190 drhd = drhds[i]; 191 192 if (VTD_DRHD_INCLUDE_PCI_ALL(drhd->Flags)) { 193 /* 194 * From Intel VT-d arch spec, version 3.0: 195 * If a DRHD structure with INCLUDE_PCI_ALL flag Set is reported 196 * for a Segment, it must be enumerated by BIOS after all other 197 * DRHD structures for the same Segment. 198 */ 199 vtdmap = vtdmaps[i]; 200 return(vtdmap); 201 } 202 203 end = (char *)drhd + drhd->Header.Length; 204 remaining = drhd->Header.Length - sizeof(ACPI_DMAR_HARDWARE_UNIT); 205 while (remaining > sizeof(ACPI_DMAR_DEVICE_SCOPE)) { 206 device_scope = (ACPI_DMAR_DEVICE_SCOPE *)(end - remaining); 207 remaining -= device_scope->Length; 208 209 switch (device_scope->EntryType){ 210 /* 0x01 and 0x02 are PCI device entries */ 211 case 0x01: 212 case 0x02: 213 break; 214 default: 215 continue; 216 } 217 218 if (PCI_RID2BUS(rid) != device_scope->Bus) 219 continue; 220 221 pathend = (char *)device_scope + device_scope->Length; 222 pathremaining = device_scope->Length - sizeof(ACPI_DMAR_DEVICE_SCOPE); 223 while (pathremaining >= sizeof(ACPI_DMAR_PCI_PATH)) { 224 path = (ACPI_DMAR_PCI_PATH *)(pathend - pathremaining); 225 pathremaining -= sizeof(ACPI_DMAR_PCI_PATH); 226 227 if (PCI_RID2SLOT(rid) != path->Device) 228 continue; 229 if (PCI_RID2FUNC(rid) != path->Function) 230 continue; 231 232 vtdmap = vtdmaps[i]; 233 return (vtdmap); 234 } 235 } 236 } 237 238 /* No matching scope */ 239 return (NULL); 240 } 241 242 static void 243 vtd_wbflush(struct vtdmap *vtdmap) 244 { 245 246 if (VTD_ECAP_COHERENCY(vtdmap->ext_cap) == 0) 247 pmap_invalidate_cache(); 248 249 if (VTD_CAP_RWBF(vtdmap->cap)) { 250 vtdmap->gcr = VTD_GCR_WBF; 251 while ((vtdmap->gsr & VTD_GSR_WBFS) != 0) 252 ; 253 } 254 } 255 256 static void 257 vtd_ctx_global_invalidate(struct vtdmap *vtdmap) 258 { 259 260 vtdmap->ccr = VTD_CCR_ICC | VTD_CCR_CIRG_GLOBAL; 261 while ((vtdmap->ccr & VTD_CCR_ICC) != 0) 262 ; 263 } 264 265 static void 266 vtd_iotlb_global_invalidate(struct vtdmap *vtdmap) 267 { 268 int offset; 269 volatile uint64_t *iotlb_reg, val; 270 271 vtd_wbflush(vtdmap); 272 273 offset = VTD_ECAP_IRO(vtdmap->ext_cap) * 16; 274 iotlb_reg = (volatile uint64_t *)((caddr_t)vtdmap + offset + 8); 275 276 *iotlb_reg = VTD_IIR_IVT | VTD_IIR_IIRG_GLOBAL | 277 VTD_IIR_DRAIN_READS | VTD_IIR_DRAIN_WRITES; 278 279 while (1) { 280 val = *iotlb_reg; 281 if ((val & VTD_IIR_IVT) == 0) 282 break; 283 } 284 } 285 286 static void 287 vtd_translation_enable(struct vtdmap *vtdmap) 288 { 289 290 vtdmap->gcr = VTD_GCR_TE; 291 while ((vtdmap->gsr & VTD_GSR_TES) == 0) 292 ; 293 } 294 295 static void 296 vtd_translation_disable(struct vtdmap *vtdmap) 297 { 298 299 vtdmap->gcr = 0; 300 while ((vtdmap->gsr & VTD_GSR_TES) != 0) 301 ; 302 } 303 304 static int 305 vtd_init(void) 306 { 307 int i, units, remaining, tmp; 308 struct vtdmap *vtdmap; 309 vm_paddr_t ctx_paddr; 310 char *end, envname[32]; 311 unsigned long mapaddr; 312 ACPI_STATUS status; 313 ACPI_TABLE_DMAR *dmar; 314 ACPI_DMAR_HEADER *hdr; 315 ACPI_DMAR_HARDWARE_UNIT *drhd; 316 317 /* 318 * Allow the user to override the ACPI DMAR table by specifying the 319 * physical address of each remapping unit. 320 * 321 * The following example specifies two remapping units at 322 * physical addresses 0xfed90000 and 0xfeda0000 respectively. 323 * set vtd.regmap.0.addr=0xfed90000 324 * set vtd.regmap.1.addr=0xfeda0000 325 */ 326 for (units = 0; units < DRHD_MAX_UNITS; units++) { 327 snprintf(envname, sizeof(envname), "vtd.regmap.%d.addr", units); 328 if (getenv_ulong(envname, &mapaddr) == 0) 329 break; 330 vtdmaps[units] = (struct vtdmap *)PHYS_TO_DMAP(mapaddr); 331 } 332 333 if (units > 0) 334 goto skip_dmar; 335 336 /* Search for DMAR table. */ 337 status = AcpiGetTable(ACPI_SIG_DMAR, 0, (ACPI_TABLE_HEADER **)&dmar); 338 if (ACPI_FAILURE(status)) 339 return (ENXIO); 340 341 end = (char *)dmar + dmar->Header.Length; 342 remaining = dmar->Header.Length - sizeof(ACPI_TABLE_DMAR); 343 while (remaining > sizeof(ACPI_DMAR_HEADER)) { 344 hdr = (ACPI_DMAR_HEADER *)(end - remaining); 345 if (hdr->Length > remaining) 346 break; 347 /* 348 * From Intel VT-d arch spec, version 1.3: 349 * BIOS implementations must report mapping structures 350 * in numerical order, i.e. All remapping structures of 351 * type 0 (DRHD) enumerated before remapping structures of 352 * type 1 (RMRR) and so forth. 353 */ 354 if (hdr->Type != ACPI_DMAR_TYPE_HARDWARE_UNIT) 355 break; 356 357 drhd = (ACPI_DMAR_HARDWARE_UNIT *)hdr; 358 drhds[units] = drhd; 359 vtdmaps[units] = (struct vtdmap *)PHYS_TO_DMAP(drhd->Address); 360 if (++units >= DRHD_MAX_UNITS) 361 break; 362 remaining -= hdr->Length; 363 } 364 365 if (units <= 0) 366 return (ENXIO); 367 368 skip_dmar: 369 drhd_num = units; 370 371 max_domains = 64 * 1024; /* maximum valid value */ 372 for (i = 0; i < drhd_num; i++){ 373 vtdmap = vtdmaps[i]; 374 375 if (VTD_CAP_CM(vtdmap->cap) != 0) 376 panic("vtd_init: invalid caching mode"); 377 378 /* take most compatible (minimum) value */ 379 if ((tmp = vtd_max_domains(vtdmap)) < max_domains) 380 max_domains = tmp; 381 } 382 383 /* 384 * Set up the root-table to point to the context-entry tables 385 */ 386 for (i = 0; i < 256; i++) { 387 ctx_paddr = vtophys(ctx_tables[i]); 388 if (ctx_paddr & PAGE_MASK) 389 panic("ctx table (0x%0lx) not page aligned", ctx_paddr); 390 391 root_table[i * 2] = ctx_paddr | VTD_ROOT_PRESENT; 392 } 393 394 return (0); 395 } 396 397 static void 398 vtd_cleanup(void) 399 { 400 } 401 402 static void 403 vtd_enable(void) 404 { 405 int i; 406 struct vtdmap *vtdmap; 407 408 for (i = 0; i < drhd_num; i++) { 409 vtdmap = vtdmaps[i]; 410 vtd_wbflush(vtdmap); 411 412 /* Update the root table address */ 413 vtdmap->rta = vtophys(root_table); 414 vtdmap->gcr = VTD_GCR_SRTP; 415 while ((vtdmap->gsr & VTD_GSR_RTPS) == 0) 416 ; 417 418 vtd_ctx_global_invalidate(vtdmap); 419 vtd_iotlb_global_invalidate(vtdmap); 420 421 vtd_translation_enable(vtdmap); 422 } 423 } 424 425 static void 426 vtd_disable(void) 427 { 428 int i; 429 struct vtdmap *vtdmap; 430 431 for (i = 0; i < drhd_num; i++) { 432 vtdmap = vtdmaps[i]; 433 vtd_translation_disable(vtdmap); 434 } 435 } 436 437 static void 438 vtd_add_device(void *arg, uint16_t rid) 439 { 440 int idx; 441 uint64_t *ctxp; 442 struct domain *dom = arg; 443 vm_paddr_t pt_paddr; 444 struct vtdmap *vtdmap; 445 uint8_t bus; 446 447 KASSERT(dom != NULL, ("domain is NULL")); 448 449 bus = PCI_RID2BUS(rid); 450 ctxp = ctx_tables[bus]; 451 pt_paddr = vtophys(dom->ptp); 452 idx = VTD_RID2IDX(rid); 453 454 if (ctxp[idx] & VTD_CTX_PRESENT) { 455 panic("vtd_add_device: device %x is already owned by " 456 "domain %d", rid, 457 (uint16_t)(ctxp[idx + 1] >> 8)); 458 } 459 460 if ((vtdmap = vtd_device_scope(rid)) == NULL) 461 panic("vtd_add_device: device %x is not in scope for " 462 "any DMA remapping unit", rid); 463 464 /* 465 * Order is important. The 'present' bit is set only after all fields 466 * of the context pointer are initialized. 467 */ 468 ctxp[idx + 1] = dom->addrwidth | (dom->id << 8); 469 470 if (VTD_ECAP_DI(vtdmap->ext_cap)) 471 ctxp[idx] = VTD_CTX_TT_ALL; 472 else 473 ctxp[idx] = 0; 474 475 ctxp[idx] |= pt_paddr | VTD_CTX_PRESENT; 476 477 /* 478 * 'Not Present' entries are not cached in either the Context Cache 479 * or in the IOTLB, so there is no need to invalidate either of them. 480 */ 481 } 482 483 static void 484 vtd_remove_device(void *arg, uint16_t rid) 485 { 486 int i, idx; 487 uint64_t *ctxp; 488 struct vtdmap *vtdmap; 489 uint8_t bus; 490 491 bus = PCI_RID2BUS(rid); 492 ctxp = ctx_tables[bus]; 493 idx = VTD_RID2IDX(rid); 494 495 /* 496 * Order is important. The 'present' bit is must be cleared first. 497 */ 498 ctxp[idx] = 0; 499 ctxp[idx + 1] = 0; 500 501 /* 502 * Invalidate the Context Cache and the IOTLB. 503 * 504 * XXX use device-selective invalidation for Context Cache 505 * XXX use domain-selective invalidation for IOTLB 506 */ 507 for (i = 0; i < drhd_num; i++) { 508 vtdmap = vtdmaps[i]; 509 vtd_ctx_global_invalidate(vtdmap); 510 vtd_iotlb_global_invalidate(vtdmap); 511 } 512 } 513 514 #define CREATE_MAPPING 0 515 #define REMOVE_MAPPING 1 516 517 static uint64_t 518 vtd_update_mapping(void *arg, vm_paddr_t gpa, vm_paddr_t hpa, uint64_t len, 519 int remove) 520 { 521 struct domain *dom; 522 int i, spshift, ptpshift, ptpindex, nlevels; 523 uint64_t spsize, *ptp; 524 525 dom = arg; 526 ptpindex = 0; 527 ptpshift = 0; 528 529 KASSERT(gpa + len > gpa, ("%s: invalid gpa range %#lx/%#lx", __func__, 530 gpa, len)); 531 KASSERT(gpa + len <= dom->maxaddr, ("%s: gpa range %#lx/%#lx beyond " 532 "domain maxaddr %#lx", __func__, gpa, len, dom->maxaddr)); 533 534 if (gpa & PAGE_MASK) 535 panic("vtd_create_mapping: unaligned gpa 0x%0lx", gpa); 536 537 if (hpa & PAGE_MASK) 538 panic("vtd_create_mapping: unaligned hpa 0x%0lx", hpa); 539 540 if (len & PAGE_MASK) 541 panic("vtd_create_mapping: unaligned len 0x%0lx", len); 542 543 /* 544 * Compute the size of the mapping that we can accommodate. 545 * 546 * This is based on three factors: 547 * - supported super page size 548 * - alignment of the region starting at 'gpa' and 'hpa' 549 * - length of the region 'len' 550 */ 551 spshift = 48; 552 for (i = 3; i >= 0; i--) { 553 spsize = 1UL << spshift; 554 if ((dom->spsmask & (1 << i)) != 0 && 555 (gpa & (spsize - 1)) == 0 && 556 (hpa & (spsize - 1)) == 0 && 557 (len >= spsize)) { 558 break; 559 } 560 spshift -= 9; 561 } 562 563 ptp = dom->ptp; 564 nlevels = dom->pt_levels; 565 while (--nlevels >= 0) { 566 ptpshift = 12 + nlevels * 9; 567 ptpindex = (gpa >> ptpshift) & 0x1FF; 568 569 /* We have reached the leaf mapping */ 570 if (spshift >= ptpshift) { 571 break; 572 } 573 574 /* 575 * We are working on a non-leaf page table page. 576 * 577 * Create a downstream page table page if necessary and point 578 * to it from the current page table. 579 */ 580 if (ptp[ptpindex] == 0) { 581 void *nlp = malloc(PAGE_SIZE, M_VTD, M_WAITOK | M_ZERO); 582 ptp[ptpindex] = vtophys(nlp)| VTD_PTE_RD | VTD_PTE_WR; 583 } 584 585 ptp = (uint64_t *)PHYS_TO_DMAP(ptp[ptpindex] & VTD_PTE_ADDR_M); 586 } 587 588 if ((gpa & ((1UL << ptpshift) - 1)) != 0) 589 panic("gpa 0x%lx and ptpshift %d mismatch", gpa, ptpshift); 590 591 /* 592 * Update the 'gpa' -> 'hpa' mapping 593 */ 594 if (remove) { 595 ptp[ptpindex] = 0; 596 } else { 597 ptp[ptpindex] = hpa | VTD_PTE_RD | VTD_PTE_WR; 598 599 if (nlevels > 0) 600 ptp[ptpindex] |= VTD_PTE_SUPERPAGE; 601 } 602 603 return (1UL << ptpshift); 604 } 605 606 static uint64_t 607 vtd_create_mapping(void *arg, vm_paddr_t gpa, vm_paddr_t hpa, uint64_t len) 608 { 609 610 return (vtd_update_mapping(arg, gpa, hpa, len, CREATE_MAPPING)); 611 } 612 613 static uint64_t 614 vtd_remove_mapping(void *arg, vm_paddr_t gpa, uint64_t len) 615 { 616 617 return (vtd_update_mapping(arg, gpa, 0, len, REMOVE_MAPPING)); 618 } 619 620 static void 621 vtd_invalidate_tlb(void *dom) 622 { 623 int i; 624 struct vtdmap *vtdmap; 625 626 /* 627 * Invalidate the IOTLB. 628 * XXX use domain-selective invalidation for IOTLB 629 */ 630 for (i = 0; i < drhd_num; i++) { 631 vtdmap = vtdmaps[i]; 632 vtd_iotlb_global_invalidate(vtdmap); 633 } 634 } 635 636 static void * 637 vtd_create_domain(vm_paddr_t maxaddr) 638 { 639 struct domain *dom; 640 vm_paddr_t addr; 641 int tmp, i, gaw, agaw, sagaw, res, pt_levels, addrwidth; 642 struct vtdmap *vtdmap; 643 644 if (drhd_num <= 0) 645 panic("vtd_create_domain: no dma remapping hardware available"); 646 647 /* 648 * Calculate AGAW. 649 * Section 3.4.2 "Adjusted Guest Address Width", Architecture Spec. 650 */ 651 addr = 0; 652 for (gaw = 0; addr < maxaddr; gaw++) 653 addr = 1ULL << gaw; 654 655 res = (gaw - 12) % 9; 656 if (res == 0) 657 agaw = gaw; 658 else 659 agaw = gaw + 9 - res; 660 661 if (agaw > 64) 662 agaw = 64; 663 664 /* 665 * Select the smallest Supported AGAW and the corresponding number 666 * of page table levels. 667 */ 668 pt_levels = 2; 669 sagaw = 30; 670 addrwidth = 0; 671 672 tmp = ~0; 673 for (i = 0; i < drhd_num; i++) { 674 vtdmap = vtdmaps[i]; 675 /* take most compatible value */ 676 tmp &= VTD_CAP_SAGAW(vtdmap->cap); 677 } 678 679 for (i = 0; i < 5; i++) { 680 if ((tmp & (1 << i)) != 0 && sagaw >= agaw) 681 break; 682 pt_levels++; 683 addrwidth++; 684 sagaw += 9; 685 if (sagaw > 64) 686 sagaw = 64; 687 } 688 689 if (i >= 5) { 690 panic("vtd_create_domain: SAGAW 0x%x does not support AGAW %d", 691 tmp, agaw); 692 } 693 694 dom = malloc(sizeof(struct domain), M_VTD, M_ZERO | M_WAITOK); 695 dom->pt_levels = pt_levels; 696 dom->addrwidth = addrwidth; 697 dom->id = domain_id(); 698 dom->maxaddr = maxaddr; 699 dom->ptp = malloc(PAGE_SIZE, M_VTD, M_ZERO | M_WAITOK); 700 if ((uintptr_t)dom->ptp & PAGE_MASK) 701 panic("vtd_create_domain: ptp (%p) not page aligned", dom->ptp); 702 703 #ifdef notyet 704 /* 705 * XXX superpage mappings for the iommu do not work correctly. 706 * 707 * By default all physical memory is mapped into the host_domain. 708 * When a VM is allocated wired memory the pages belonging to it 709 * are removed from the host_domain and added to the vm's domain. 710 * 711 * If the page being removed was mapped using a superpage mapping 712 * in the host_domain then we need to demote the mapping before 713 * removing the page. 714 * 715 * There is not any code to deal with the demotion at the moment 716 * so we disable superpage mappings altogether. 717 */ 718 dom->spsmask = ~0; 719 for (i = 0; i < drhd_num; i++) { 720 vtdmap = vtdmaps[i]; 721 /* take most compatible value */ 722 dom->spsmask &= VTD_CAP_SPS(vtdmap->cap); 723 } 724 #endif 725 726 SLIST_INSERT_HEAD(&domhead, dom, next); 727 728 return (dom); 729 } 730 731 static void 732 vtd_free_ptp(uint64_t *ptp, int level) 733 { 734 int i; 735 uint64_t *nlp; 736 737 if (level > 1) { 738 for (i = 0; i < 512; i++) { 739 if ((ptp[i] & (VTD_PTE_RD | VTD_PTE_WR)) == 0) 740 continue; 741 if ((ptp[i] & VTD_PTE_SUPERPAGE) != 0) 742 continue; 743 nlp = (uint64_t *)PHYS_TO_DMAP(ptp[i] & VTD_PTE_ADDR_M); 744 vtd_free_ptp(nlp, level - 1); 745 } 746 } 747 748 bzero(ptp, PAGE_SIZE); 749 free(ptp, M_VTD); 750 } 751 752 static void 753 vtd_destroy_domain(void *arg) 754 { 755 struct domain *dom; 756 757 dom = arg; 758 759 SLIST_REMOVE(&domhead, dom, domain, next); 760 vtd_free_ptp(dom->ptp, dom->pt_levels); 761 free(dom, M_VTD); 762 } 763 764 const struct iommu_ops iommu_ops_intel = { 765 .init = vtd_init, 766 .cleanup = vtd_cleanup, 767 .enable = vtd_enable, 768 .disable = vtd_disable, 769 .create_domain = vtd_create_domain, 770 .destroy_domain = vtd_destroy_domain, 771 .create_mapping = vtd_create_mapping, 772 .remove_mapping = vtd_remove_mapping, 773 .add_device = vtd_add_device, 774 .remove_device = vtd_remove_device, 775 .invalidate_tlb = vtd_invalidate_tlb, 776 }; 777