1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2019-2020 Ruslan Bukin <br@bsdpad.com> 5 * 6 * This software was developed by SRI International and the University of 7 * Cambridge Computer Laboratory (Department of Computer Science and 8 * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the 9 * DARPA SSITH research programme. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 */ 32 33 /* 34 * Hardware overview. 35 * 36 * An incoming transaction from a peripheral device has an address, size, 37 * attributes and StreamID. 38 * 39 * In case of PCI-based devices, StreamID is a PCI rid. 40 * 41 * The StreamID is used to select a Stream Table Entry (STE) in a Stream table, 42 * which contains per-device configuration. 43 * 44 * Stream table is a linear or 2-level walk table (this driver supports both). 45 * Note that a linear table could occupy 1GB or more of memory depending on 46 * sid_bits value. 47 * 48 * STE is used to locate a Context Descriptor, which is a struct in memory 49 * that describes stages of translation, translation table type, pointer to 50 * level 0 of page tables, ASID, etc. 51 * 52 * Hardware supports two stages of translation: Stage1 (S1) and Stage2 (S2): 53 * o S1 is used for the host machine traffic translation 54 * o S2 is for a hypervisor 55 * 56 * This driver enables S1 stage with standard AArch64 page tables. 57 * 58 * Note that SMMU does not share TLB with a main CPU. 59 * Command queue is used by this driver to Invalidate SMMU TLB, STE cache. 60 * 61 * An arm64 SoC could have more than one SMMU instance. 62 * ACPI IORT table describes which SMMU unit is assigned for a particular 63 * peripheral device. 64 * 65 * Queues. 66 * 67 * Register interface and Memory-based circular buffer queues are used 68 * to inferface SMMU. 69 * 70 * These are a Command queue for commands to send to the SMMU and an Event 71 * queue for event/fault reports from the SMMU. Optionally PRI queue is 72 * designed for PCIe page requests reception. 73 * 74 * Note that not every hardware supports PRI services. For instance they were 75 * not found in Neoverse N1 SDP machine. 76 * (This drivers does not implement PRI queue.) 77 * 78 * All SMMU queues are arranged as circular buffers in memory. They are used 79 * in a producer-consumer fashion so that an output queue contains data 80 * produced by the SMMU and consumed by software. 81 * An input queue contains data produced by software, consumed by the SMMU. 82 * 83 * Interrupts. 84 * 85 * Interrupts are not required by this driver for normal operation. 86 * The standard wired interrupt is only triggered when an event comes from 87 * the SMMU, which is only in a case of errors (e.g. translation fault). 88 */ 89 90 #include "opt_platform.h" 91 #include "opt_acpi.h" 92 93 #include <sys/cdefs.h> 94 __FBSDID("$FreeBSD$"); 95 96 #include <sys/param.h> 97 #include <sys/bitstring.h> 98 #include <sys/bus.h> 99 #include <sys/kernel.h> 100 #include <sys/malloc.h> 101 #include <sys/rman.h> 102 #include <sys/lock.h> 103 #include <sys/tree.h> 104 #include <sys/taskqueue.h> 105 #include <vm/vm.h> 106 #include <vm/vm_page.h> 107 #if DEV_ACPI 108 #include <contrib/dev/acpica/include/acpi.h> 109 #include <dev/acpica/acpivar.h> 110 #endif 111 #include <dev/pci/pcireg.h> 112 #include <dev/pci/pcivar.h> 113 #include <dev/iommu/iommu.h> 114 #include <arm64/iommu/iommu_pmap.h> 115 116 #include "iommu.h" 117 #include "iommu_if.h" 118 119 #include "smmureg.h" 120 #include "smmuvar.h" 121 122 #define STRTAB_L1_SZ_SHIFT 20 123 #define STRTAB_SPLIT 8 124 125 #define STRTAB_L1_DESC_L2PTR_M (0x3fffffffffff << 6) 126 #define STRTAB_L1_DESC_DWORDS 1 127 128 #define STRTAB_STE_DWORDS 8 129 130 #define CMDQ_ENTRY_DWORDS 2 131 #define EVTQ_ENTRY_DWORDS 4 132 #define PRIQ_ENTRY_DWORDS 2 133 134 #define CD_DWORDS 8 135 136 #define Q_WRP(q, p) ((p) & (1 << (q)->size_log2)) 137 #define Q_IDX(q, p) ((p) & ((1 << (q)->size_log2) - 1)) 138 #define Q_OVF(p) ((p) & (1 << 31)) /* Event queue overflowed */ 139 140 #define SMMU_Q_ALIGN (64 * 1024) 141 142 static struct resource_spec smmu_spec[] = { 143 { SYS_RES_MEMORY, 0, RF_ACTIVE }, 144 { SYS_RES_IRQ, 0, RF_ACTIVE }, 145 { SYS_RES_IRQ, 1, RF_ACTIVE }, 146 { SYS_RES_IRQ, 2, RF_ACTIVE }, 147 RESOURCE_SPEC_END 148 }; 149 150 MALLOC_DEFINE(M_SMMU, "SMMU", SMMU_DEVSTR); 151 152 #define dprintf(fmt, ...) 153 154 struct smmu_event { 155 int ident; 156 char *str; 157 char *msg; 158 }; 159 160 static struct smmu_event events[] = { 161 { 0x01, "F_UUT", 162 "Unsupported Upstream Transaction."}, 163 { 0x02, "C_BAD_STREAMID", 164 "Transaction StreamID out of range."}, 165 { 0x03, "F_STE_FETCH", 166 "Fetch of STE caused external abort."}, 167 { 0x04, "C_BAD_STE", 168 "Used STE invalid."}, 169 { 0x05, "F_BAD_ATS_TREQ", 170 "Address Translation Request disallowed for a StreamID " 171 "and a PCIe ATS Translation Request received."}, 172 { 0x06, "F_STREAM_DISABLED", 173 "The STE of a transaction marks non-substream transactions " 174 "disabled."}, 175 { 0x07, "F_TRANSL_FORBIDDEN", 176 "An incoming PCIe transaction is marked Translated but " 177 "SMMU bypass is disallowed for this StreamID."}, 178 { 0x08, "C_BAD_SUBSTREAMID", 179 "Incoming SubstreamID present, but configuration is invalid."}, 180 { 0x09, "F_CD_FETCH", 181 "Fetch of CD caused external abort."}, 182 { 0x0a, "C_BAD_CD", 183 "Fetched CD invalid."}, 184 { 0x0b, "F_WALK_EABT", 185 "An external abort occurred fetching (or updating) " 186 "a translation table descriptor."}, 187 { 0x10, "F_TRANSLATION", 188 "Translation fault."}, 189 { 0x11, "F_ADDR_SIZE", 190 "Address Size fault."}, 191 { 0x12, "F_ACCESS", 192 "Access flag fault due to AF == 0 in a page or block TTD."}, 193 { 0x13, "F_PERMISSION", 194 "Permission fault occurred on page access."}, 195 { 0x20, "F_TLB_CONFLICT", 196 "A TLB conflict occurred because of the transaction."}, 197 { 0x21, "F_CFG_CONFLICT", 198 "A configuration cache conflict occurred due to " 199 "the transaction."}, 200 { 0x24, "E_PAGE_REQUEST", 201 "Speculative page request hint."}, 202 { 0x25, "F_VMS_FETCH", 203 "Fetch of VMS caused external abort."}, 204 { 0, NULL, NULL }, 205 }; 206 207 static int 208 smmu_q_has_space(struct smmu_queue *q) 209 { 210 211 /* 212 * See 6.3.27 SMMU_CMDQ_PROD 213 * 214 * There is space in the queue for additional commands if: 215 * SMMU_CMDQ_CONS.RD != SMMU_CMDQ_PROD.WR || 216 * SMMU_CMDQ_CONS.RD_WRAP == SMMU_CMDQ_PROD.WR_WRAP 217 */ 218 219 if (Q_IDX(q, q->lc.cons) != Q_IDX(q, q->lc.prod) || 220 Q_WRP(q, q->lc.cons) == Q_WRP(q, q->lc.prod)) 221 return (1); 222 223 return (0); 224 } 225 226 static int 227 smmu_q_empty(struct smmu_queue *q) 228 { 229 230 if (Q_IDX(q, q->lc.cons) == Q_IDX(q, q->lc.prod) && 231 Q_WRP(q, q->lc.cons) == Q_WRP(q, q->lc.prod)) 232 return (1); 233 234 return (0); 235 } 236 237 static int __unused 238 smmu_q_consumed(struct smmu_queue *q, uint32_t prod) 239 { 240 241 if ((Q_WRP(q, q->lc.cons) == Q_WRP(q, prod)) && 242 (Q_IDX(q, q->lc.cons) >= Q_IDX(q, prod))) 243 return (1); 244 245 if ((Q_WRP(q, q->lc.cons) != Q_WRP(q, prod)) && 246 (Q_IDX(q, q->lc.cons) <= Q_IDX(q, prod))) 247 return (1); 248 249 return (0); 250 } 251 252 static uint32_t 253 smmu_q_inc_cons(struct smmu_queue *q) 254 { 255 uint32_t cons; 256 uint32_t val; 257 258 cons = (Q_WRP(q, q->lc.cons) | Q_IDX(q, q->lc.cons)) + 1; 259 val = (Q_OVF(q->lc.cons) | Q_WRP(q, cons) | Q_IDX(q, cons)); 260 261 return (val); 262 } 263 264 static uint32_t 265 smmu_q_inc_prod(struct smmu_queue *q) 266 { 267 uint32_t prod; 268 uint32_t val; 269 270 prod = (Q_WRP(q, q->lc.prod) | Q_IDX(q, q->lc.prod)) + 1; 271 val = (Q_OVF(q->lc.prod) | Q_WRP(q, prod) | Q_IDX(q, prod)); 272 273 return (val); 274 } 275 276 static int 277 smmu_write_ack(struct smmu_softc *sc, uint32_t reg, 278 uint32_t reg_ack, uint32_t val) 279 { 280 uint32_t v; 281 int timeout; 282 283 timeout = 100000; 284 285 bus_write_4(sc->res[0], reg, val); 286 287 do { 288 v = bus_read_4(sc->res[0], reg_ack); 289 if (v == val) 290 break; 291 } while (timeout--); 292 293 if (timeout <= 0) { 294 device_printf(sc->dev, "Failed to write reg.\n"); 295 return (-1); 296 } 297 298 return (0); 299 } 300 301 static inline int 302 ilog2(long x) 303 { 304 305 KASSERT(x > 0 && powerof2(x), ("%s: invalid arg %ld", __func__, x)); 306 307 return (flsl(x) - 1); 308 } 309 310 static int 311 smmu_init_queue(struct smmu_softc *sc, struct smmu_queue *q, 312 uint32_t prod_off, uint32_t cons_off, uint32_t dwords) 313 { 314 int sz; 315 316 sz = (1 << q->size_log2) * dwords * 8; 317 318 /* Set up the command circular buffer */ 319 q->vaddr = contigmalloc(sz, M_SMMU, 320 M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, SMMU_Q_ALIGN, 0); 321 if (q->vaddr == NULL) { 322 device_printf(sc->dev, "failed to allocate %d bytes\n", sz); 323 return (-1); 324 } 325 326 q->prod_off = prod_off; 327 q->cons_off = cons_off; 328 q->paddr = vtophys(q->vaddr); 329 330 q->base = CMDQ_BASE_RA | EVENTQ_BASE_WA | PRIQ_BASE_WA; 331 q->base |= q->paddr & Q_BASE_ADDR_M; 332 q->base |= q->size_log2 << Q_LOG2SIZE_S; 333 334 return (0); 335 } 336 337 static int 338 smmu_init_queues(struct smmu_softc *sc) 339 { 340 int err; 341 342 /* Command queue. */ 343 err = smmu_init_queue(sc, &sc->cmdq, 344 SMMU_CMDQ_PROD, SMMU_CMDQ_CONS, CMDQ_ENTRY_DWORDS); 345 if (err) 346 return (ENXIO); 347 348 /* Event queue. */ 349 err = smmu_init_queue(sc, &sc->evtq, 350 SMMU_EVENTQ_PROD, SMMU_EVENTQ_CONS, EVTQ_ENTRY_DWORDS); 351 if (err) 352 return (ENXIO); 353 354 if (!(sc->features & SMMU_FEATURE_PRI)) 355 return (0); 356 357 /* PRI queue. */ 358 err = smmu_init_queue(sc, &sc->priq, 359 SMMU_PRIQ_PROD, SMMU_PRIQ_CONS, PRIQ_ENTRY_DWORDS); 360 if (err) 361 return (ENXIO); 362 363 return (0); 364 } 365 366 /* 367 * Dump 2LVL or linear STE. 368 */ 369 static void 370 smmu_dump_ste(struct smmu_softc *sc, int sid) 371 { 372 struct smmu_strtab *strtab; 373 struct l1_desc *l1_desc; 374 uint64_t *ste, *l1; 375 int i; 376 377 strtab = &sc->strtab; 378 379 if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) { 380 i = sid >> STRTAB_SPLIT; 381 l1 = (void *)((uint64_t)strtab->vaddr + 382 STRTAB_L1_DESC_DWORDS * 8 * i); 383 device_printf(sc->dev, "L1 ste == %lx\n", l1[0]); 384 385 l1_desc = &strtab->l1[i]; 386 ste = l1_desc->va; 387 if (ste == NULL) /* L2 is not initialized */ 388 return; 389 } else { 390 ste = (void *)((uint64_t)strtab->vaddr + 391 sid * (STRTAB_STE_DWORDS << 3)); 392 } 393 394 /* Dump L2 or linear STE. */ 395 for (i = 0; i < STRTAB_STE_DWORDS; i++) 396 device_printf(sc->dev, "ste[%d] == %lx\n", i, ste[i]); 397 } 398 399 static void __unused 400 smmu_dump_cd(struct smmu_softc *sc, struct smmu_cd *cd) 401 { 402 uint64_t *vaddr; 403 int i; 404 405 device_printf(sc->dev, "%s\n", __func__); 406 407 vaddr = cd->vaddr; 408 for (i = 0; i < CD_DWORDS; i++) 409 device_printf(sc->dev, "cd[%d] == %lx\n", i, vaddr[i]); 410 } 411 412 static void 413 smmu_evtq_dequeue(struct smmu_softc *sc, uint32_t *evt) 414 { 415 struct smmu_queue *evtq; 416 void *entry_addr; 417 418 evtq = &sc->evtq; 419 420 evtq->lc.val = bus_read_8(sc->res[0], evtq->prod_off); 421 entry_addr = (void *)((uint64_t)evtq->vaddr + 422 evtq->lc.cons * EVTQ_ENTRY_DWORDS * 8); 423 memcpy(evt, entry_addr, EVTQ_ENTRY_DWORDS * 8); 424 evtq->lc.cons = smmu_q_inc_cons(evtq); 425 bus_write_4(sc->res[0], evtq->cons_off, evtq->lc.cons); 426 } 427 428 static void 429 smmu_print_event(struct smmu_softc *sc, uint32_t *evt) 430 { 431 struct smmu_event *ev; 432 uintptr_t input_addr; 433 uint8_t event_id; 434 device_t dev; 435 int sid; 436 int i; 437 438 dev = sc->dev; 439 440 ev = NULL; 441 event_id = evt[0] & 0xff; 442 for (i = 0; events[i].ident != 0; i++) { 443 if (events[i].ident == event_id) { 444 ev = &events[i]; 445 break; 446 } 447 } 448 449 sid = evt[1]; 450 input_addr = evt[5]; 451 input_addr <<= 32; 452 input_addr |= evt[4]; 453 454 if (smmu_quirks_check(dev, sid, event_id, input_addr)) { 455 /* The event is known. Don't print anything. */ 456 return; 457 } 458 459 if (ev) { 460 device_printf(sc->dev, 461 "Event %s (%s) received.\n", ev->str, ev->msg); 462 } else 463 device_printf(sc->dev, "Event 0x%x received\n", event_id); 464 465 device_printf(sc->dev, "SID %x, Input Address: %jx\n", 466 sid, input_addr); 467 468 for (i = 0; i < 8; i++) 469 device_printf(sc->dev, "evt[%d] %x\n", i, evt[i]); 470 471 smmu_dump_ste(sc, sid); 472 } 473 474 static void 475 make_cmd(struct smmu_softc *sc, uint64_t *cmd, 476 struct smmu_cmdq_entry *entry) 477 { 478 479 memset(cmd, 0, CMDQ_ENTRY_DWORDS * 8); 480 cmd[0] = entry->opcode << CMD_QUEUE_OPCODE_S; 481 482 switch (entry->opcode) { 483 case CMD_TLBI_NH_VA: 484 cmd[0] |= (uint64_t)entry->tlbi.asid << TLBI_0_ASID_S; 485 cmd[1] = entry->tlbi.addr & TLBI_1_ADDR_M; 486 if (entry->tlbi.leaf) { 487 /* 488 * Leaf flag means that only cached entries 489 * for the last level of translation table walk 490 * are required to be invalidated. 491 */ 492 cmd[1] |= TLBI_1_LEAF; 493 } 494 break; 495 case CMD_TLBI_NH_ASID: 496 cmd[0] |= (uint64_t)entry->tlbi.asid << TLBI_0_ASID_S; 497 break; 498 case CMD_TLBI_NSNH_ALL: 499 case CMD_TLBI_NH_ALL: 500 case CMD_TLBI_EL2_ALL: 501 break; 502 case CMD_CFGI_CD: 503 cmd[0] |= ((uint64_t)entry->cfgi.ssid << CFGI_0_SSID_S); 504 /* FALLTROUGH */ 505 case CMD_CFGI_STE: 506 cmd[0] |= ((uint64_t)entry->cfgi.sid << CFGI_0_STE_SID_S); 507 cmd[1] |= ((uint64_t)entry->cfgi.leaf << CFGI_1_LEAF_S); 508 break; 509 case CMD_CFGI_STE_RANGE: 510 cmd[1] = (31 << CFGI_1_STE_RANGE_S); 511 break; 512 case CMD_SYNC: 513 cmd[0] |= SYNC_0_MSH_IS | SYNC_0_MSIATTR_OIWB; 514 if (entry->sync.msiaddr) { 515 cmd[0] |= SYNC_0_CS_SIG_IRQ; 516 cmd[1] |= (entry->sync.msiaddr & SYNC_1_MSIADDRESS_M); 517 } else 518 cmd[0] |= SYNC_0_CS_SIG_SEV; 519 break; 520 case CMD_PREFETCH_CONFIG: 521 cmd[0] |= ((uint64_t)entry->prefetch.sid << PREFETCH_0_SID_S); 522 break; 523 }; 524 } 525 526 static void 527 smmu_cmdq_enqueue_cmd(struct smmu_softc *sc, struct smmu_cmdq_entry *entry) 528 { 529 uint64_t cmd[CMDQ_ENTRY_DWORDS]; 530 struct smmu_queue *cmdq; 531 void *entry_addr; 532 533 cmdq = &sc->cmdq; 534 535 make_cmd(sc, cmd, entry); 536 537 SMMU_LOCK(sc); 538 539 /* Ensure that a space is available. */ 540 do { 541 cmdq->lc.cons = bus_read_4(sc->res[0], cmdq->cons_off); 542 } while (smmu_q_has_space(cmdq) == 0); 543 544 /* Write the command to the current prod entry. */ 545 entry_addr = (void *)((uint64_t)cmdq->vaddr + 546 Q_IDX(cmdq, cmdq->lc.prod) * CMDQ_ENTRY_DWORDS * 8); 547 memcpy(entry_addr, cmd, CMDQ_ENTRY_DWORDS * 8); 548 549 /* Increment prod index. */ 550 cmdq->lc.prod = smmu_q_inc_prod(cmdq); 551 bus_write_4(sc->res[0], cmdq->prod_off, cmdq->lc.prod); 552 553 SMMU_UNLOCK(sc); 554 } 555 556 static void __unused 557 smmu_poll_until_consumed(struct smmu_softc *sc, struct smmu_queue *q) 558 { 559 560 while (1) { 561 q->lc.val = bus_read_8(sc->res[0], q->prod_off); 562 if (smmu_q_empty(q)) 563 break; 564 cpu_spinwait(); 565 } 566 } 567 568 static int 569 smmu_sync(struct smmu_softc *sc) 570 { 571 struct smmu_cmdq_entry cmd; 572 struct smmu_queue *q; 573 uint32_t *base; 574 int timeout; 575 int prod; 576 577 q = &sc->cmdq; 578 prod = q->lc.prod; 579 580 /* Enqueue sync command. */ 581 cmd.opcode = CMD_SYNC; 582 cmd.sync.msiaddr = q->paddr + Q_IDX(q, prod) * CMDQ_ENTRY_DWORDS * 8; 583 smmu_cmdq_enqueue_cmd(sc, &cmd); 584 585 /* Wait for the sync completion. */ 586 base = (void *)((uint64_t)q->vaddr + 587 Q_IDX(q, prod) * CMDQ_ENTRY_DWORDS * 8); 588 589 /* 590 * It takes around 200 loops (6 instructions each) 591 * on Neoverse N1 to complete the sync. 592 */ 593 timeout = 10000; 594 595 do { 596 if (*base == 0) { 597 /* MSI write completed. */ 598 break; 599 } 600 cpu_spinwait(); 601 } while (timeout--); 602 603 if (timeout < 0) 604 device_printf(sc->dev, "Failed to sync\n"); 605 606 return (0); 607 } 608 609 static int 610 smmu_sync_cd(struct smmu_softc *sc, int sid, int ssid, bool leaf) 611 { 612 struct smmu_cmdq_entry cmd; 613 614 cmd.opcode = CMD_CFGI_CD; 615 cmd.cfgi.sid = sid; 616 cmd.cfgi.ssid = ssid; 617 cmd.cfgi.leaf = leaf; 618 smmu_cmdq_enqueue_cmd(sc, &cmd); 619 620 return (0); 621 } 622 623 static void 624 smmu_invalidate_all_sid(struct smmu_softc *sc) 625 { 626 struct smmu_cmdq_entry cmd; 627 628 /* Invalidate cached config */ 629 cmd.opcode = CMD_CFGI_STE_RANGE; 630 smmu_cmdq_enqueue_cmd(sc, &cmd); 631 smmu_sync(sc); 632 } 633 634 static void 635 smmu_tlbi_all(struct smmu_softc *sc) 636 { 637 struct smmu_cmdq_entry cmd; 638 639 /* Invalidate entire TLB */ 640 cmd.opcode = CMD_TLBI_NSNH_ALL; 641 smmu_cmdq_enqueue_cmd(sc, &cmd); 642 smmu_sync(sc); 643 } 644 645 static void 646 smmu_tlbi_asid(struct smmu_softc *sc, uint16_t asid) 647 { 648 struct smmu_cmdq_entry cmd; 649 650 /* Invalidate TLB for an ASID. */ 651 cmd.opcode = CMD_TLBI_NH_ASID; 652 cmd.tlbi.asid = asid; 653 smmu_cmdq_enqueue_cmd(sc, &cmd); 654 smmu_sync(sc); 655 } 656 657 static void 658 smmu_tlbi_va(struct smmu_softc *sc, vm_offset_t va, uint16_t asid) 659 { 660 struct smmu_cmdq_entry cmd; 661 662 /* Invalidate specific range */ 663 cmd.opcode = CMD_TLBI_NH_VA; 664 cmd.tlbi.asid = asid; 665 cmd.tlbi.vmid = 0; 666 cmd.tlbi.leaf = true; /* We change only L3. */ 667 cmd.tlbi.addr = va; 668 smmu_cmdq_enqueue_cmd(sc, &cmd); 669 } 670 671 static void 672 smmu_invalidate_sid(struct smmu_softc *sc, uint32_t sid) 673 { 674 struct smmu_cmdq_entry cmd; 675 676 /* Invalidate cached config */ 677 cmd.opcode = CMD_CFGI_STE; 678 cmd.cfgi.sid = sid; 679 smmu_cmdq_enqueue_cmd(sc, &cmd); 680 smmu_sync(sc); 681 } 682 683 static void 684 smmu_prefetch_sid(struct smmu_softc *sc, uint32_t sid) 685 { 686 struct smmu_cmdq_entry cmd; 687 688 cmd.opcode = CMD_PREFETCH_CONFIG; 689 cmd.prefetch.sid = sid; 690 smmu_cmdq_enqueue_cmd(sc, &cmd); 691 smmu_sync(sc); 692 } 693 694 /* 695 * Init STE in bypass mode. Traffic is not translated for the sid. 696 */ 697 static void 698 smmu_init_ste_bypass(struct smmu_softc *sc, uint32_t sid, uint64_t *ste) 699 { 700 uint64_t val; 701 702 val = STE0_VALID | STE0_CONFIG_BYPASS; 703 704 ste[1] = STE1_SHCFG_INCOMING | STE1_EATS_FULLATS; 705 ste[2] = 0; 706 ste[3] = 0; 707 ste[4] = 0; 708 ste[5] = 0; 709 ste[6] = 0; 710 ste[7] = 0; 711 712 smmu_invalidate_sid(sc, sid); 713 ste[0] = val; 714 dsb(sy); 715 smmu_invalidate_sid(sc, sid); 716 717 smmu_prefetch_sid(sc, sid); 718 } 719 720 /* 721 * Enable Stage1 (S1) translation for the sid. 722 */ 723 static int 724 smmu_init_ste_s1(struct smmu_softc *sc, struct smmu_cd *cd, 725 uint32_t sid, uint64_t *ste) 726 { 727 uint64_t val; 728 729 val = STE0_VALID; 730 731 /* S1 */ 732 ste[1] = STE1_EATS_FULLATS | 733 STE1_S1CSH_IS | 734 STE1_S1CIR_WBRA | 735 STE1_S1COR_WBRA | 736 STE1_STRW_NS_EL1; 737 ste[2] = 0; 738 ste[3] = 0; 739 ste[4] = 0; 740 ste[5] = 0; 741 ste[6] = 0; 742 ste[7] = 0; 743 744 if (sc->features & SMMU_FEATURE_STALL && 745 ((sc->features & SMMU_FEATURE_STALL_FORCE) == 0)) 746 ste[1] |= STE1_S1STALLD; 747 748 /* Configure STE */ 749 val |= (cd->paddr & STE0_S1CONTEXTPTR_M); 750 val |= STE0_CONFIG_S1_TRANS; 751 752 smmu_invalidate_sid(sc, sid); 753 754 /* The STE[0] has to be written in a single blast, last of all. */ 755 ste[0] = val; 756 dsb(sy); 757 758 smmu_invalidate_sid(sc, sid); 759 smmu_sync_cd(sc, sid, 0, true); 760 smmu_invalidate_sid(sc, sid); 761 762 /* The sid will be used soon most likely. */ 763 smmu_prefetch_sid(sc, sid); 764 765 return (0); 766 } 767 768 static int 769 smmu_init_ste(struct smmu_softc *sc, struct smmu_cd *cd, int sid, bool bypass) 770 { 771 struct smmu_strtab *strtab; 772 struct l1_desc *l1_desc; 773 uint64_t *addr; 774 775 strtab = &sc->strtab; 776 777 if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) { 778 l1_desc = &strtab->l1[sid >> STRTAB_SPLIT]; 779 addr = l1_desc->va; 780 addr += (sid & ((1 << STRTAB_SPLIT) - 1)) * STRTAB_STE_DWORDS; 781 } else { 782 addr = (void *)((uint64_t)strtab->vaddr + 783 STRTAB_STE_DWORDS * 8 * sid); 784 }; 785 786 if (bypass) 787 smmu_init_ste_bypass(sc, sid, addr); 788 else 789 smmu_init_ste_s1(sc, cd, sid, addr); 790 791 smmu_sync(sc); 792 793 return (0); 794 } 795 796 static int 797 smmu_init_cd(struct smmu_softc *sc, struct smmu_domain *domain) 798 { 799 vm_paddr_t paddr; 800 uint64_t *ptr; 801 uint64_t val; 802 vm_size_t size; 803 struct smmu_cd *cd; 804 pmap_t p; 805 806 size = 1 * (CD_DWORDS << 3); 807 808 p = &domain->p; 809 cd = domain->cd = malloc(sizeof(struct smmu_cd), 810 M_SMMU, M_WAITOK | M_ZERO); 811 812 cd->vaddr = contigmalloc(size, M_SMMU, 813 M_WAITOK | M_ZERO, /* flags */ 814 0, /* low */ 815 (1ul << 40) - 1, /* high */ 816 size, /* alignment */ 817 0); /* boundary */ 818 if (cd->vaddr == NULL) { 819 device_printf(sc->dev, "Failed to allocate CD\n"); 820 return (ENXIO); 821 } 822 823 cd->size = size; 824 cd->paddr = vtophys(cd->vaddr); 825 826 ptr = cd->vaddr; 827 828 val = CD0_VALID; 829 val |= CD0_AA64; 830 val |= CD0_R; 831 val |= CD0_A; 832 val |= CD0_ASET; 833 val |= (uint64_t)domain->asid << CD0_ASID_S; 834 val |= CD0_TG0_4KB; 835 val |= CD0_EPD1; /* Disable TT1 */ 836 val |= ((64 - sc->ias) << CD0_T0SZ_S); 837 val |= CD0_IPS_48BITS; 838 839 paddr = p->pm_l0_paddr & CD1_TTB0_M; 840 KASSERT(paddr == p->pm_l0_paddr, ("bad allocation 1")); 841 842 ptr[1] = paddr; 843 ptr[2] = 0; 844 ptr[3] = MAIR_ATTR(MAIR_DEVICE_nGnRnE, VM_MEMATTR_DEVICE) | 845 MAIR_ATTR(MAIR_NORMAL_NC, VM_MEMATTR_UNCACHEABLE) | 846 MAIR_ATTR(MAIR_NORMAL_WB, VM_MEMATTR_WRITE_BACK) | 847 MAIR_ATTR(MAIR_NORMAL_WT, VM_MEMATTR_WRITE_THROUGH); 848 849 /* Install the CD. */ 850 ptr[0] = val; 851 852 return (0); 853 } 854 855 static int 856 smmu_init_strtab_linear(struct smmu_softc *sc) 857 { 858 struct smmu_strtab *strtab; 859 vm_paddr_t base; 860 uint32_t size; 861 uint64_t reg; 862 863 strtab = &sc->strtab; 864 strtab->num_l1_entries = (1 << sc->sid_bits); 865 866 size = strtab->num_l1_entries * (STRTAB_STE_DWORDS << 3); 867 868 if (bootverbose) 869 device_printf(sc->dev, 870 "%s: linear strtab size %d, num_l1_entries %d\n", 871 __func__, size, strtab->num_l1_entries); 872 873 strtab->vaddr = contigmalloc(size, M_SMMU, 874 M_WAITOK | M_ZERO, /* flags */ 875 0, /* low */ 876 (1ul << 48) - 1, /* high */ 877 size, /* alignment */ 878 0); /* boundary */ 879 if (strtab->vaddr == NULL) { 880 device_printf(sc->dev, "failed to allocate strtab\n"); 881 return (ENXIO); 882 } 883 884 reg = STRTAB_BASE_CFG_FMT_LINEAR; 885 reg |= sc->sid_bits << STRTAB_BASE_CFG_LOG2SIZE_S; 886 strtab->base_cfg = (uint32_t)reg; 887 888 base = vtophys(strtab->vaddr); 889 890 reg = base & STRTAB_BASE_ADDR_M; 891 KASSERT(reg == base, ("bad allocation 2")); 892 reg |= STRTAB_BASE_RA; 893 strtab->base = reg; 894 895 return (0); 896 } 897 898 static int 899 smmu_init_strtab_2lvl(struct smmu_softc *sc) 900 { 901 struct smmu_strtab *strtab; 902 vm_paddr_t base; 903 uint64_t reg_base; 904 uint32_t l1size; 905 uint32_t size; 906 uint32_t reg; 907 int sz; 908 909 strtab = &sc->strtab; 910 911 size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3); 912 size = min(size, sc->sid_bits - STRTAB_SPLIT); 913 strtab->num_l1_entries = (1 << size); 914 size += STRTAB_SPLIT; 915 916 l1size = strtab->num_l1_entries * (STRTAB_L1_DESC_DWORDS << 3); 917 918 if (bootverbose) 919 device_printf(sc->dev, 920 "%s: size %d, l1 entries %d, l1size %d\n", 921 __func__, size, strtab->num_l1_entries, l1size); 922 923 strtab->vaddr = contigmalloc(l1size, M_SMMU, 924 M_WAITOK | M_ZERO, /* flags */ 925 0, /* low */ 926 (1ul << 48) - 1, /* high */ 927 l1size, /* alignment */ 928 0); /* boundary */ 929 if (strtab->vaddr == NULL) { 930 device_printf(sc->dev, "Failed to allocate 2lvl strtab.\n"); 931 return (ENOMEM); 932 } 933 934 sz = strtab->num_l1_entries * sizeof(struct l1_desc); 935 936 strtab->l1 = malloc(sz, M_SMMU, M_WAITOK | M_ZERO); 937 if (strtab->l1 == NULL) { 938 contigfree(strtab->vaddr, l1size, M_SMMU); 939 return (ENOMEM); 940 } 941 942 reg = STRTAB_BASE_CFG_FMT_2LVL; 943 reg |= size << STRTAB_BASE_CFG_LOG2SIZE_S; 944 reg |= STRTAB_SPLIT << STRTAB_BASE_CFG_SPLIT_S; 945 strtab->base_cfg = (uint32_t)reg; 946 947 base = vtophys(strtab->vaddr); 948 949 reg_base = base & STRTAB_BASE_ADDR_M; 950 KASSERT(reg_base == base, ("bad allocation 3")); 951 reg_base |= STRTAB_BASE_RA; 952 strtab->base = reg_base; 953 954 return (0); 955 } 956 957 static int 958 smmu_init_strtab(struct smmu_softc *sc) 959 { 960 int error; 961 962 if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) 963 error = smmu_init_strtab_2lvl(sc); 964 else 965 error = smmu_init_strtab_linear(sc); 966 967 return (error); 968 } 969 970 static int 971 smmu_init_l1_entry(struct smmu_softc *sc, int sid) 972 { 973 struct smmu_strtab *strtab; 974 struct l1_desc *l1_desc; 975 uint64_t *addr; 976 uint64_t val; 977 size_t size; 978 int i; 979 980 strtab = &sc->strtab; 981 l1_desc = &strtab->l1[sid >> STRTAB_SPLIT]; 982 983 size = 1 << (STRTAB_SPLIT + ilog2(STRTAB_STE_DWORDS) + 3); 984 985 l1_desc->span = STRTAB_SPLIT + 1; 986 l1_desc->size = size; 987 l1_desc->va = contigmalloc(size, M_SMMU, 988 M_WAITOK | M_ZERO, /* flags */ 989 0, /* low */ 990 (1ul << 48) - 1, /* high */ 991 size, /* alignment */ 992 0); /* boundary */ 993 if (l1_desc->va == NULL) { 994 device_printf(sc->dev, "failed to allocate l2 entry\n"); 995 return (ENXIO); 996 } 997 998 l1_desc->pa = vtophys(l1_desc->va); 999 1000 i = sid >> STRTAB_SPLIT; 1001 addr = (void *)((uint64_t)strtab->vaddr + 1002 STRTAB_L1_DESC_DWORDS * 8 * i); 1003 1004 /* Install the L1 entry. */ 1005 val = l1_desc->pa & STRTAB_L1_DESC_L2PTR_M; 1006 KASSERT(val == l1_desc->pa, ("bad allocation 4")); 1007 val |= l1_desc->span; 1008 *addr = val; 1009 1010 return (0); 1011 } 1012 1013 static void 1014 smmu_deinit_l1_entry(struct smmu_softc *sc, int sid) 1015 { 1016 struct smmu_strtab *strtab; 1017 struct l1_desc *l1_desc; 1018 uint64_t *addr; 1019 int i; 1020 1021 strtab = &sc->strtab; 1022 1023 i = sid >> STRTAB_SPLIT; 1024 addr = (void *)((uint64_t)strtab->vaddr + 1025 STRTAB_L1_DESC_DWORDS * 8 * i); 1026 *addr = 0; 1027 1028 if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) { 1029 l1_desc = &strtab->l1[sid >> STRTAB_SPLIT]; 1030 contigfree(l1_desc->va, l1_desc->size, M_SMMU); 1031 } 1032 } 1033 1034 static int 1035 smmu_disable(struct smmu_softc *sc) 1036 { 1037 uint32_t reg; 1038 int error; 1039 1040 /* Disable SMMU */ 1041 reg = bus_read_4(sc->res[0], SMMU_CR0); 1042 reg &= ~CR0_SMMUEN; 1043 error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg); 1044 if (error) 1045 device_printf(sc->dev, "Could not disable SMMU.\n"); 1046 1047 return (0); 1048 } 1049 1050 static int 1051 smmu_event_intr(void *arg) 1052 { 1053 uint32_t evt[EVTQ_ENTRY_DWORDS * 2]; 1054 struct smmu_softc *sc; 1055 1056 sc = arg; 1057 1058 do { 1059 smmu_evtq_dequeue(sc, evt); 1060 smmu_print_event(sc, evt); 1061 } while (!smmu_q_empty(&sc->evtq)); 1062 1063 return (FILTER_HANDLED); 1064 } 1065 1066 static int __unused 1067 smmu_sync_intr(void *arg) 1068 { 1069 struct smmu_softc *sc; 1070 1071 sc = arg; 1072 1073 device_printf(sc->dev, "%s\n", __func__); 1074 1075 return (FILTER_HANDLED); 1076 } 1077 1078 static int 1079 smmu_gerr_intr(void *arg) 1080 { 1081 struct smmu_softc *sc; 1082 1083 sc = arg; 1084 1085 device_printf(sc->dev, "SMMU Global Error\n"); 1086 1087 return (FILTER_HANDLED); 1088 } 1089 1090 static int 1091 smmu_enable_interrupts(struct smmu_softc *sc) 1092 { 1093 uint32_t reg; 1094 int error; 1095 1096 /* Disable MSI. */ 1097 bus_write_8(sc->res[0], SMMU_GERROR_IRQ_CFG0, 0); 1098 bus_write_4(sc->res[0], SMMU_GERROR_IRQ_CFG1, 0); 1099 bus_write_4(sc->res[0], SMMU_GERROR_IRQ_CFG2, 0); 1100 1101 bus_write_8(sc->res[0], SMMU_EVENTQ_IRQ_CFG0, 0); 1102 bus_write_4(sc->res[0], SMMU_EVENTQ_IRQ_CFG1, 0); 1103 bus_write_4(sc->res[0], SMMU_EVENTQ_IRQ_CFG2, 0); 1104 1105 if (sc->features & CR0_PRIQEN) { 1106 bus_write_8(sc->res[0], SMMU_PRIQ_IRQ_CFG0, 0); 1107 bus_write_4(sc->res[0], SMMU_PRIQ_IRQ_CFG1, 0); 1108 bus_write_4(sc->res[0], SMMU_PRIQ_IRQ_CFG2, 0); 1109 } 1110 1111 /* Disable any interrupts. */ 1112 error = smmu_write_ack(sc, SMMU_IRQ_CTRL, SMMU_IRQ_CTRLACK, 0); 1113 if (error) { 1114 device_printf(sc->dev, "Could not disable interrupts.\n"); 1115 return (ENXIO); 1116 } 1117 1118 /* Enable interrupts. */ 1119 reg = IRQ_CTRL_EVENTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN; 1120 if (sc->features & SMMU_FEATURE_PRI) 1121 reg |= IRQ_CTRL_PRIQ_IRQEN; 1122 1123 error = smmu_write_ack(sc, SMMU_IRQ_CTRL, SMMU_IRQ_CTRLACK, reg); 1124 if (error) { 1125 device_printf(sc->dev, "Could not enable interrupts.\n"); 1126 return (ENXIO); 1127 } 1128 1129 return (0); 1130 } 1131 1132 #if DEV_ACPI 1133 static void 1134 smmu_configure_intr(struct smmu_softc *sc, struct resource *res) 1135 { 1136 struct intr_map_data_acpi *ad; 1137 struct intr_map_data *data; 1138 1139 data = rman_get_virtual(res); 1140 KASSERT(data != NULL, ("data is NULL")); 1141 1142 if (data->type == INTR_MAP_DATA_ACPI) { 1143 ad = (struct intr_map_data_acpi *)data; 1144 ad->trig = INTR_TRIGGER_EDGE; 1145 ad->pol = INTR_POLARITY_HIGH; 1146 } 1147 } 1148 #endif 1149 1150 static int 1151 smmu_setup_interrupts(struct smmu_softc *sc) 1152 { 1153 device_t dev; 1154 int error; 1155 1156 dev = sc->dev; 1157 1158 #if DEV_ACPI 1159 /* 1160 * Configure SMMU interrupts as EDGE triggered manually 1161 * as ACPI tables carries no information for that. 1162 */ 1163 smmu_configure_intr(sc, sc->res[1]); 1164 smmu_configure_intr(sc, sc->res[2]); 1165 smmu_configure_intr(sc, sc->res[3]); 1166 #endif 1167 1168 error = bus_setup_intr(dev, sc->res[1], INTR_TYPE_MISC, 1169 smmu_event_intr, NULL, sc, &sc->intr_cookie[0]); 1170 if (error) { 1171 device_printf(dev, "Couldn't setup Event interrupt handler\n"); 1172 return (ENXIO); 1173 } 1174 1175 error = bus_setup_intr(dev, sc->res[3], INTR_TYPE_MISC, 1176 smmu_gerr_intr, NULL, sc, &sc->intr_cookie[2]); 1177 if (error) { 1178 device_printf(dev, "Couldn't setup Gerr interrupt handler\n"); 1179 return (ENXIO); 1180 } 1181 1182 return (0); 1183 } 1184 1185 static int 1186 smmu_reset(struct smmu_softc *sc) 1187 { 1188 struct smmu_cmdq_entry cmd; 1189 struct smmu_strtab *strtab; 1190 int error; 1191 int reg; 1192 1193 reg = bus_read_4(sc->res[0], SMMU_CR0); 1194 1195 if (reg & CR0_SMMUEN) 1196 device_printf(sc->dev, 1197 "%s: Warning: SMMU is enabled\n", __func__); 1198 1199 error = smmu_disable(sc); 1200 if (error) 1201 device_printf(sc->dev, 1202 "%s: Could not disable SMMU.\n", __func__); 1203 1204 if (smmu_enable_interrupts(sc) != 0) { 1205 device_printf(sc->dev, "Could not enable interrupts.\n"); 1206 return (ENXIO); 1207 } 1208 1209 reg = CR1_TABLE_SH_IS | 1210 CR1_TABLE_OC_WBC | 1211 CR1_TABLE_IC_WBC | 1212 CR1_QUEUE_SH_IS | 1213 CR1_QUEUE_OC_WBC | 1214 CR1_QUEUE_IC_WBC; 1215 bus_write_4(sc->res[0], SMMU_CR1, reg); 1216 1217 reg = CR2_PTM | CR2_RECINVSID | CR2_E2H; 1218 bus_write_4(sc->res[0], SMMU_CR2, reg); 1219 1220 /* Stream table. */ 1221 strtab = &sc->strtab; 1222 bus_write_8(sc->res[0], SMMU_STRTAB_BASE, strtab->base); 1223 bus_write_4(sc->res[0], SMMU_STRTAB_BASE_CFG, strtab->base_cfg); 1224 1225 /* Command queue. */ 1226 bus_write_8(sc->res[0], SMMU_CMDQ_BASE, sc->cmdq.base); 1227 bus_write_4(sc->res[0], SMMU_CMDQ_PROD, sc->cmdq.lc.prod); 1228 bus_write_4(sc->res[0], SMMU_CMDQ_CONS, sc->cmdq.lc.cons); 1229 1230 reg = CR0_CMDQEN; 1231 error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg); 1232 if (error) { 1233 device_printf(sc->dev, "Could not enable command queue\n"); 1234 return (ENXIO); 1235 } 1236 1237 /* Invalidate cached configuration. */ 1238 smmu_invalidate_all_sid(sc); 1239 1240 if (sc->features & SMMU_FEATURE_HYP) { 1241 cmd.opcode = CMD_TLBI_EL2_ALL; 1242 smmu_cmdq_enqueue_cmd(sc, &cmd); 1243 }; 1244 1245 /* Invalidate TLB. */ 1246 smmu_tlbi_all(sc); 1247 1248 /* Event queue */ 1249 bus_write_8(sc->res[0], SMMU_EVENTQ_BASE, sc->evtq.base); 1250 bus_write_4(sc->res[0], SMMU_EVENTQ_PROD, sc->evtq.lc.prod); 1251 bus_write_4(sc->res[0], SMMU_EVENTQ_CONS, sc->evtq.lc.cons); 1252 1253 reg |= CR0_EVENTQEN; 1254 error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg); 1255 if (error) { 1256 device_printf(sc->dev, "Could not enable event queue\n"); 1257 return (ENXIO); 1258 } 1259 1260 if (sc->features & SMMU_FEATURE_PRI) { 1261 /* PRI queue */ 1262 bus_write_8(sc->res[0], SMMU_PRIQ_BASE, sc->priq.base); 1263 bus_write_4(sc->res[0], SMMU_PRIQ_PROD, sc->priq.lc.prod); 1264 bus_write_4(sc->res[0], SMMU_PRIQ_CONS, sc->priq.lc.cons); 1265 1266 reg |= CR0_PRIQEN; 1267 error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg); 1268 if (error) { 1269 device_printf(sc->dev, "Could not enable PRI queue\n"); 1270 return (ENXIO); 1271 } 1272 } 1273 1274 if (sc->features & SMMU_FEATURE_ATS) { 1275 reg |= CR0_ATSCHK; 1276 error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg); 1277 if (error) { 1278 device_printf(sc->dev, "Could not enable ATS check.\n"); 1279 return (ENXIO); 1280 } 1281 } 1282 1283 reg |= CR0_SMMUEN; 1284 error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg); 1285 if (error) { 1286 device_printf(sc->dev, "Could not enable SMMU.\n"); 1287 return (ENXIO); 1288 } 1289 1290 return (0); 1291 } 1292 1293 static int 1294 smmu_check_features(struct smmu_softc *sc) 1295 { 1296 uint32_t reg; 1297 uint32_t val; 1298 1299 sc->features = 0; 1300 1301 reg = bus_read_4(sc->res[0], SMMU_IDR0); 1302 1303 if (reg & IDR0_ST_LVL_2) { 1304 if (bootverbose) 1305 device_printf(sc->dev, 1306 "2-level stream table supported.\n"); 1307 sc->features |= SMMU_FEATURE_2_LVL_STREAM_TABLE; 1308 } 1309 1310 if (reg & IDR0_CD2L) { 1311 if (bootverbose) 1312 device_printf(sc->dev, 1313 "2-level CD table supported.\n"); 1314 sc->features |= SMMU_FEATURE_2_LVL_CD; 1315 } 1316 1317 switch (reg & IDR0_TTENDIAN_M) { 1318 case IDR0_TTENDIAN_MIXED: 1319 if (bootverbose) 1320 device_printf(sc->dev, "Mixed endianess supported.\n"); 1321 sc->features |= SMMU_FEATURE_TT_LE; 1322 sc->features |= SMMU_FEATURE_TT_BE; 1323 break; 1324 case IDR0_TTENDIAN_LITTLE: 1325 if (bootverbose) 1326 device_printf(sc->dev, 1327 "Little endian supported only.\n"); 1328 sc->features |= SMMU_FEATURE_TT_LE; 1329 break; 1330 case IDR0_TTENDIAN_BIG: 1331 if (bootverbose) 1332 device_printf(sc->dev, "Big endian supported only.\n"); 1333 sc->features |= SMMU_FEATURE_TT_BE; 1334 break; 1335 default: 1336 device_printf(sc->dev, "Unsupported endianness.\n"); 1337 return (ENXIO); 1338 } 1339 1340 if (reg & IDR0_SEV) 1341 sc->features |= SMMU_FEATURE_SEV; 1342 1343 if (reg & IDR0_MSI) { 1344 if (bootverbose) 1345 device_printf(sc->dev, "MSI feature present.\n"); 1346 sc->features |= SMMU_FEATURE_MSI; 1347 } 1348 1349 if (reg & IDR0_HYP) { 1350 if (bootverbose) 1351 device_printf(sc->dev, "HYP feature present.\n"); 1352 sc->features |= SMMU_FEATURE_HYP; 1353 } 1354 1355 if (reg & IDR0_ATS) 1356 sc->features |= SMMU_FEATURE_ATS; 1357 1358 if (reg & IDR0_PRI) 1359 sc->features |= SMMU_FEATURE_PRI; 1360 1361 switch (reg & IDR0_STALL_MODEL_M) { 1362 case IDR0_STALL_MODEL_FORCE: 1363 /* Stall is forced. */ 1364 sc->features |= SMMU_FEATURE_STALL_FORCE; 1365 /* FALLTHROUGH */ 1366 case IDR0_STALL_MODEL_STALL: 1367 sc->features |= SMMU_FEATURE_STALL; 1368 break; 1369 } 1370 1371 /* Grab translation stages supported. */ 1372 if (reg & IDR0_S1P) { 1373 if (bootverbose) 1374 device_printf(sc->dev, 1375 "Stage 1 translation supported.\n"); 1376 sc->features |= SMMU_FEATURE_S1P; 1377 } 1378 if (reg & IDR0_S2P) { 1379 if (bootverbose) 1380 device_printf(sc->dev, 1381 "Stage 2 translation supported.\n"); 1382 sc->features |= SMMU_FEATURE_S2P; 1383 } 1384 1385 switch (reg & IDR0_TTF_M) { 1386 case IDR0_TTF_ALL: 1387 case IDR0_TTF_AA64: 1388 sc->ias = 40; 1389 break; 1390 default: 1391 device_printf(sc->dev, "No AArch64 table format support.\n"); 1392 return (ENXIO); 1393 } 1394 1395 if (reg & IDR0_ASID16) 1396 sc->asid_bits = 16; 1397 else 1398 sc->asid_bits = 8; 1399 1400 if (bootverbose) 1401 device_printf(sc->dev, "ASID bits %d\n", sc->asid_bits); 1402 1403 if (reg & IDR0_VMID16) 1404 sc->vmid_bits = 16; 1405 else 1406 sc->vmid_bits = 8; 1407 1408 reg = bus_read_4(sc->res[0], SMMU_IDR1); 1409 1410 if (reg & (IDR1_TABLES_PRESET | IDR1_QUEUES_PRESET | IDR1_REL)) { 1411 device_printf(sc->dev, 1412 "Embedded implementations not supported by this driver.\n"); 1413 return (ENXIO); 1414 } 1415 1416 val = (reg & IDR1_CMDQS_M) >> IDR1_CMDQS_S; 1417 sc->cmdq.size_log2 = val; 1418 if (bootverbose) 1419 device_printf(sc->dev, "CMD queue bits %d\n", val); 1420 1421 val = (reg & IDR1_EVENTQS_M) >> IDR1_EVENTQS_S; 1422 sc->evtq.size_log2 = val; 1423 if (bootverbose) 1424 device_printf(sc->dev, "EVENT queue bits %d\n", val); 1425 1426 if (sc->features & SMMU_FEATURE_PRI) { 1427 val = (reg & IDR1_PRIQS_M) >> IDR1_PRIQS_S; 1428 sc->priq.size_log2 = val; 1429 if (bootverbose) 1430 device_printf(sc->dev, "PRI queue bits %d\n", val); 1431 } 1432 1433 sc->ssid_bits = (reg & IDR1_SSIDSIZE_M) >> IDR1_SSIDSIZE_S; 1434 sc->sid_bits = (reg & IDR1_SIDSIZE_M) >> IDR1_SIDSIZE_S; 1435 1436 if (sc->sid_bits <= STRTAB_SPLIT) 1437 sc->features &= ~SMMU_FEATURE_2_LVL_STREAM_TABLE; 1438 1439 if (bootverbose) { 1440 device_printf(sc->dev, "SSID bits %d\n", sc->ssid_bits); 1441 device_printf(sc->dev, "SID bits %d\n", sc->sid_bits); 1442 } 1443 1444 /* IDR3 */ 1445 reg = bus_read_4(sc->res[0], SMMU_IDR3); 1446 if (reg & IDR3_RIL) 1447 sc->features |= SMMU_FEATURE_RANGE_INV; 1448 1449 /* IDR5 */ 1450 reg = bus_read_4(sc->res[0], SMMU_IDR5); 1451 1452 switch (reg & IDR5_OAS_M) { 1453 case IDR5_OAS_32: 1454 sc->oas = 32; 1455 break; 1456 case IDR5_OAS_36: 1457 sc->oas = 36; 1458 break; 1459 case IDR5_OAS_40: 1460 sc->oas = 40; 1461 break; 1462 case IDR5_OAS_42: 1463 sc->oas = 42; 1464 break; 1465 case IDR5_OAS_44: 1466 sc->oas = 44; 1467 break; 1468 case IDR5_OAS_48: 1469 sc->oas = 48; 1470 break; 1471 case IDR5_OAS_52: 1472 sc->oas = 52; 1473 break; 1474 } 1475 1476 sc->pgsizes = 0; 1477 if (reg & IDR5_GRAN64K) 1478 sc->pgsizes |= 64 * 1024; 1479 if (reg & IDR5_GRAN16K) 1480 sc->pgsizes |= 16 * 1024; 1481 if (reg & IDR5_GRAN4K) 1482 sc->pgsizes |= 4 * 1024; 1483 1484 if ((reg & IDR5_VAX_M) == IDR5_VAX_52) 1485 sc->features |= SMMU_FEATURE_VAX; 1486 1487 return (0); 1488 } 1489 1490 static void 1491 smmu_init_asids(struct smmu_softc *sc) 1492 { 1493 1494 sc->asid_set_size = (1 << sc->asid_bits); 1495 sc->asid_set = bit_alloc(sc->asid_set_size, M_SMMU, M_WAITOK); 1496 mtx_init(&sc->asid_set_mutex, "asid set", NULL, MTX_SPIN); 1497 } 1498 1499 static int 1500 smmu_asid_alloc(struct smmu_softc *sc, int *new_asid) 1501 { 1502 1503 mtx_lock_spin(&sc->asid_set_mutex); 1504 bit_ffc(sc->asid_set, sc->asid_set_size, new_asid); 1505 if (*new_asid == -1) { 1506 mtx_unlock_spin(&sc->asid_set_mutex); 1507 return (ENOMEM); 1508 } 1509 bit_set(sc->asid_set, *new_asid); 1510 mtx_unlock_spin(&sc->asid_set_mutex); 1511 1512 return (0); 1513 } 1514 1515 static void 1516 smmu_asid_free(struct smmu_softc *sc, int asid) 1517 { 1518 1519 mtx_lock_spin(&sc->asid_set_mutex); 1520 bit_clear(sc->asid_set, asid); 1521 mtx_unlock_spin(&sc->asid_set_mutex); 1522 } 1523 1524 /* 1525 * Device interface. 1526 */ 1527 int 1528 smmu_attach(device_t dev) 1529 { 1530 struct smmu_softc *sc; 1531 int error; 1532 1533 sc = device_get_softc(dev); 1534 sc->dev = dev; 1535 1536 mtx_init(&sc->sc_mtx, device_get_nameunit(sc->dev), "smmu", MTX_DEF); 1537 1538 error = bus_alloc_resources(dev, smmu_spec, sc->res); 1539 if (error) { 1540 device_printf(dev, "Couldn't allocate resources.\n"); 1541 return (ENXIO); 1542 } 1543 1544 error = smmu_setup_interrupts(sc); 1545 if (error) { 1546 bus_release_resources(dev, smmu_spec, sc->res); 1547 return (ENXIO); 1548 } 1549 1550 error = smmu_check_features(sc); 1551 if (error) { 1552 device_printf(dev, "Some features are required " 1553 "but not supported by hardware.\n"); 1554 return (ENXIO); 1555 } 1556 1557 smmu_init_asids(sc); 1558 1559 error = smmu_init_queues(sc); 1560 if (error) { 1561 device_printf(dev, "Couldn't allocate queues.\n"); 1562 return (ENXIO); 1563 } 1564 1565 error = smmu_init_strtab(sc); 1566 if (error) { 1567 device_printf(dev, "Couldn't allocate strtab.\n"); 1568 return (ENXIO); 1569 } 1570 1571 error = smmu_reset(sc); 1572 if (error) { 1573 device_printf(dev, "Couldn't reset SMMU.\n"); 1574 return (ENXIO); 1575 } 1576 1577 return (0); 1578 } 1579 1580 int 1581 smmu_detach(device_t dev) 1582 { 1583 struct smmu_softc *sc; 1584 1585 sc = device_get_softc(dev); 1586 1587 bus_release_resources(dev, smmu_spec, sc->res); 1588 1589 return (0); 1590 } 1591 1592 static int 1593 smmu_read_ivar(device_t dev, device_t child, int which, uintptr_t *result) 1594 { 1595 struct smmu_softc *sc; 1596 1597 sc = device_get_softc(dev); 1598 1599 device_printf(sc->dev, "%s\n", __func__); 1600 1601 return (ENOENT); 1602 } 1603 1604 static int 1605 smmu_unmap(device_t dev, struct iommu_domain *iodom, 1606 vm_offset_t va, bus_size_t size) 1607 { 1608 struct smmu_domain *domain; 1609 struct smmu_softc *sc; 1610 int err; 1611 int i; 1612 1613 sc = device_get_softc(dev); 1614 1615 domain = (struct smmu_domain *)iodom; 1616 1617 err = 0; 1618 1619 dprintf("%s: %lx, %ld, domain %d\n", __func__, va, size, domain->asid); 1620 1621 for (i = 0; i < size; i += PAGE_SIZE) { 1622 if (pmap_smmu_remove(&domain->p, va) == 0) { 1623 /* pmap entry removed, invalidate TLB. */ 1624 smmu_tlbi_va(sc, va, domain->asid); 1625 } else { 1626 err = ENOENT; 1627 break; 1628 } 1629 va += PAGE_SIZE; 1630 } 1631 1632 smmu_sync(sc); 1633 1634 return (err); 1635 } 1636 1637 static int 1638 smmu_map(device_t dev, struct iommu_domain *iodom, 1639 vm_offset_t va, vm_page_t *ma, vm_size_t size, 1640 vm_prot_t prot) 1641 { 1642 struct smmu_domain *domain; 1643 struct smmu_softc *sc; 1644 vm_paddr_t pa; 1645 int error; 1646 int i; 1647 1648 sc = device_get_softc(dev); 1649 1650 domain = (struct smmu_domain *)iodom; 1651 1652 dprintf("%s: %lx -> %lx, %ld, domain %d\n", __func__, va, pa, size, 1653 domain->asid); 1654 1655 for (i = 0; size > 0; size -= PAGE_SIZE) { 1656 pa = VM_PAGE_TO_PHYS(ma[i++]); 1657 error = pmap_smmu_enter(&domain->p, va, pa, prot, 0); 1658 if (error) 1659 return (error); 1660 smmu_tlbi_va(sc, va, domain->asid); 1661 va += PAGE_SIZE; 1662 } 1663 1664 smmu_sync(sc); 1665 1666 return (0); 1667 } 1668 1669 static struct iommu_domain * 1670 smmu_domain_alloc(device_t dev, struct iommu_unit *iommu) 1671 { 1672 struct smmu_domain *domain; 1673 struct smmu_unit *unit; 1674 struct smmu_softc *sc; 1675 int error; 1676 int new_asid; 1677 1678 sc = device_get_softc(dev); 1679 1680 unit = (struct smmu_unit *)iommu; 1681 1682 domain = malloc(sizeof(*domain), M_SMMU, M_WAITOK | M_ZERO); 1683 1684 error = smmu_asid_alloc(sc, &new_asid); 1685 if (error) { 1686 free(domain, M_SMMU); 1687 device_printf(sc->dev, 1688 "Could not allocate ASID for a new domain.\n"); 1689 return (NULL); 1690 } 1691 1692 domain->asid = (uint16_t)new_asid; 1693 1694 iommu_pmap_pinit(&domain->p); 1695 PMAP_LOCK_INIT(&domain->p); 1696 1697 error = smmu_init_cd(sc, domain); 1698 if (error) { 1699 free(domain, M_SMMU); 1700 device_printf(sc->dev, "Could not initialize CD\n"); 1701 return (NULL); 1702 } 1703 1704 smmu_tlbi_asid(sc, domain->asid); 1705 1706 LIST_INIT(&domain->ctx_list); 1707 1708 IOMMU_LOCK(iommu); 1709 LIST_INSERT_HEAD(&unit->domain_list, domain, next); 1710 IOMMU_UNLOCK(iommu); 1711 1712 return (&domain->iodom); 1713 } 1714 1715 static void 1716 smmu_domain_free(device_t dev, struct iommu_domain *iodom) 1717 { 1718 struct smmu_domain *domain; 1719 struct smmu_softc *sc; 1720 struct smmu_cd *cd; 1721 1722 sc = device_get_softc(dev); 1723 1724 domain = (struct smmu_domain *)iodom; 1725 1726 LIST_REMOVE(domain, next); 1727 1728 cd = domain->cd; 1729 1730 iommu_pmap_remove_pages(&domain->p); 1731 iommu_pmap_release(&domain->p); 1732 1733 smmu_tlbi_asid(sc, domain->asid); 1734 smmu_asid_free(sc, domain->asid); 1735 1736 contigfree(cd->vaddr, cd->size, M_SMMU); 1737 free(cd, M_SMMU); 1738 1739 free(domain, M_SMMU); 1740 } 1741 1742 static int 1743 smmu_set_buswide(device_t dev, struct smmu_domain *domain, 1744 struct smmu_ctx *ctx) 1745 { 1746 struct smmu_softc *sc; 1747 int i; 1748 1749 sc = device_get_softc(dev); 1750 1751 for (i = 0; i < PCI_SLOTMAX; i++) 1752 smmu_init_ste(sc, domain->cd, (ctx->sid | i), ctx->bypass); 1753 1754 return (0); 1755 } 1756 1757 static struct iommu_ctx * 1758 smmu_ctx_alloc(device_t dev, struct iommu_domain *iodom, device_t child, 1759 bool disabled) 1760 { 1761 struct smmu_domain *domain; 1762 struct smmu_softc *sc; 1763 struct smmu_ctx *ctx; 1764 uint16_t rid; 1765 u_int xref, sid; 1766 int seg; 1767 int err; 1768 1769 sc = device_get_softc(dev); 1770 domain = (struct smmu_domain *)iodom; 1771 1772 seg = pci_get_domain(child); 1773 rid = pci_get_rid(child); 1774 err = acpi_iort_map_pci_smmuv3(seg, rid, &xref, &sid); 1775 if (err) 1776 return (NULL); 1777 1778 if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) { 1779 err = smmu_init_l1_entry(sc, sid); 1780 if (err) 1781 return (NULL); 1782 } 1783 1784 ctx = malloc(sizeof(struct smmu_ctx), M_SMMU, M_WAITOK | M_ZERO); 1785 ctx->vendor = pci_get_vendor(child); 1786 ctx->device = pci_get_device(child); 1787 ctx->dev = child; 1788 ctx->sid = sid; 1789 ctx->domain = domain; 1790 if (disabled) 1791 ctx->bypass = true; 1792 1793 /* 1794 * Neoverse N1 SDP: 1795 * 0x800 xhci 1796 * 0x700 re 1797 * 0x600 sata 1798 */ 1799 1800 smmu_init_ste(sc, domain->cd, ctx->sid, ctx->bypass); 1801 1802 if (iommu_is_buswide_ctx(iodom->iommu, pci_get_bus(ctx->dev))) 1803 smmu_set_buswide(dev, domain, ctx); 1804 1805 IOMMU_DOMAIN_LOCK(iodom); 1806 LIST_INSERT_HEAD(&domain->ctx_list, ctx, next); 1807 IOMMU_DOMAIN_UNLOCK(iodom); 1808 1809 return (&ctx->ioctx); 1810 } 1811 1812 static void 1813 smmu_ctx_free(device_t dev, struct iommu_ctx *ioctx) 1814 { 1815 struct smmu_softc *sc; 1816 struct smmu_ctx *ctx; 1817 1818 IOMMU_ASSERT_LOCKED(ioctx->domain->iommu); 1819 1820 sc = device_get_softc(dev); 1821 ctx = (struct smmu_ctx *)ioctx; 1822 1823 smmu_deinit_l1_entry(sc, ctx->sid); 1824 1825 LIST_REMOVE(ctx, next); 1826 1827 free(ctx, M_SMMU); 1828 } 1829 1830 struct smmu_ctx * 1831 smmu_ctx_lookup_by_sid(device_t dev, u_int sid) 1832 { 1833 struct smmu_softc *sc; 1834 struct smmu_domain *domain; 1835 struct smmu_unit *unit; 1836 struct smmu_ctx *ctx; 1837 1838 sc = device_get_softc(dev); 1839 1840 unit = &sc->unit; 1841 1842 LIST_FOREACH(domain, &unit->domain_list, next) { 1843 LIST_FOREACH(ctx, &domain->ctx_list, next) { 1844 if (ctx->sid == sid) 1845 return (ctx); 1846 } 1847 } 1848 1849 return (NULL); 1850 } 1851 1852 static struct iommu_ctx * 1853 smmu_ctx_lookup(device_t dev, device_t child) 1854 { 1855 struct iommu_unit *iommu; 1856 struct smmu_softc *sc; 1857 struct smmu_domain *domain; 1858 struct smmu_unit *unit; 1859 struct smmu_ctx *ctx; 1860 1861 sc = device_get_softc(dev); 1862 1863 unit = &sc->unit; 1864 iommu = &unit->iommu; 1865 1866 IOMMU_ASSERT_LOCKED(iommu); 1867 1868 LIST_FOREACH(domain, &unit->domain_list, next) { 1869 IOMMU_DOMAIN_LOCK(&domain->iodom); 1870 LIST_FOREACH(ctx, &domain->ctx_list, next) { 1871 if (ctx->dev == child) { 1872 IOMMU_DOMAIN_UNLOCK(&domain->iodom); 1873 return (&ctx->ioctx); 1874 } 1875 } 1876 IOMMU_DOMAIN_UNLOCK(&domain->iodom); 1877 } 1878 1879 return (NULL); 1880 } 1881 1882 static int 1883 smmu_find(device_t dev, device_t child) 1884 { 1885 struct smmu_softc *sc; 1886 u_int xref, sid; 1887 uint16_t rid; 1888 int error; 1889 int seg; 1890 1891 sc = device_get_softc(dev); 1892 1893 rid = pci_get_rid(child); 1894 seg = pci_get_domain(child); 1895 1896 /* 1897 * Find an xref of an IOMMU controller that serves traffic for dev. 1898 */ 1899 #ifdef DEV_ACPI 1900 error = acpi_iort_map_pci_smmuv3(seg, rid, &xref, &sid); 1901 if (error) { 1902 /* Could not find reference to an SMMU device. */ 1903 return (ENOENT); 1904 } 1905 #else 1906 /* TODO: add FDT support. */ 1907 return (ENXIO); 1908 #endif 1909 1910 /* Check if xref is ours. */ 1911 if (xref != sc->xref) 1912 return (EFAULT); 1913 1914 return (0); 1915 } 1916 1917 static device_method_t smmu_methods[] = { 1918 /* Device interface */ 1919 DEVMETHOD(device_detach, smmu_detach), 1920 1921 /* SMMU interface */ 1922 DEVMETHOD(iommu_find, smmu_find), 1923 DEVMETHOD(iommu_map, smmu_map), 1924 DEVMETHOD(iommu_unmap, smmu_unmap), 1925 DEVMETHOD(iommu_domain_alloc, smmu_domain_alloc), 1926 DEVMETHOD(iommu_domain_free, smmu_domain_free), 1927 DEVMETHOD(iommu_ctx_alloc, smmu_ctx_alloc), 1928 DEVMETHOD(iommu_ctx_free, smmu_ctx_free), 1929 DEVMETHOD(iommu_ctx_lookup, smmu_ctx_lookup), 1930 1931 /* Bus interface */ 1932 DEVMETHOD(bus_read_ivar, smmu_read_ivar), 1933 1934 /* End */ 1935 DEVMETHOD_END 1936 }; 1937 1938 DEFINE_CLASS_0(smmu, smmu_driver, smmu_methods, sizeof(struct smmu_softc)); 1939