1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2019-2020 Ruslan Bukin <br@bsdpad.com> 5 * 6 * This software was developed by SRI International and the University of 7 * Cambridge Computer Laboratory (Department of Computer Science and 8 * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the 9 * DARPA SSITH research programme. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 */ 32 33 /* 34 * Hardware overview. 35 * 36 * An incoming transaction from a peripheral device has an address, size, 37 * attributes and StreamID. 38 * 39 * In case of PCI-based devices, StreamID is a PCI rid. 40 * 41 * The StreamID is used to select a Stream Table Entry (STE) in a Stream table, 42 * which contains per-device configuration. 43 * 44 * Stream table is a linear or 2-level walk table (this driver supports both). 45 * Note that a linear table could occupy 1GB or more of memory depending on 46 * sid_bits value. 47 * 48 * STE is used to locate a Context Descriptor, which is a struct in memory 49 * that describes stages of translation, translation table type, pointer to 50 * level 0 of page tables, ASID, etc. 51 * 52 * Hardware supports two stages of translation: Stage1 (S1) and Stage2 (S2): 53 * o S1 is used for the host machine traffic translation 54 * o S2 is for a hypervisor 55 * 56 * This driver enables S1 stage with standard AArch64 page tables. 57 * 58 * Note that SMMU does not share TLB with a main CPU. 59 * Command queue is used by this driver to Invalidate SMMU TLB, STE cache. 60 * 61 * An arm64 SoC could have more than one SMMU instance. 62 * ACPI IORT table describes which SMMU unit is assigned for a particular 63 * peripheral device. 64 * 65 * Queues. 66 * 67 * Register interface and Memory-based circular buffer queues are used 68 * to interface SMMU. 69 * 70 * These are a Command queue for commands to send to the SMMU and an Event 71 * queue for event/fault reports from the SMMU. Optionally PRI queue is 72 * designed for PCIe page requests reception. 73 * 74 * Note that not every hardware supports PRI services. For instance they were 75 * not found in Neoverse N1 SDP machine. 76 * (This drivers does not implement PRI queue.) 77 * 78 * All SMMU queues are arranged as circular buffers in memory. They are used 79 * in a producer-consumer fashion so that an output queue contains data 80 * produced by the SMMU and consumed by software. 81 * An input queue contains data produced by software, consumed by the SMMU. 82 * 83 * Interrupts. 84 * 85 * Interrupts are not required by this driver for normal operation. 86 * The standard wired interrupt is only triggered when an event comes from 87 * the SMMU, which is only in a case of errors (e.g. translation fault). 88 */ 89 90 #include "opt_platform.h" 91 #include "opt_acpi.h" 92 93 #include <sys/param.h> 94 #include <sys/bitstring.h> 95 #include <sys/bus.h> 96 #include <sys/kernel.h> 97 #include <sys/malloc.h> 98 #include <sys/mutex.h> 99 #include <sys/rman.h> 100 #include <sys/lock.h> 101 #include <sys/sysctl.h> 102 #include <sys/tree.h> 103 #include <sys/taskqueue.h> 104 #include <vm/vm.h> 105 #include <vm/vm_page.h> 106 #ifdef DEV_ACPI 107 #include <contrib/dev/acpica/include/acpi.h> 108 #include <dev/acpica/acpivar.h> 109 #endif 110 #include <dev/pci/pcireg.h> 111 #include <dev/pci/pcivar.h> 112 #include <dev/iommu/iommu.h> 113 #include <arm64/iommu/iommu_pmap.h> 114 115 #include <machine/bus.h> 116 117 #ifdef FDT 118 #include <dev/fdt/fdt_common.h> 119 #include <dev/ofw/ofw_bus.h> 120 #include <dev/ofw/ofw_bus_subr.h> 121 #endif 122 123 #include "iommu.h" 124 #include "iommu_if.h" 125 126 #include "smmureg.h" 127 #include "smmuvar.h" 128 129 #define STRTAB_L1_SZ_SHIFT 20 130 #define STRTAB_SPLIT 8 131 132 #define STRTAB_L1_DESC_L2PTR_M (0x3fffffffffff << 6) 133 #define STRTAB_L1_DESC_DWORDS 1 134 135 #define STRTAB_STE_DWORDS 8 136 137 #define CMDQ_ENTRY_DWORDS 2 138 #define EVTQ_ENTRY_DWORDS 4 139 #define PRIQ_ENTRY_DWORDS 2 140 141 #define CD_DWORDS 8 142 143 #define Q_WRP(q, p) ((p) & (1 << (q)->size_log2)) 144 #define Q_IDX(q, p) ((p) & ((1 << (q)->size_log2) - 1)) 145 #define Q_OVF(p) ((p) & (1 << 31)) /* Event queue overflowed */ 146 147 #define SMMU_Q_ALIGN (64 * 1024) 148 149 #define MAXADDR_48BIT 0xFFFFFFFFFFFFUL 150 #define MAXADDR_52BIT 0xFFFFFFFFFFFFFUL 151 152 static struct resource_spec smmu_spec[] = { 153 { SYS_RES_MEMORY, 0, RF_ACTIVE }, 154 { SYS_RES_IRQ, 0, RF_ACTIVE }, 155 { SYS_RES_IRQ, 1, RF_ACTIVE | RF_OPTIONAL }, 156 { SYS_RES_IRQ, 2, RF_ACTIVE }, 157 { SYS_RES_IRQ, 3, RF_ACTIVE }, 158 RESOURCE_SPEC_END 159 }; 160 161 MALLOC_DEFINE(M_SMMU, "SMMU", SMMU_DEVSTR); 162 163 #define dprintf(fmt, ...) 164 165 struct smmu_event { 166 int ident; 167 char *str; 168 char *msg; 169 }; 170 171 static struct smmu_event events[] = { 172 { 0x01, "F_UUT", 173 "Unsupported Upstream Transaction."}, 174 { 0x02, "C_BAD_STREAMID", 175 "Transaction StreamID out of range."}, 176 { 0x03, "F_STE_FETCH", 177 "Fetch of STE caused external abort."}, 178 { 0x04, "C_BAD_STE", 179 "Used STE invalid."}, 180 { 0x05, "F_BAD_ATS_TREQ", 181 "Address Translation Request disallowed for a StreamID " 182 "and a PCIe ATS Translation Request received."}, 183 { 0x06, "F_STREAM_DISABLED", 184 "The STE of a transaction marks non-substream transactions " 185 "disabled."}, 186 { 0x07, "F_TRANSL_FORBIDDEN", 187 "An incoming PCIe transaction is marked Translated but " 188 "SMMU bypass is disallowed for this StreamID."}, 189 { 0x08, "C_BAD_SUBSTREAMID", 190 "Incoming SubstreamID present, but configuration is invalid."}, 191 { 0x09, "F_CD_FETCH", 192 "Fetch of CD caused external abort."}, 193 { 0x0a, "C_BAD_CD", 194 "Fetched CD invalid."}, 195 { 0x0b, "F_WALK_EABT", 196 "An external abort occurred fetching (or updating) " 197 "a translation table descriptor."}, 198 { 0x10, "F_TRANSLATION", 199 "Translation fault."}, 200 { 0x11, "F_ADDR_SIZE", 201 "Address Size fault."}, 202 { 0x12, "F_ACCESS", 203 "Access flag fault due to AF == 0 in a page or block TTD."}, 204 { 0x13, "F_PERMISSION", 205 "Permission fault occurred on page access."}, 206 { 0x20, "F_TLB_CONFLICT", 207 "A TLB conflict occurred because of the transaction."}, 208 { 0x21, "F_CFG_CONFLICT", 209 "A configuration cache conflict occurred due to " 210 "the transaction."}, 211 { 0x24, "E_PAGE_REQUEST", 212 "Speculative page request hint."}, 213 { 0x25, "F_VMS_FETCH", 214 "Fetch of VMS caused external abort."}, 215 { 0, NULL, NULL }, 216 }; 217 218 static int 219 smmu_q_has_space(struct smmu_queue *q) 220 { 221 222 /* 223 * See 6.3.27 SMMU_CMDQ_PROD 224 * 225 * There is space in the queue for additional commands if: 226 * SMMU_CMDQ_CONS.RD != SMMU_CMDQ_PROD.WR || 227 * SMMU_CMDQ_CONS.RD_WRAP == SMMU_CMDQ_PROD.WR_WRAP 228 */ 229 230 if (Q_IDX(q, q->lc.cons) != Q_IDX(q, q->lc.prod) || 231 Q_WRP(q, q->lc.cons) == Q_WRP(q, q->lc.prod)) 232 return (1); 233 234 return (0); 235 } 236 237 static int 238 smmu_q_empty(struct smmu_queue *q) 239 { 240 241 if (Q_IDX(q, q->lc.cons) == Q_IDX(q, q->lc.prod) && 242 Q_WRP(q, q->lc.cons) == Q_WRP(q, q->lc.prod)) 243 return (1); 244 245 return (0); 246 } 247 248 static int __unused 249 smmu_q_consumed(struct smmu_queue *q, uint32_t prod) 250 { 251 252 if ((Q_WRP(q, q->lc.cons) == Q_WRP(q, prod)) && 253 (Q_IDX(q, q->lc.cons) >= Q_IDX(q, prod))) 254 return (1); 255 256 if ((Q_WRP(q, q->lc.cons) != Q_WRP(q, prod)) && 257 (Q_IDX(q, q->lc.cons) <= Q_IDX(q, prod))) 258 return (1); 259 260 return (0); 261 } 262 263 static uint32_t 264 smmu_q_inc_cons(struct smmu_queue *q) 265 { 266 uint32_t cons; 267 uint32_t val; 268 269 cons = (Q_WRP(q, q->lc.cons) | Q_IDX(q, q->lc.cons)) + 1; 270 val = (Q_OVF(q->lc.cons) | Q_WRP(q, cons) | Q_IDX(q, cons)); 271 272 return (val); 273 } 274 275 static uint32_t 276 smmu_q_inc_prod(struct smmu_queue *q) 277 { 278 uint32_t prod; 279 uint32_t val; 280 281 prod = (Q_WRP(q, q->lc.prod) | Q_IDX(q, q->lc.prod)) + 1; 282 val = (Q_OVF(q->lc.prod) | Q_WRP(q, prod) | Q_IDX(q, prod)); 283 284 return (val); 285 } 286 287 static int 288 smmu_write_ack(struct smmu_softc *sc, uint32_t reg, 289 uint32_t reg_ack, uint32_t val) 290 { 291 uint32_t v; 292 int timeout; 293 294 timeout = 100000; 295 296 bus_write_4(sc->res[0], reg, val); 297 298 do { 299 v = bus_read_4(sc->res[0], reg_ack); 300 if (v == val) 301 break; 302 } while (timeout--); 303 304 if (timeout <= 0) { 305 device_printf(sc->dev, "Failed to write reg.\n"); 306 return (-1); 307 } 308 309 return (0); 310 } 311 312 static inline int 313 ilog2(long x) 314 { 315 316 KASSERT(x > 0 && powerof2(x), ("%s: invalid arg %ld", __func__, x)); 317 318 return (flsl(x) - 1); 319 } 320 321 static int 322 smmu_init_queue(struct smmu_softc *sc, struct smmu_queue *q, 323 uint32_t prod_off, uint32_t cons_off, uint32_t dwords) 324 { 325 int sz; 326 327 sz = (1 << q->size_log2) * dwords * 8; 328 329 /* Set up the command circular buffer */ 330 q->vaddr = contigmalloc(sz, M_SMMU, 331 M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, SMMU_Q_ALIGN, 0); 332 if (q->vaddr == NULL) { 333 device_printf(sc->dev, "failed to allocate %d bytes\n", sz); 334 return (-1); 335 } 336 337 q->prod_off = prod_off; 338 q->cons_off = cons_off; 339 q->paddr = vtophys(q->vaddr); 340 341 q->base = CMDQ_BASE_RA | EVENTQ_BASE_WA | PRIQ_BASE_WA; 342 q->base |= q->paddr & Q_BASE_ADDR_M; 343 q->base |= q->size_log2 << Q_LOG2SIZE_S; 344 345 return (0); 346 } 347 348 static int 349 smmu_init_queues(struct smmu_softc *sc) 350 { 351 int err; 352 353 /* Command queue. */ 354 err = smmu_init_queue(sc, &sc->cmdq, 355 SMMU_CMDQ_PROD, SMMU_CMDQ_CONS, CMDQ_ENTRY_DWORDS); 356 if (err) 357 return (ENXIO); 358 359 /* Event queue. */ 360 err = smmu_init_queue(sc, &sc->evtq, 361 SMMU_EVENTQ_PROD, SMMU_EVENTQ_CONS, EVTQ_ENTRY_DWORDS); 362 if (err) 363 return (ENXIO); 364 365 if (!(sc->features & SMMU_FEATURE_PRI)) 366 return (0); 367 368 /* PRI queue. */ 369 err = smmu_init_queue(sc, &sc->priq, 370 SMMU_PRIQ_PROD, SMMU_PRIQ_CONS, PRIQ_ENTRY_DWORDS); 371 if (err) 372 return (ENXIO); 373 374 return (0); 375 } 376 377 /* 378 * Dump 2LVL or linear STE. 379 */ 380 static void 381 smmu_dump_ste(struct smmu_softc *sc, int sid) 382 { 383 struct smmu_strtab *strtab; 384 struct l1_desc *l1_desc; 385 uint64_t *ste, *l1; 386 int i; 387 388 strtab = &sc->strtab; 389 390 if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) { 391 i = sid >> STRTAB_SPLIT; 392 l1 = (void *)((uint64_t)strtab->vaddr + 393 STRTAB_L1_DESC_DWORDS * 8 * i); 394 device_printf(sc->dev, "L1 ste == %lx\n", l1[0]); 395 396 l1_desc = &strtab->l1[i]; 397 ste = l1_desc->va; 398 if (ste == NULL) /* L2 is not initialized */ 399 return; 400 } else { 401 ste = (void *)((uint64_t)strtab->vaddr + 402 sid * (STRTAB_STE_DWORDS << 3)); 403 } 404 405 /* Dump L2 or linear STE. */ 406 for (i = 0; i < STRTAB_STE_DWORDS; i++) 407 device_printf(sc->dev, "ste[%d] == %lx\n", i, ste[i]); 408 } 409 410 static void __unused 411 smmu_dump_cd(struct smmu_softc *sc, struct smmu_cd *cd) 412 { 413 uint64_t *vaddr; 414 int i; 415 416 device_printf(sc->dev, "%s\n", __func__); 417 418 vaddr = cd->vaddr; 419 for (i = 0; i < CD_DWORDS; i++) 420 device_printf(sc->dev, "cd[%d] == %lx\n", i, vaddr[i]); 421 } 422 423 static void 424 smmu_evtq_dequeue(struct smmu_softc *sc, uint32_t *evt) 425 { 426 struct smmu_queue *evtq; 427 void *entry_addr; 428 429 evtq = &sc->evtq; 430 431 evtq->lc.val = bus_read_8(sc->res[0], evtq->prod_off); 432 entry_addr = (void *)((uint64_t)evtq->vaddr + 433 evtq->lc.cons * EVTQ_ENTRY_DWORDS * 8); 434 memcpy(evt, entry_addr, EVTQ_ENTRY_DWORDS * 8); 435 evtq->lc.cons = smmu_q_inc_cons(evtq); 436 bus_write_4(sc->res[0], evtq->cons_off, evtq->lc.cons); 437 } 438 439 static void 440 smmu_print_event(struct smmu_softc *sc, uint32_t *evt) 441 { 442 struct smmu_event *ev; 443 uintptr_t input_addr; 444 uint8_t event_id; 445 device_t dev; 446 int sid; 447 int i; 448 449 dev = sc->dev; 450 451 ev = NULL; 452 event_id = evt[0] & 0xff; 453 for (i = 0; events[i].ident != 0; i++) { 454 if (events[i].ident == event_id) { 455 ev = &events[i]; 456 break; 457 } 458 } 459 460 sid = evt[1]; 461 input_addr = evt[5]; 462 input_addr <<= 32; 463 input_addr |= evt[4]; 464 465 if (smmu_quirks_check(dev, sid, event_id, input_addr)) { 466 /* The event is known. Don't print anything. */ 467 return; 468 } 469 470 if (ev) { 471 device_printf(sc->dev, 472 "Event %s (%s) received.\n", ev->str, ev->msg); 473 } else 474 device_printf(sc->dev, "Event 0x%x received\n", event_id); 475 476 device_printf(sc->dev, "SID %x, Input Address: %jx\n", 477 sid, input_addr); 478 479 for (i = 0; i < 8; i++) 480 device_printf(sc->dev, "evt[%d] %x\n", i, evt[i]); 481 482 smmu_dump_ste(sc, sid); 483 } 484 485 static void 486 make_cmd(struct smmu_softc *sc, uint64_t *cmd, 487 struct smmu_cmdq_entry *entry) 488 { 489 490 memset(cmd, 0, CMDQ_ENTRY_DWORDS * 8); 491 cmd[0] = entry->opcode << CMD_QUEUE_OPCODE_S; 492 493 switch (entry->opcode) { 494 case CMD_TLBI_NH_VA: 495 cmd[0] |= (uint64_t)entry->tlbi.asid << TLBI_0_ASID_S; 496 cmd[1] = entry->tlbi.addr & TLBI_1_ADDR_M; 497 if (entry->tlbi.leaf) { 498 /* 499 * Leaf flag means that only cached entries 500 * for the last level of translation table walk 501 * are required to be invalidated. 502 */ 503 cmd[1] |= TLBI_1_LEAF; 504 } 505 break; 506 case CMD_TLBI_NH_ASID: 507 cmd[0] |= (uint64_t)entry->tlbi.asid << TLBI_0_ASID_S; 508 break; 509 case CMD_TLBI_NSNH_ALL: 510 case CMD_TLBI_NH_ALL: 511 case CMD_TLBI_EL2_ALL: 512 break; 513 case CMD_CFGI_CD: 514 cmd[0] |= ((uint64_t)entry->cfgi.ssid << CFGI_0_SSID_S); 515 /* FALLTROUGH */ 516 case CMD_CFGI_STE: 517 cmd[0] |= ((uint64_t)entry->cfgi.sid << CFGI_0_STE_SID_S); 518 cmd[1] |= ((uint64_t)entry->cfgi.leaf << CFGI_1_LEAF_S); 519 break; 520 case CMD_CFGI_STE_RANGE: 521 cmd[1] = (31 << CFGI_1_STE_RANGE_S); 522 break; 523 case CMD_SYNC: 524 cmd[0] |= SYNC_0_MSH_IS | SYNC_0_MSIATTR_OIWB; 525 if (entry->sync.msiaddr) { 526 cmd[0] |= SYNC_0_CS_SIG_IRQ; 527 cmd[1] |= (entry->sync.msiaddr & SYNC_1_MSIADDRESS_M); 528 } else 529 cmd[0] |= SYNC_0_CS_SIG_SEV; 530 break; 531 case CMD_PREFETCH_CONFIG: 532 cmd[0] |= ((uint64_t)entry->prefetch.sid << PREFETCH_0_SID_S); 533 break; 534 }; 535 } 536 537 static void 538 smmu_cmdq_enqueue_cmd(struct smmu_softc *sc, struct smmu_cmdq_entry *entry) 539 { 540 uint64_t cmd[CMDQ_ENTRY_DWORDS]; 541 struct smmu_queue *cmdq; 542 void *entry_addr; 543 544 cmdq = &sc->cmdq; 545 546 make_cmd(sc, cmd, entry); 547 548 SMMU_LOCK(sc); 549 550 /* Ensure that a space is available. */ 551 do { 552 cmdq->lc.cons = bus_read_4(sc->res[0], cmdq->cons_off); 553 } while (smmu_q_has_space(cmdq) == 0); 554 555 /* Write the command to the current prod entry. */ 556 entry_addr = (void *)((uint64_t)cmdq->vaddr + 557 Q_IDX(cmdq, cmdq->lc.prod) * CMDQ_ENTRY_DWORDS * 8); 558 memcpy(entry_addr, cmd, CMDQ_ENTRY_DWORDS * 8); 559 560 /* Increment prod index. */ 561 cmdq->lc.prod = smmu_q_inc_prod(cmdq); 562 bus_write_4(sc->res[0], cmdq->prod_off, cmdq->lc.prod); 563 564 SMMU_UNLOCK(sc); 565 } 566 567 static void __unused 568 smmu_poll_until_consumed(struct smmu_softc *sc, struct smmu_queue *q) 569 { 570 571 while (1) { 572 q->lc.val = bus_read_8(sc->res[0], q->prod_off); 573 if (smmu_q_empty(q)) 574 break; 575 cpu_spinwait(); 576 } 577 } 578 579 static int 580 smmu_sync(struct smmu_softc *sc) 581 { 582 struct smmu_cmdq_entry cmd; 583 struct smmu_queue *q; 584 uint32_t *base; 585 int timeout; 586 int prod; 587 588 q = &sc->cmdq; 589 prod = q->lc.prod; 590 591 /* Enqueue sync command. */ 592 cmd.opcode = CMD_SYNC; 593 cmd.sync.msiaddr = q->paddr + Q_IDX(q, prod) * CMDQ_ENTRY_DWORDS * 8; 594 smmu_cmdq_enqueue_cmd(sc, &cmd); 595 596 /* Wait for the sync completion. */ 597 base = (void *)((uint64_t)q->vaddr + 598 Q_IDX(q, prod) * CMDQ_ENTRY_DWORDS * 8); 599 600 /* 601 * It takes around 200 loops (6 instructions each) 602 * on Neoverse N1 to complete the sync. 603 */ 604 timeout = 10000; 605 606 do { 607 if (*base == 0) { 608 /* MSI write completed. */ 609 break; 610 } 611 cpu_spinwait(); 612 } while (timeout--); 613 614 if (timeout < 0) 615 device_printf(sc->dev, "Failed to sync\n"); 616 617 return (0); 618 } 619 620 static int 621 smmu_sync_cd(struct smmu_softc *sc, int sid, int ssid, bool leaf) 622 { 623 struct smmu_cmdq_entry cmd; 624 625 cmd.opcode = CMD_CFGI_CD; 626 cmd.cfgi.sid = sid; 627 cmd.cfgi.ssid = ssid; 628 cmd.cfgi.leaf = leaf; 629 smmu_cmdq_enqueue_cmd(sc, &cmd); 630 631 return (0); 632 } 633 634 static void 635 smmu_invalidate_all_sid(struct smmu_softc *sc) 636 { 637 struct smmu_cmdq_entry cmd; 638 639 /* Invalidate cached config */ 640 cmd.opcode = CMD_CFGI_STE_RANGE; 641 smmu_cmdq_enqueue_cmd(sc, &cmd); 642 smmu_sync(sc); 643 } 644 645 static void 646 smmu_tlbi_all(struct smmu_softc *sc) 647 { 648 struct smmu_cmdq_entry cmd; 649 650 /* Invalidate entire TLB */ 651 cmd.opcode = CMD_TLBI_NSNH_ALL; 652 smmu_cmdq_enqueue_cmd(sc, &cmd); 653 smmu_sync(sc); 654 } 655 656 static void 657 smmu_tlbi_asid(struct smmu_softc *sc, uint16_t asid) 658 { 659 struct smmu_cmdq_entry cmd; 660 661 /* Invalidate TLB for an ASID. */ 662 cmd.opcode = CMD_TLBI_NH_ASID; 663 cmd.tlbi.asid = asid; 664 smmu_cmdq_enqueue_cmd(sc, &cmd); 665 smmu_sync(sc); 666 } 667 668 static void 669 smmu_tlbi_va(struct smmu_softc *sc, vm_offset_t va, uint16_t asid) 670 { 671 struct smmu_cmdq_entry cmd; 672 673 /* Invalidate specific range */ 674 cmd.opcode = CMD_TLBI_NH_VA; 675 cmd.tlbi.asid = asid; 676 cmd.tlbi.vmid = 0; 677 cmd.tlbi.leaf = true; /* We change only L3. */ 678 cmd.tlbi.addr = va; 679 smmu_cmdq_enqueue_cmd(sc, &cmd); 680 } 681 682 static void 683 smmu_invalidate_sid(struct smmu_softc *sc, uint32_t sid) 684 { 685 struct smmu_cmdq_entry cmd; 686 687 /* Invalidate cached config */ 688 cmd.opcode = CMD_CFGI_STE; 689 cmd.cfgi.sid = sid; 690 smmu_cmdq_enqueue_cmd(sc, &cmd); 691 smmu_sync(sc); 692 } 693 694 static void 695 smmu_prefetch_sid(struct smmu_softc *sc, uint32_t sid) 696 { 697 struct smmu_cmdq_entry cmd; 698 699 cmd.opcode = CMD_PREFETCH_CONFIG; 700 cmd.prefetch.sid = sid; 701 smmu_cmdq_enqueue_cmd(sc, &cmd); 702 smmu_sync(sc); 703 } 704 705 /* 706 * Init STE in bypass mode. Traffic is not translated for the sid. 707 */ 708 static void 709 smmu_init_ste_bypass(struct smmu_softc *sc, uint32_t sid, uint64_t *ste) 710 { 711 uint64_t val; 712 713 val = STE0_VALID | STE0_CONFIG_BYPASS; 714 715 ste[1] = STE1_SHCFG_INCOMING | STE1_EATS_FULLATS; 716 ste[2] = 0; 717 ste[3] = 0; 718 ste[4] = 0; 719 ste[5] = 0; 720 ste[6] = 0; 721 ste[7] = 0; 722 723 smmu_invalidate_sid(sc, sid); 724 ste[0] = val; 725 dsb(sy); 726 smmu_invalidate_sid(sc, sid); 727 728 smmu_prefetch_sid(sc, sid); 729 } 730 731 /* 732 * Enable Stage1 (S1) translation for the sid. 733 */ 734 static int 735 smmu_init_ste_s1(struct smmu_softc *sc, struct smmu_cd *cd, 736 uint32_t sid, uint64_t *ste) 737 { 738 uint64_t val; 739 740 val = STE0_VALID; 741 742 /* S1 */ 743 ste[1] = STE1_EATS_FULLATS | 744 STE1_S1CSH_IS | 745 STE1_S1CIR_WBRA | 746 STE1_S1COR_WBRA | 747 STE1_STRW_NS_EL1; 748 ste[2] = 0; 749 ste[3] = 0; 750 ste[4] = 0; 751 ste[5] = 0; 752 ste[6] = 0; 753 ste[7] = 0; 754 755 if (sc->features & SMMU_FEATURE_STALL && 756 ((sc->features & SMMU_FEATURE_STALL_FORCE) == 0)) 757 ste[1] |= STE1_S1STALLD; 758 759 /* Configure STE */ 760 val |= (cd->paddr & STE0_S1CONTEXTPTR_M); 761 val |= STE0_CONFIG_S1_TRANS; 762 763 smmu_invalidate_sid(sc, sid); 764 765 /* The STE[0] has to be written in a single blast, last of all. */ 766 ste[0] = val; 767 dsb(sy); 768 769 smmu_invalidate_sid(sc, sid); 770 smmu_sync_cd(sc, sid, 0, true); 771 smmu_invalidate_sid(sc, sid); 772 773 /* The sid will be used soon most likely. */ 774 smmu_prefetch_sid(sc, sid); 775 776 return (0); 777 } 778 779 static uint64_t * 780 smmu_get_ste_addr(struct smmu_softc *sc, int sid) 781 { 782 struct smmu_strtab *strtab; 783 struct l1_desc *l1_desc; 784 uint64_t *addr; 785 786 strtab = &sc->strtab; 787 788 if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) { 789 l1_desc = &strtab->l1[sid >> STRTAB_SPLIT]; 790 addr = l1_desc->va; 791 addr += (sid & ((1 << STRTAB_SPLIT) - 1)) * STRTAB_STE_DWORDS; 792 } else { 793 addr = (void *)((uint64_t)strtab->vaddr + 794 STRTAB_STE_DWORDS * 8 * sid); 795 }; 796 797 return (addr); 798 } 799 800 static int 801 smmu_init_ste(struct smmu_softc *sc, struct smmu_cd *cd, int sid, bool bypass) 802 { 803 uint64_t *addr; 804 805 addr = smmu_get_ste_addr(sc, sid); 806 807 if (bypass) 808 smmu_init_ste_bypass(sc, sid, addr); 809 else 810 smmu_init_ste_s1(sc, cd, sid, addr); 811 812 smmu_sync(sc); 813 814 return (0); 815 } 816 817 static void 818 smmu_deinit_ste(struct smmu_softc *sc, int sid) 819 { 820 uint64_t *ste; 821 822 ste = smmu_get_ste_addr(sc, sid); 823 ste[0] = 0; 824 825 smmu_invalidate_sid(sc, sid); 826 smmu_sync_cd(sc, sid, 0, true); 827 smmu_invalidate_sid(sc, sid); 828 829 smmu_sync(sc); 830 } 831 832 static int 833 smmu_init_cd(struct smmu_softc *sc, struct smmu_domain *domain) 834 { 835 vm_paddr_t paddr; 836 uint64_t *ptr; 837 uint64_t val; 838 vm_size_t size; 839 struct smmu_cd *cd; 840 struct smmu_pmap *p; 841 842 size = 1 * (CD_DWORDS << 3); 843 844 p = &domain->p; 845 cd = domain->cd = malloc(sizeof(struct smmu_cd), 846 M_SMMU, M_WAITOK | M_ZERO); 847 848 cd->vaddr = contigmalloc(size, M_SMMU, 849 M_WAITOK | M_ZERO, /* flags */ 850 0, /* low */ 851 (1ul << 40) - 1, /* high */ 852 size, /* alignment */ 853 0); /* boundary */ 854 if (cd->vaddr == NULL) { 855 device_printf(sc->dev, "Failed to allocate CD\n"); 856 return (ENXIO); 857 } 858 859 cd->size = size; 860 cd->paddr = vtophys(cd->vaddr); 861 862 ptr = cd->vaddr; 863 864 val = CD0_VALID; 865 val |= CD0_AA64; 866 val |= CD0_R; 867 val |= CD0_A; 868 val |= CD0_ASET; 869 val |= (uint64_t)domain->asid << CD0_ASID_S; 870 val |= CD0_TG0_4KB; 871 val |= CD0_EPD1; /* Disable TT1 */ 872 val |= ((64 - sc->ias) << CD0_T0SZ_S); 873 val |= CD0_IPS_48BITS; 874 875 paddr = p->sp_l0_paddr & CD1_TTB0_M; 876 KASSERT(paddr == p->sp_l0_paddr, ("bad allocation 1")); 877 878 ptr[1] = paddr; 879 ptr[2] = 0; 880 ptr[3] = MAIR_ATTR(MAIR_DEVICE_nGnRnE, VM_MEMATTR_DEVICE) | 881 MAIR_ATTR(MAIR_NORMAL_NC, VM_MEMATTR_UNCACHEABLE) | 882 MAIR_ATTR(MAIR_NORMAL_WB, VM_MEMATTR_WRITE_BACK) | 883 MAIR_ATTR(MAIR_NORMAL_WT, VM_MEMATTR_WRITE_THROUGH); 884 885 /* Install the CD. */ 886 ptr[0] = val; 887 888 return (0); 889 } 890 891 static int 892 smmu_init_strtab_linear(struct smmu_softc *sc) 893 { 894 struct smmu_strtab *strtab; 895 vm_paddr_t base; 896 uint32_t size; 897 uint64_t reg; 898 899 strtab = &sc->strtab; 900 strtab->num_l1_entries = (1 << sc->sid_bits); 901 902 size = strtab->num_l1_entries * (STRTAB_STE_DWORDS << 3); 903 904 if (bootverbose) 905 device_printf(sc->dev, 906 "%s: linear strtab size %d, num_l1_entries %d\n", 907 __func__, size, strtab->num_l1_entries); 908 909 strtab->vaddr = contigmalloc(size, M_SMMU, 910 M_WAITOK | M_ZERO, /* flags */ 911 0, /* low */ 912 (1ul << 48) - 1, /* high */ 913 size, /* alignment */ 914 0); /* boundary */ 915 if (strtab->vaddr == NULL) { 916 device_printf(sc->dev, "failed to allocate strtab\n"); 917 return (ENXIO); 918 } 919 920 reg = STRTAB_BASE_CFG_FMT_LINEAR; 921 reg |= sc->sid_bits << STRTAB_BASE_CFG_LOG2SIZE_S; 922 strtab->base_cfg = (uint32_t)reg; 923 924 base = vtophys(strtab->vaddr); 925 926 reg = base & STRTAB_BASE_ADDR_M; 927 KASSERT(reg == base, ("bad allocation 2")); 928 reg |= STRTAB_BASE_RA; 929 strtab->base = reg; 930 931 return (0); 932 } 933 934 static int 935 smmu_init_strtab_2lvl(struct smmu_softc *sc) 936 { 937 struct smmu_strtab *strtab; 938 vm_paddr_t base; 939 uint64_t reg_base; 940 uint32_t l1size; 941 uint32_t size; 942 uint32_t reg; 943 int sz; 944 945 strtab = &sc->strtab; 946 947 size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3); 948 size = min(size, sc->sid_bits - STRTAB_SPLIT); 949 strtab->num_l1_entries = (1 << size); 950 size += STRTAB_SPLIT; 951 952 l1size = strtab->num_l1_entries * (STRTAB_L1_DESC_DWORDS << 3); 953 954 if (bootverbose) 955 device_printf(sc->dev, 956 "%s: size %d, l1 entries %d, l1size %d\n", 957 __func__, size, strtab->num_l1_entries, l1size); 958 959 strtab->vaddr = contigmalloc(l1size, M_SMMU, 960 M_WAITOK | M_ZERO, /* flags */ 961 0, /* low */ 962 (1ul << 48) - 1, /* high */ 963 l1size, /* alignment */ 964 0); /* boundary */ 965 if (strtab->vaddr == NULL) { 966 device_printf(sc->dev, "Failed to allocate 2lvl strtab.\n"); 967 return (ENOMEM); 968 } 969 970 sz = strtab->num_l1_entries * sizeof(struct l1_desc); 971 972 strtab->l1 = malloc(sz, M_SMMU, M_WAITOK | M_ZERO); 973 if (strtab->l1 == NULL) { 974 contigfree(strtab->vaddr, l1size, M_SMMU); 975 return (ENOMEM); 976 } 977 978 reg = STRTAB_BASE_CFG_FMT_2LVL; 979 reg |= size << STRTAB_BASE_CFG_LOG2SIZE_S; 980 reg |= STRTAB_SPLIT << STRTAB_BASE_CFG_SPLIT_S; 981 strtab->base_cfg = (uint32_t)reg; 982 983 base = vtophys(strtab->vaddr); 984 985 reg_base = base & STRTAB_BASE_ADDR_M; 986 KASSERT(reg_base == base, ("bad allocation 3")); 987 reg_base |= STRTAB_BASE_RA; 988 strtab->base = reg_base; 989 990 return (0); 991 } 992 993 static int 994 smmu_init_strtab(struct smmu_softc *sc) 995 { 996 int error; 997 998 if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) 999 error = smmu_init_strtab_2lvl(sc); 1000 else 1001 error = smmu_init_strtab_linear(sc); 1002 1003 return (error); 1004 } 1005 1006 static int 1007 smmu_init_l1_entry(struct smmu_softc *sc, int sid) 1008 { 1009 struct smmu_strtab *strtab; 1010 struct l1_desc *l1_desc; 1011 uint64_t *addr; 1012 uint64_t val; 1013 size_t size; 1014 int i; 1015 1016 strtab = &sc->strtab; 1017 l1_desc = &strtab->l1[sid >> STRTAB_SPLIT]; 1018 if (l1_desc->va) { 1019 /* Already allocated. */ 1020 return (0); 1021 } 1022 1023 size = 1 << (STRTAB_SPLIT + ilog2(STRTAB_STE_DWORDS) + 3); 1024 1025 l1_desc->span = STRTAB_SPLIT + 1; 1026 l1_desc->size = size; 1027 l1_desc->va = contigmalloc(size, M_SMMU, 1028 M_WAITOK | M_ZERO, /* flags */ 1029 0, /* low */ 1030 (1ul << 48) - 1, /* high */ 1031 size, /* alignment */ 1032 0); /* boundary */ 1033 if (l1_desc->va == NULL) { 1034 device_printf(sc->dev, "failed to allocate l2 entry\n"); 1035 return (ENXIO); 1036 } 1037 1038 l1_desc->pa = vtophys(l1_desc->va); 1039 1040 i = sid >> STRTAB_SPLIT; 1041 addr = (void *)((uint64_t)strtab->vaddr + 1042 STRTAB_L1_DESC_DWORDS * 8 * i); 1043 1044 /* Install the L1 entry. */ 1045 val = l1_desc->pa & STRTAB_L1_DESC_L2PTR_M; 1046 KASSERT(val == l1_desc->pa, ("bad allocation 4")); 1047 val |= l1_desc->span; 1048 *addr = val; 1049 1050 return (0); 1051 } 1052 1053 static void __unused 1054 smmu_deinit_l1_entry(struct smmu_softc *sc, int sid) 1055 { 1056 struct smmu_strtab *strtab; 1057 struct l1_desc *l1_desc; 1058 uint64_t *addr; 1059 int i; 1060 1061 strtab = &sc->strtab; 1062 1063 i = sid >> STRTAB_SPLIT; 1064 addr = (void *)((uint64_t)strtab->vaddr + 1065 STRTAB_L1_DESC_DWORDS * 8 * i); 1066 *addr = 0; 1067 1068 l1_desc = &strtab->l1[sid >> STRTAB_SPLIT]; 1069 contigfree(l1_desc->va, l1_desc->size, M_SMMU); 1070 } 1071 1072 static int 1073 smmu_disable(struct smmu_softc *sc) 1074 { 1075 uint32_t reg; 1076 int error; 1077 1078 /* Disable SMMU */ 1079 reg = bus_read_4(sc->res[0], SMMU_CR0); 1080 reg &= ~CR0_SMMUEN; 1081 error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg); 1082 if (error) 1083 device_printf(sc->dev, "Could not disable SMMU.\n"); 1084 1085 return (0); 1086 } 1087 1088 static int 1089 smmu_event_intr(void *arg) 1090 { 1091 uint32_t evt[EVTQ_ENTRY_DWORDS * 2]; 1092 struct smmu_softc *sc; 1093 1094 sc = arg; 1095 1096 do { 1097 smmu_evtq_dequeue(sc, evt); 1098 smmu_print_event(sc, evt); 1099 } while (!smmu_q_empty(&sc->evtq)); 1100 1101 return (FILTER_HANDLED); 1102 } 1103 1104 static int __unused 1105 smmu_sync_intr(void *arg) 1106 { 1107 struct smmu_softc *sc; 1108 1109 sc = arg; 1110 1111 device_printf(sc->dev, "%s\n", __func__); 1112 1113 return (FILTER_HANDLED); 1114 } 1115 1116 static int 1117 smmu_gerr_intr(void *arg) 1118 { 1119 struct smmu_softc *sc; 1120 1121 sc = arg; 1122 1123 device_printf(sc->dev, "SMMU Global Error\n"); 1124 1125 return (FILTER_HANDLED); 1126 } 1127 1128 static int 1129 smmu_enable_interrupts(struct smmu_softc *sc) 1130 { 1131 uint32_t reg; 1132 int error; 1133 1134 /* Disable MSI. */ 1135 bus_write_8(sc->res[0], SMMU_GERROR_IRQ_CFG0, 0); 1136 bus_write_4(sc->res[0], SMMU_GERROR_IRQ_CFG1, 0); 1137 bus_write_4(sc->res[0], SMMU_GERROR_IRQ_CFG2, 0); 1138 1139 bus_write_8(sc->res[0], SMMU_EVENTQ_IRQ_CFG0, 0); 1140 bus_write_4(sc->res[0], SMMU_EVENTQ_IRQ_CFG1, 0); 1141 bus_write_4(sc->res[0], SMMU_EVENTQ_IRQ_CFG2, 0); 1142 1143 if (sc->features & CR0_PRIQEN) { 1144 bus_write_8(sc->res[0], SMMU_PRIQ_IRQ_CFG0, 0); 1145 bus_write_4(sc->res[0], SMMU_PRIQ_IRQ_CFG1, 0); 1146 bus_write_4(sc->res[0], SMMU_PRIQ_IRQ_CFG2, 0); 1147 } 1148 1149 /* Disable any interrupts. */ 1150 error = smmu_write_ack(sc, SMMU_IRQ_CTRL, SMMU_IRQ_CTRLACK, 0); 1151 if (error) { 1152 device_printf(sc->dev, "Could not disable interrupts.\n"); 1153 return (ENXIO); 1154 } 1155 1156 /* Enable interrupts. */ 1157 reg = IRQ_CTRL_EVENTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN; 1158 if (sc->features & SMMU_FEATURE_PRI) 1159 reg |= IRQ_CTRL_PRIQ_IRQEN; 1160 1161 error = smmu_write_ack(sc, SMMU_IRQ_CTRL, SMMU_IRQ_CTRLACK, reg); 1162 if (error) { 1163 device_printf(sc->dev, "Could not enable interrupts.\n"); 1164 return (ENXIO); 1165 } 1166 1167 return (0); 1168 } 1169 1170 #ifdef DEV_ACPI 1171 static void 1172 smmu_configure_intr(struct smmu_softc *sc, struct resource *res) 1173 { 1174 struct intr_map_data_acpi *ad; 1175 struct intr_map_data *data; 1176 1177 data = rman_get_virtual(res); 1178 KASSERT(data != NULL, ("data is NULL")); 1179 1180 if (data->type == INTR_MAP_DATA_ACPI) { 1181 ad = (struct intr_map_data_acpi *)data; 1182 ad->trig = INTR_TRIGGER_EDGE; 1183 ad->pol = INTR_POLARITY_HIGH; 1184 } 1185 } 1186 #endif 1187 1188 static int 1189 smmu_setup_interrupts(struct smmu_softc *sc) 1190 { 1191 device_t dev; 1192 int error; 1193 1194 dev = sc->dev; 1195 1196 #ifdef DEV_ACPI 1197 /* 1198 * Configure SMMU interrupts as EDGE triggered manually 1199 * as ACPI tables carries no information for that. 1200 */ 1201 smmu_configure_intr(sc, sc->res[1]); 1202 /* PRIQ is not in use. */ 1203 smmu_configure_intr(sc, sc->res[3]); 1204 smmu_configure_intr(sc, sc->res[4]); 1205 #endif 1206 1207 error = bus_setup_intr(dev, sc->res[1], INTR_TYPE_MISC, 1208 smmu_event_intr, NULL, sc, &sc->intr_cookie[0]); 1209 if (error) { 1210 device_printf(dev, "Couldn't setup Event interrupt handler\n"); 1211 return (ENXIO); 1212 } 1213 1214 error = bus_setup_intr(dev, sc->res[4], INTR_TYPE_MISC, 1215 smmu_gerr_intr, NULL, sc, &sc->intr_cookie[2]); 1216 if (error) { 1217 device_printf(dev, "Couldn't setup Gerr interrupt handler\n"); 1218 return (ENXIO); 1219 } 1220 1221 return (0); 1222 } 1223 1224 static int 1225 smmu_reset(struct smmu_softc *sc) 1226 { 1227 struct smmu_cmdq_entry cmd; 1228 struct smmu_strtab *strtab; 1229 int error; 1230 int reg; 1231 1232 reg = bus_read_4(sc->res[0], SMMU_CR0); 1233 1234 if (reg & CR0_SMMUEN) 1235 device_printf(sc->dev, 1236 "%s: Warning: SMMU is enabled\n", __func__); 1237 1238 error = smmu_disable(sc); 1239 if (error) 1240 device_printf(sc->dev, 1241 "%s: Could not disable SMMU.\n", __func__); 1242 1243 if (smmu_enable_interrupts(sc) != 0) { 1244 device_printf(sc->dev, "Could not enable interrupts.\n"); 1245 return (ENXIO); 1246 } 1247 1248 reg = CR1_TABLE_SH_IS | 1249 CR1_TABLE_OC_WBC | 1250 CR1_TABLE_IC_WBC | 1251 CR1_QUEUE_SH_IS | 1252 CR1_QUEUE_OC_WBC | 1253 CR1_QUEUE_IC_WBC; 1254 bus_write_4(sc->res[0], SMMU_CR1, reg); 1255 1256 reg = CR2_PTM | CR2_RECINVSID | CR2_E2H; 1257 bus_write_4(sc->res[0], SMMU_CR2, reg); 1258 1259 /* Stream table. */ 1260 strtab = &sc->strtab; 1261 bus_write_8(sc->res[0], SMMU_STRTAB_BASE, strtab->base); 1262 bus_write_4(sc->res[0], SMMU_STRTAB_BASE_CFG, strtab->base_cfg); 1263 1264 /* Command queue. */ 1265 bus_write_8(sc->res[0], SMMU_CMDQ_BASE, sc->cmdq.base); 1266 bus_write_4(sc->res[0], SMMU_CMDQ_PROD, sc->cmdq.lc.prod); 1267 bus_write_4(sc->res[0], SMMU_CMDQ_CONS, sc->cmdq.lc.cons); 1268 1269 reg = CR0_CMDQEN; 1270 error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg); 1271 if (error) { 1272 device_printf(sc->dev, "Could not enable command queue\n"); 1273 return (ENXIO); 1274 } 1275 1276 /* Invalidate cached configuration. */ 1277 smmu_invalidate_all_sid(sc); 1278 1279 if (sc->features & SMMU_FEATURE_HYP) { 1280 cmd.opcode = CMD_TLBI_EL2_ALL; 1281 smmu_cmdq_enqueue_cmd(sc, &cmd); 1282 }; 1283 1284 /* Invalidate TLB. */ 1285 smmu_tlbi_all(sc); 1286 1287 /* Event queue */ 1288 bus_write_8(sc->res[0], SMMU_EVENTQ_BASE, sc->evtq.base); 1289 bus_write_4(sc->res[0], SMMU_EVENTQ_PROD, sc->evtq.lc.prod); 1290 bus_write_4(sc->res[0], SMMU_EVENTQ_CONS, sc->evtq.lc.cons); 1291 1292 reg |= CR0_EVENTQEN; 1293 error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg); 1294 if (error) { 1295 device_printf(sc->dev, "Could not enable event queue\n"); 1296 return (ENXIO); 1297 } 1298 1299 if (sc->features & SMMU_FEATURE_PRI) { 1300 /* PRI queue */ 1301 bus_write_8(sc->res[0], SMMU_PRIQ_BASE, sc->priq.base); 1302 bus_write_4(sc->res[0], SMMU_PRIQ_PROD, sc->priq.lc.prod); 1303 bus_write_4(sc->res[0], SMMU_PRIQ_CONS, sc->priq.lc.cons); 1304 1305 reg |= CR0_PRIQEN; 1306 error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg); 1307 if (error) { 1308 device_printf(sc->dev, "Could not enable PRI queue\n"); 1309 return (ENXIO); 1310 } 1311 } 1312 1313 if (sc->features & SMMU_FEATURE_ATS) { 1314 reg |= CR0_ATSCHK; 1315 error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg); 1316 if (error) { 1317 device_printf(sc->dev, "Could not enable ATS check.\n"); 1318 return (ENXIO); 1319 } 1320 } 1321 1322 reg |= CR0_SMMUEN; 1323 error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg); 1324 if (error) { 1325 device_printf(sc->dev, "Could not enable SMMU.\n"); 1326 return (ENXIO); 1327 } 1328 1329 return (0); 1330 } 1331 1332 static int 1333 smmu_check_features(struct smmu_softc *sc) 1334 { 1335 uint32_t reg; 1336 uint32_t val; 1337 1338 sc->features = 0; 1339 1340 reg = bus_read_4(sc->res[0], SMMU_IDR0); 1341 1342 if (reg & IDR0_ST_LVL_2) { 1343 if (bootverbose) 1344 device_printf(sc->dev, 1345 "2-level stream table supported.\n"); 1346 sc->features |= SMMU_FEATURE_2_LVL_STREAM_TABLE; 1347 } 1348 1349 if (reg & IDR0_CD2L) { 1350 if (bootverbose) 1351 device_printf(sc->dev, 1352 "2-level CD table supported.\n"); 1353 sc->features |= SMMU_FEATURE_2_LVL_CD; 1354 } 1355 1356 switch (reg & IDR0_TTENDIAN_M) { 1357 case IDR0_TTENDIAN_MIXED: 1358 if (bootverbose) 1359 device_printf(sc->dev, "Mixed endianness supported.\n"); 1360 sc->features |= SMMU_FEATURE_TT_LE; 1361 sc->features |= SMMU_FEATURE_TT_BE; 1362 break; 1363 case IDR0_TTENDIAN_LITTLE: 1364 if (bootverbose) 1365 device_printf(sc->dev, 1366 "Little endian supported only.\n"); 1367 sc->features |= SMMU_FEATURE_TT_LE; 1368 break; 1369 case IDR0_TTENDIAN_BIG: 1370 if (bootverbose) 1371 device_printf(sc->dev, "Big endian supported only.\n"); 1372 sc->features |= SMMU_FEATURE_TT_BE; 1373 break; 1374 default: 1375 device_printf(sc->dev, "Unsupported endianness.\n"); 1376 return (ENXIO); 1377 } 1378 1379 if (reg & IDR0_SEV) 1380 sc->features |= SMMU_FEATURE_SEV; 1381 1382 if (reg & IDR0_MSI) { 1383 if (bootverbose) 1384 device_printf(sc->dev, "MSI feature present.\n"); 1385 sc->features |= SMMU_FEATURE_MSI; 1386 } 1387 1388 if (reg & IDR0_HYP) { 1389 if (bootverbose) 1390 device_printf(sc->dev, "HYP feature present.\n"); 1391 sc->features |= SMMU_FEATURE_HYP; 1392 } 1393 1394 if (reg & IDR0_ATS) 1395 sc->features |= SMMU_FEATURE_ATS; 1396 1397 if (reg & IDR0_PRI) 1398 sc->features |= SMMU_FEATURE_PRI; 1399 1400 switch (reg & IDR0_STALL_MODEL_M) { 1401 case IDR0_STALL_MODEL_FORCE: 1402 /* Stall is forced. */ 1403 sc->features |= SMMU_FEATURE_STALL_FORCE; 1404 /* FALLTHROUGH */ 1405 case IDR0_STALL_MODEL_STALL: 1406 sc->features |= SMMU_FEATURE_STALL; 1407 break; 1408 } 1409 1410 /* Grab translation stages supported. */ 1411 if (reg & IDR0_S1P) { 1412 if (bootverbose) 1413 device_printf(sc->dev, 1414 "Stage 1 translation supported.\n"); 1415 sc->features |= SMMU_FEATURE_S1P; 1416 } 1417 if (reg & IDR0_S2P) { 1418 if (bootverbose) 1419 device_printf(sc->dev, 1420 "Stage 2 translation supported.\n"); 1421 sc->features |= SMMU_FEATURE_S2P; 1422 } 1423 1424 switch (reg & IDR0_TTF_M) { 1425 case IDR0_TTF_ALL: 1426 case IDR0_TTF_AA64: 1427 sc->ias = 40; 1428 break; 1429 default: 1430 device_printf(sc->dev, "No AArch64 table format support.\n"); 1431 return (ENXIO); 1432 } 1433 1434 if (reg & IDR0_ASID16) 1435 sc->asid_bits = 16; 1436 else 1437 sc->asid_bits = 8; 1438 1439 if (bootverbose) 1440 device_printf(sc->dev, "ASID bits %d\n", sc->asid_bits); 1441 1442 if (reg & IDR0_VMID16) 1443 sc->vmid_bits = 16; 1444 else 1445 sc->vmid_bits = 8; 1446 1447 reg = bus_read_4(sc->res[0], SMMU_IDR1); 1448 1449 if (reg & (IDR1_TABLES_PRESET | IDR1_QUEUES_PRESET | IDR1_REL)) { 1450 device_printf(sc->dev, 1451 "Embedded implementations not supported by this driver.\n"); 1452 return (ENXIO); 1453 } 1454 1455 val = (reg & IDR1_CMDQS_M) >> IDR1_CMDQS_S; 1456 sc->cmdq.size_log2 = val; 1457 if (bootverbose) 1458 device_printf(sc->dev, "CMD queue bits %d\n", val); 1459 1460 val = (reg & IDR1_EVENTQS_M) >> IDR1_EVENTQS_S; 1461 sc->evtq.size_log2 = val; 1462 if (bootverbose) 1463 device_printf(sc->dev, "EVENT queue bits %d\n", val); 1464 1465 if (sc->features & SMMU_FEATURE_PRI) { 1466 val = (reg & IDR1_PRIQS_M) >> IDR1_PRIQS_S; 1467 sc->priq.size_log2 = val; 1468 if (bootverbose) 1469 device_printf(sc->dev, "PRI queue bits %d\n", val); 1470 } 1471 1472 sc->ssid_bits = (reg & IDR1_SSIDSIZE_M) >> IDR1_SSIDSIZE_S; 1473 sc->sid_bits = (reg & IDR1_SIDSIZE_M) >> IDR1_SIDSIZE_S; 1474 1475 if (sc->sid_bits <= STRTAB_SPLIT) 1476 sc->features &= ~SMMU_FEATURE_2_LVL_STREAM_TABLE; 1477 1478 if (bootverbose) { 1479 device_printf(sc->dev, "SSID bits %d\n", sc->ssid_bits); 1480 device_printf(sc->dev, "SID bits %d\n", sc->sid_bits); 1481 } 1482 1483 /* IDR3 */ 1484 reg = bus_read_4(sc->res[0], SMMU_IDR3); 1485 if (reg & IDR3_RIL) 1486 sc->features |= SMMU_FEATURE_RANGE_INV; 1487 1488 /* IDR5 */ 1489 reg = bus_read_4(sc->res[0], SMMU_IDR5); 1490 1491 switch (reg & IDR5_OAS_M) { 1492 case IDR5_OAS_32: 1493 sc->oas = 32; 1494 break; 1495 case IDR5_OAS_36: 1496 sc->oas = 36; 1497 break; 1498 case IDR5_OAS_40: 1499 sc->oas = 40; 1500 break; 1501 case IDR5_OAS_42: 1502 sc->oas = 42; 1503 break; 1504 case IDR5_OAS_44: 1505 sc->oas = 44; 1506 break; 1507 case IDR5_OAS_48: 1508 sc->oas = 48; 1509 break; 1510 case IDR5_OAS_52: 1511 sc->oas = 52; 1512 break; 1513 } 1514 1515 sc->pgsizes = 0; 1516 if (reg & IDR5_GRAN64K) 1517 sc->pgsizes |= 64 * 1024; 1518 if (reg & IDR5_GRAN16K) 1519 sc->pgsizes |= 16 * 1024; 1520 if (reg & IDR5_GRAN4K) 1521 sc->pgsizes |= 4 * 1024; 1522 1523 if ((reg & IDR5_VAX_M) == IDR5_VAX_52) 1524 sc->features |= SMMU_FEATURE_VAX; 1525 1526 return (0); 1527 } 1528 1529 static void 1530 smmu_init_asids(struct smmu_softc *sc) 1531 { 1532 1533 sc->asid_set_size = (1 << sc->asid_bits); 1534 sc->asid_set = bit_alloc(sc->asid_set_size, M_SMMU, M_WAITOK); 1535 mtx_init(&sc->asid_set_mutex, "asid set", NULL, MTX_SPIN); 1536 } 1537 1538 static int 1539 smmu_asid_alloc(struct smmu_softc *sc, int *new_asid) 1540 { 1541 1542 mtx_lock_spin(&sc->asid_set_mutex); 1543 bit_ffc(sc->asid_set, sc->asid_set_size, new_asid); 1544 if (*new_asid == -1) { 1545 mtx_unlock_spin(&sc->asid_set_mutex); 1546 return (ENOMEM); 1547 } 1548 bit_set(sc->asid_set, *new_asid); 1549 mtx_unlock_spin(&sc->asid_set_mutex); 1550 1551 return (0); 1552 } 1553 1554 static void 1555 smmu_asid_free(struct smmu_softc *sc, int asid) 1556 { 1557 1558 mtx_lock_spin(&sc->asid_set_mutex); 1559 bit_clear(sc->asid_set, asid); 1560 mtx_unlock_spin(&sc->asid_set_mutex); 1561 } 1562 1563 /* 1564 * Device interface. 1565 */ 1566 int 1567 smmu_attach(device_t dev) 1568 { 1569 struct smmu_softc *sc; 1570 int error; 1571 1572 sc = device_get_softc(dev); 1573 sc->dev = dev; 1574 1575 mtx_init(&sc->sc_mtx, device_get_nameunit(sc->dev), "smmu", MTX_DEF); 1576 1577 error = smmu_setup_interrupts(sc); 1578 if (error) { 1579 bus_release_resources(dev, smmu_spec, sc->res); 1580 return (ENXIO); 1581 } 1582 1583 error = smmu_check_features(sc); 1584 if (error) { 1585 device_printf(dev, "Some features are required " 1586 "but not supported by hardware.\n"); 1587 return (ENXIO); 1588 } 1589 1590 smmu_init_asids(sc); 1591 1592 error = smmu_init_queues(sc); 1593 if (error) { 1594 device_printf(dev, "Couldn't allocate queues.\n"); 1595 return (ENXIO); 1596 } 1597 1598 error = smmu_init_strtab(sc); 1599 if (error) { 1600 device_printf(dev, "Couldn't allocate strtab.\n"); 1601 return (ENXIO); 1602 } 1603 1604 error = smmu_reset(sc); 1605 if (error) { 1606 device_printf(dev, "Couldn't reset SMMU.\n"); 1607 return (ENXIO); 1608 } 1609 1610 return (0); 1611 } 1612 1613 int 1614 smmu_detach(device_t dev) 1615 { 1616 struct smmu_softc *sc; 1617 1618 sc = device_get_softc(dev); 1619 1620 bus_release_resources(dev, smmu_spec, sc->res); 1621 1622 return (0); 1623 } 1624 1625 static int 1626 smmu_read_ivar(device_t dev, device_t child, int which, uintptr_t *result) 1627 { 1628 struct smmu_softc *sc; 1629 1630 sc = device_get_softc(dev); 1631 1632 device_printf(sc->dev, "%s\n", __func__); 1633 1634 return (ENOENT); 1635 } 1636 1637 static int 1638 smmu_unmap(device_t dev, struct iommu_domain *iodom, 1639 vm_offset_t va, bus_size_t size) 1640 { 1641 struct smmu_domain *domain; 1642 struct smmu_softc *sc; 1643 int err; 1644 int i; 1645 1646 sc = device_get_softc(dev); 1647 1648 domain = (struct smmu_domain *)iodom; 1649 1650 err = 0; 1651 1652 dprintf("%s: %lx, %ld, domain %d\n", __func__, va, size, domain->asid); 1653 1654 for (i = 0; i < size; i += PAGE_SIZE) { 1655 if (smmu_pmap_remove(&domain->p, va) == 0) { 1656 /* pmap entry removed, invalidate TLB. */ 1657 smmu_tlbi_va(sc, va, domain->asid); 1658 } else { 1659 err = ENOENT; 1660 break; 1661 } 1662 va += PAGE_SIZE; 1663 } 1664 1665 smmu_sync(sc); 1666 1667 return (err); 1668 } 1669 1670 static int 1671 smmu_map(device_t dev, struct iommu_domain *iodom, 1672 vm_offset_t va, vm_page_t *ma, vm_size_t size, 1673 vm_prot_t prot) 1674 { 1675 struct smmu_domain *domain; 1676 struct smmu_softc *sc; 1677 vm_paddr_t pa; 1678 int error; 1679 int i; 1680 1681 sc = device_get_softc(dev); 1682 1683 domain = (struct smmu_domain *)iodom; 1684 1685 dprintf("%s: %lx -> %lx, %ld, domain %d\n", __func__, va, pa, size, 1686 domain->asid); 1687 1688 for (i = 0; size > 0; size -= PAGE_SIZE) { 1689 pa = VM_PAGE_TO_PHYS(ma[i++]); 1690 error = smmu_pmap_enter(&domain->p, va, pa, prot, 0); 1691 if (error) 1692 return (error); 1693 smmu_tlbi_va(sc, va, domain->asid); 1694 va += PAGE_SIZE; 1695 } 1696 1697 smmu_sync(sc); 1698 1699 return (0); 1700 } 1701 1702 static struct iommu_domain * 1703 smmu_domain_alloc(device_t dev, struct iommu_unit *iommu) 1704 { 1705 struct iommu_domain *iodom; 1706 struct smmu_domain *domain; 1707 struct smmu_unit *unit; 1708 struct smmu_softc *sc; 1709 int error; 1710 int new_asid; 1711 1712 sc = device_get_softc(dev); 1713 1714 unit = (struct smmu_unit *)iommu; 1715 1716 domain = malloc(sizeof(*domain), M_SMMU, M_WAITOK | M_ZERO); 1717 1718 error = smmu_asid_alloc(sc, &new_asid); 1719 if (error) { 1720 free(domain, M_SMMU); 1721 device_printf(sc->dev, 1722 "Could not allocate ASID for a new domain.\n"); 1723 return (NULL); 1724 } 1725 1726 domain->asid = (uint16_t)new_asid; 1727 1728 smmu_pmap_pinit(&domain->p); 1729 1730 error = smmu_init_cd(sc, domain); 1731 if (error) { 1732 free(domain, M_SMMU); 1733 device_printf(sc->dev, "Could not initialize CD\n"); 1734 return (NULL); 1735 } 1736 1737 smmu_tlbi_asid(sc, domain->asid); 1738 1739 LIST_INIT(&domain->ctx_list); 1740 1741 IOMMU_LOCK(iommu); 1742 LIST_INSERT_HEAD(&unit->domain_list, domain, next); 1743 IOMMU_UNLOCK(iommu); 1744 1745 iodom = &domain->iodom; 1746 1747 /* 1748 * Use 48-bit address space regardless of VAX bit 1749 * as we need 64k IOMMU_PAGE_SIZE for 52-bit space. 1750 */ 1751 iodom->end = MAXADDR_48BIT; 1752 1753 return (iodom); 1754 } 1755 1756 static void 1757 smmu_domain_free(device_t dev, struct iommu_domain *iodom) 1758 { 1759 struct smmu_domain *domain; 1760 struct smmu_softc *sc; 1761 struct smmu_cd *cd; 1762 1763 sc = device_get_softc(dev); 1764 1765 domain = (struct smmu_domain *)iodom; 1766 1767 LIST_REMOVE(domain, next); 1768 1769 cd = domain->cd; 1770 1771 smmu_pmap_remove_pages(&domain->p); 1772 smmu_pmap_release(&domain->p); 1773 1774 smmu_tlbi_asid(sc, domain->asid); 1775 smmu_asid_free(sc, domain->asid); 1776 1777 contigfree(cd->vaddr, cd->size, M_SMMU); 1778 free(cd, M_SMMU); 1779 1780 free(domain, M_SMMU); 1781 } 1782 1783 static int 1784 smmu_set_buswide(device_t dev, struct smmu_domain *domain, 1785 struct smmu_ctx *ctx) 1786 { 1787 struct smmu_softc *sc; 1788 int i; 1789 1790 sc = device_get_softc(dev); 1791 1792 for (i = 0; i < PCI_SLOTMAX; i++) 1793 smmu_init_ste(sc, domain->cd, (ctx->sid | i), ctx->bypass); 1794 1795 return (0); 1796 } 1797 1798 static int 1799 smmu_pci_get_sid(device_t child, u_int *xref0, u_int *sid0) 1800 { 1801 struct pci_id_ofw_iommu pi; 1802 int err; 1803 1804 err = pci_get_id(child, PCI_ID_OFW_IOMMU, (uintptr_t *)&pi); 1805 if (err == 0) { 1806 if (sid0) 1807 *sid0 = pi.id; 1808 if (xref0) 1809 *xref0 = pi.xref; 1810 } 1811 1812 return (err); 1813 } 1814 1815 static struct iommu_ctx * 1816 smmu_ctx_alloc(device_t dev, struct iommu_domain *iodom, device_t child, 1817 bool disabled) 1818 { 1819 struct smmu_domain *domain; 1820 struct smmu_ctx *ctx; 1821 1822 domain = (struct smmu_domain *)iodom; 1823 1824 ctx = malloc(sizeof(struct smmu_ctx), M_SMMU, M_WAITOK | M_ZERO); 1825 ctx->dev = child; 1826 ctx->domain = domain; 1827 if (disabled) 1828 ctx->bypass = true; 1829 1830 IOMMU_DOMAIN_LOCK(iodom); 1831 LIST_INSERT_HEAD(&domain->ctx_list, ctx, next); 1832 IOMMU_DOMAIN_UNLOCK(iodom); 1833 1834 return (&ctx->ioctx); 1835 } 1836 1837 static int 1838 smmu_ctx_init(device_t dev, struct iommu_ctx *ioctx) 1839 { 1840 struct smmu_domain *domain; 1841 struct iommu_domain *iodom; 1842 struct smmu_softc *sc; 1843 struct smmu_ctx *ctx; 1844 devclass_t pci_class; 1845 u_int sid; 1846 int err; 1847 1848 ctx = (struct smmu_ctx *)ioctx; 1849 1850 sc = device_get_softc(dev); 1851 1852 domain = ctx->domain; 1853 iodom = (struct iommu_domain *)domain; 1854 1855 pci_class = devclass_find("pci"); 1856 if (device_get_devclass(device_get_parent(ctx->dev)) == pci_class) { 1857 err = smmu_pci_get_sid(ctx->dev, NULL, &sid); 1858 if (err) 1859 return (err); 1860 1861 ioctx->rid = pci_get_rid(dev); 1862 ctx->sid = sid; 1863 ctx->vendor = pci_get_vendor(ctx->dev); 1864 ctx->device = pci_get_device(ctx->dev); 1865 } 1866 1867 if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) { 1868 err = smmu_init_l1_entry(sc, ctx->sid); 1869 if (err) 1870 return (err); 1871 } 1872 1873 /* 1874 * Neoverse N1 SDP: 1875 * 0x800 xhci 1876 * 0x700 re 1877 * 0x600 sata 1878 */ 1879 1880 smmu_init_ste(sc, domain->cd, ctx->sid, ctx->bypass); 1881 1882 if (device_get_devclass(device_get_parent(ctx->dev)) == pci_class) 1883 if (iommu_is_buswide_ctx(iodom->iommu, pci_get_bus(ctx->dev))) 1884 smmu_set_buswide(dev, domain, ctx); 1885 1886 return (0); 1887 } 1888 1889 static void 1890 smmu_ctx_free(device_t dev, struct iommu_ctx *ioctx) 1891 { 1892 struct smmu_softc *sc; 1893 struct smmu_ctx *ctx; 1894 1895 IOMMU_ASSERT_LOCKED(ioctx->domain->iommu); 1896 1897 sc = device_get_softc(dev); 1898 ctx = (struct smmu_ctx *)ioctx; 1899 1900 smmu_deinit_ste(sc, ctx->sid); 1901 1902 LIST_REMOVE(ctx, next); 1903 1904 free(ctx, M_SMMU); 1905 } 1906 1907 struct smmu_ctx * 1908 smmu_ctx_lookup_by_sid(device_t dev, u_int sid) 1909 { 1910 struct smmu_softc *sc; 1911 struct smmu_domain *domain; 1912 struct smmu_unit *unit; 1913 struct smmu_ctx *ctx; 1914 1915 sc = device_get_softc(dev); 1916 1917 unit = &sc->unit; 1918 1919 LIST_FOREACH(domain, &unit->domain_list, next) { 1920 LIST_FOREACH(ctx, &domain->ctx_list, next) { 1921 if (ctx->sid == sid) 1922 return (ctx); 1923 } 1924 } 1925 1926 return (NULL); 1927 } 1928 1929 static struct iommu_ctx * 1930 smmu_ctx_lookup(device_t dev, device_t child) 1931 { 1932 struct iommu_unit *iommu __diagused; 1933 struct smmu_softc *sc; 1934 struct smmu_domain *domain; 1935 struct smmu_unit *unit; 1936 struct smmu_ctx *ctx; 1937 1938 sc = device_get_softc(dev); 1939 1940 unit = &sc->unit; 1941 iommu = &unit->iommu; 1942 1943 IOMMU_ASSERT_LOCKED(iommu); 1944 1945 LIST_FOREACH(domain, &unit->domain_list, next) { 1946 IOMMU_DOMAIN_LOCK(&domain->iodom); 1947 LIST_FOREACH(ctx, &domain->ctx_list, next) { 1948 if (ctx->dev == child) { 1949 IOMMU_DOMAIN_UNLOCK(&domain->iodom); 1950 return (&ctx->ioctx); 1951 } 1952 } 1953 IOMMU_DOMAIN_UNLOCK(&domain->iodom); 1954 } 1955 1956 return (NULL); 1957 } 1958 1959 static int 1960 smmu_find(device_t dev, device_t child) 1961 { 1962 struct smmu_softc *sc; 1963 u_int xref; 1964 int err; 1965 1966 sc = device_get_softc(dev); 1967 1968 err = smmu_pci_get_sid(child, &xref, NULL); 1969 if (err) 1970 return (ENOENT); 1971 1972 /* Check if xref is ours. */ 1973 if (xref != sc->xref) 1974 return (EFAULT); 1975 1976 return (0); 1977 } 1978 1979 #ifdef FDT 1980 static int 1981 smmu_ofw_md_data(device_t dev, struct iommu_ctx *ioctx, pcell_t *cells, 1982 int ncells) 1983 { 1984 struct smmu_ctx *ctx; 1985 1986 ctx = (struct smmu_ctx *)ioctx; 1987 1988 if (ncells != 1) 1989 return (-1); 1990 1991 ctx->sid = cells[0]; 1992 1993 return (0); 1994 } 1995 #endif 1996 1997 static device_method_t smmu_methods[] = { 1998 /* Device interface */ 1999 DEVMETHOD(device_detach, smmu_detach), 2000 2001 /* SMMU interface */ 2002 DEVMETHOD(iommu_find, smmu_find), 2003 DEVMETHOD(iommu_map, smmu_map), 2004 DEVMETHOD(iommu_unmap, smmu_unmap), 2005 DEVMETHOD(iommu_domain_alloc, smmu_domain_alloc), 2006 DEVMETHOD(iommu_domain_free, smmu_domain_free), 2007 DEVMETHOD(iommu_ctx_alloc, smmu_ctx_alloc), 2008 DEVMETHOD(iommu_ctx_init, smmu_ctx_init), 2009 DEVMETHOD(iommu_ctx_free, smmu_ctx_free), 2010 DEVMETHOD(iommu_ctx_lookup, smmu_ctx_lookup), 2011 #ifdef FDT 2012 DEVMETHOD(iommu_ofw_md_data, smmu_ofw_md_data), 2013 #endif 2014 2015 /* Bus interface */ 2016 DEVMETHOD(bus_read_ivar, smmu_read_ivar), 2017 2018 /* End */ 2019 DEVMETHOD_END 2020 }; 2021 2022 DEFINE_CLASS_0(smmu, smmu_driver, smmu_methods, sizeof(struct smmu_softc)); 2023