1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2019-2020 Ruslan Bukin <br@bsdpad.com> 5 * 6 * This software was developed by SRI International and the University of 7 * Cambridge Computer Laboratory (Department of Computer Science and 8 * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the 9 * DARPA SSITH research programme. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 */ 32 33 /* 34 * Hardware overview. 35 * 36 * An incoming transaction from a peripheral device has an address, size, 37 * attributes and StreamID. 38 * 39 * In case of PCI-based devices, StreamID is a PCI rid. 40 * 41 * The StreamID is used to select a Stream Table Entry (STE) in a Stream table, 42 * which contains per-device configuration. 43 * 44 * Stream table is a linear or 2-level walk table (this driver supports both). 45 * Note that a linear table could occupy 1GB or more of memory depending on 46 * sid_bits value. 47 * 48 * STE is used to locate a Context Descriptor, which is a struct in memory 49 * that describes stages of translation, translation table type, pointer to 50 * level 0 of page tables, ASID, etc. 51 * 52 * Hardware supports two stages of translation: Stage1 (S1) and Stage2 (S2): 53 * o S1 is used for the host machine traffic translation 54 * o S2 is for a hypervisor 55 * 56 * This driver enables S1 stage with standard AArch64 page tables. 57 * 58 * Note that SMMU does not share TLB with a main CPU. 59 * Command queue is used by this driver to Invalidate SMMU TLB, STE cache. 60 * 61 * An arm64 SoC could have more than one SMMU instance. 62 * ACPI IORT table describes which SMMU unit is assigned for a particular 63 * peripheral device. 64 * 65 * Queues. 66 * 67 * Register interface and Memory-based circular buffer queues are used 68 * to interface SMMU. 69 * 70 * These are a Command queue for commands to send to the SMMU and an Event 71 * queue for event/fault reports from the SMMU. Optionally PRI queue is 72 * designed for PCIe page requests reception. 73 * 74 * Note that not every hardware supports PRI services. For instance they were 75 * not found in Neoverse N1 SDP machine. 76 * (This drivers does not implement PRI queue.) 77 * 78 * All SMMU queues are arranged as circular buffers in memory. They are used 79 * in a producer-consumer fashion so that an output queue contains data 80 * produced by the SMMU and consumed by software. 81 * An input queue contains data produced by software, consumed by the SMMU. 82 * 83 * Interrupts. 84 * 85 * Interrupts are not required by this driver for normal operation. 86 * The standard wired interrupt is only triggered when an event comes from 87 * the SMMU, which is only in a case of errors (e.g. translation fault). 88 */ 89 90 #include "opt_platform.h" 91 #include "opt_acpi.h" 92 93 #include <sys/param.h> 94 #include <sys/bitstring.h> 95 #include <sys/bus.h> 96 #include <sys/kernel.h> 97 #include <sys/malloc.h> 98 #include <sys/mutex.h> 99 #include <sys/rman.h> 100 #include <sys/lock.h> 101 #include <sys/sysctl.h> 102 #include <sys/tree.h> 103 #include <sys/taskqueue.h> 104 #include <vm/vm.h> 105 #include <vm/vm_page.h> 106 #ifdef DEV_ACPI 107 #include <contrib/dev/acpica/include/acpi.h> 108 #include <dev/acpica/acpivar.h> 109 #endif 110 #include <dev/pci/pcireg.h> 111 #include <dev/pci/pcivar.h> 112 #include <dev/iommu/iommu.h> 113 #include <arm64/iommu/iommu_pmap.h> 114 115 #include <machine/bus.h> 116 117 #ifdef FDT 118 #include <dev/fdt/fdt_common.h> 119 #include <dev/ofw/ofw_bus.h> 120 #include <dev/ofw/ofw_bus_subr.h> 121 #endif 122 123 #include "iommu.h" 124 #include "iommu_if.h" 125 126 #include "smmureg.h" 127 #include "smmuvar.h" 128 129 #define STRTAB_L1_SZ_SHIFT 20 130 #define STRTAB_SPLIT 8 131 132 #define STRTAB_L1_DESC_L2PTR_M (0x3fffffffffff << 6) 133 #define STRTAB_L1_DESC_DWORDS 1 134 135 #define STRTAB_STE_DWORDS 8 136 137 #define CMDQ_ENTRY_DWORDS 2 138 #define EVTQ_ENTRY_DWORDS 4 139 #define PRIQ_ENTRY_DWORDS 2 140 141 #define CD_DWORDS 8 142 143 #define Q_WRP(q, p) ((p) & (1 << (q)->size_log2)) 144 #define Q_IDX(q, p) ((p) & ((1 << (q)->size_log2) - 1)) 145 #define Q_OVF(p) ((p) & (1 << 31)) /* Event queue overflowed */ 146 147 #define SMMU_Q_ALIGN (64 * 1024) 148 149 #define MAXADDR_48BIT 0xFFFFFFFFFFFFUL 150 #define MAXADDR_52BIT 0xFFFFFFFFFFFFFUL 151 152 static struct resource_spec smmu_spec[] = { 153 { SYS_RES_MEMORY, 0, RF_ACTIVE }, 154 { SYS_RES_IRQ, 0, RF_ACTIVE }, 155 { SYS_RES_IRQ, 1, RF_ACTIVE | RF_OPTIONAL }, 156 { SYS_RES_IRQ, 2, RF_ACTIVE }, 157 { SYS_RES_IRQ, 3, RF_ACTIVE }, 158 RESOURCE_SPEC_END 159 }; 160 161 MALLOC_DEFINE(M_SMMU, "SMMU", SMMU_DEVSTR); 162 163 #define dprintf(fmt, ...) 164 165 struct smmu_event { 166 int ident; 167 char *str; 168 char *msg; 169 }; 170 171 static struct smmu_event events[] = { 172 { 0x01, "F_UUT", 173 "Unsupported Upstream Transaction."}, 174 { 0x02, "C_BAD_STREAMID", 175 "Transaction StreamID out of range."}, 176 { 0x03, "F_STE_FETCH", 177 "Fetch of STE caused external abort."}, 178 { 0x04, "C_BAD_STE", 179 "Used STE invalid."}, 180 { 0x05, "F_BAD_ATS_TREQ", 181 "Address Translation Request disallowed for a StreamID " 182 "and a PCIe ATS Translation Request received."}, 183 { 0x06, "F_STREAM_DISABLED", 184 "The STE of a transaction marks non-substream transactions " 185 "disabled."}, 186 { 0x07, "F_TRANSL_FORBIDDEN", 187 "An incoming PCIe transaction is marked Translated but " 188 "SMMU bypass is disallowed for this StreamID."}, 189 { 0x08, "C_BAD_SUBSTREAMID", 190 "Incoming SubstreamID present, but configuration is invalid."}, 191 { 0x09, "F_CD_FETCH", 192 "Fetch of CD caused external abort."}, 193 { 0x0a, "C_BAD_CD", 194 "Fetched CD invalid."}, 195 { 0x0b, "F_WALK_EABT", 196 "An external abort occurred fetching (or updating) " 197 "a translation table descriptor."}, 198 { 0x10, "F_TRANSLATION", 199 "Translation fault."}, 200 { 0x11, "F_ADDR_SIZE", 201 "Address Size fault."}, 202 { 0x12, "F_ACCESS", 203 "Access flag fault due to AF == 0 in a page or block TTD."}, 204 { 0x13, "F_PERMISSION", 205 "Permission fault occurred on page access."}, 206 { 0x20, "F_TLB_CONFLICT", 207 "A TLB conflict occurred because of the transaction."}, 208 { 0x21, "F_CFG_CONFLICT", 209 "A configuration cache conflict occurred due to " 210 "the transaction."}, 211 { 0x24, "E_PAGE_REQUEST", 212 "Speculative page request hint."}, 213 { 0x25, "F_VMS_FETCH", 214 "Fetch of VMS caused external abort."}, 215 { 0, NULL, NULL }, 216 }; 217 218 static int 219 smmu_q_has_space(struct smmu_queue *q) 220 { 221 222 /* 223 * See 6.3.27 SMMU_CMDQ_PROD 224 * 225 * There is space in the queue for additional commands if: 226 * SMMU_CMDQ_CONS.RD != SMMU_CMDQ_PROD.WR || 227 * SMMU_CMDQ_CONS.RD_WRAP == SMMU_CMDQ_PROD.WR_WRAP 228 */ 229 230 if (Q_IDX(q, q->lc.cons) != Q_IDX(q, q->lc.prod) || 231 Q_WRP(q, q->lc.cons) == Q_WRP(q, q->lc.prod)) 232 return (1); 233 234 return (0); 235 } 236 237 static int 238 smmu_q_empty(struct smmu_queue *q) 239 { 240 241 if (Q_IDX(q, q->lc.cons) == Q_IDX(q, q->lc.prod) && 242 Q_WRP(q, q->lc.cons) == Q_WRP(q, q->lc.prod)) 243 return (1); 244 245 return (0); 246 } 247 248 static int __unused 249 smmu_q_consumed(struct smmu_queue *q, uint32_t prod) 250 { 251 252 if ((Q_WRP(q, q->lc.cons) == Q_WRP(q, prod)) && 253 (Q_IDX(q, q->lc.cons) >= Q_IDX(q, prod))) 254 return (1); 255 256 if ((Q_WRP(q, q->lc.cons) != Q_WRP(q, prod)) && 257 (Q_IDX(q, q->lc.cons) <= Q_IDX(q, prod))) 258 return (1); 259 260 return (0); 261 } 262 263 static uint32_t 264 smmu_q_inc_cons(struct smmu_queue *q) 265 { 266 uint32_t cons; 267 uint32_t val; 268 269 cons = (Q_WRP(q, q->lc.cons) | Q_IDX(q, q->lc.cons)) + 1; 270 val = (Q_OVF(q->lc.cons) | Q_WRP(q, cons) | Q_IDX(q, cons)); 271 272 return (val); 273 } 274 275 static uint32_t 276 smmu_q_inc_prod(struct smmu_queue *q) 277 { 278 uint32_t prod; 279 uint32_t val; 280 281 prod = (Q_WRP(q, q->lc.prod) | Q_IDX(q, q->lc.prod)) + 1; 282 val = (Q_OVF(q->lc.prod) | Q_WRP(q, prod) | Q_IDX(q, prod)); 283 284 return (val); 285 } 286 287 static int 288 smmu_write_ack(struct smmu_softc *sc, uint32_t reg, 289 uint32_t reg_ack, uint32_t val) 290 { 291 uint32_t v; 292 int timeout; 293 294 timeout = 100000; 295 296 bus_write_4(sc->res[0], reg, val); 297 298 do { 299 v = bus_read_4(sc->res[0], reg_ack); 300 if (v == val) 301 break; 302 } while (timeout--); 303 304 if (timeout <= 0) { 305 device_printf(sc->dev, "Failed to write reg.\n"); 306 return (-1); 307 } 308 309 return (0); 310 } 311 312 static int 313 smmu_init_queue(struct smmu_softc *sc, struct smmu_queue *q, 314 uint32_t prod_off, uint32_t cons_off, uint32_t dwords) 315 { 316 int sz; 317 318 sz = (1 << q->size_log2) * dwords * 8; 319 320 /* Set up the command circular buffer */ 321 q->vaddr = contigmalloc(sz, M_SMMU, 322 M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, SMMU_Q_ALIGN, 0); 323 if (q->vaddr == NULL) { 324 device_printf(sc->dev, "failed to allocate %d bytes\n", sz); 325 return (-1); 326 } 327 328 q->prod_off = prod_off; 329 q->cons_off = cons_off; 330 q->paddr = vtophys(q->vaddr); 331 332 q->base = CMDQ_BASE_RA | EVENTQ_BASE_WA | PRIQ_BASE_WA; 333 q->base |= q->paddr & Q_BASE_ADDR_M; 334 q->base |= q->size_log2 << Q_LOG2SIZE_S; 335 336 return (0); 337 } 338 339 static int 340 smmu_init_queues(struct smmu_softc *sc) 341 { 342 int err; 343 344 /* Command queue. */ 345 err = smmu_init_queue(sc, &sc->cmdq, 346 SMMU_CMDQ_PROD, SMMU_CMDQ_CONS, CMDQ_ENTRY_DWORDS); 347 if (err) 348 return (ENXIO); 349 350 /* Event queue. */ 351 err = smmu_init_queue(sc, &sc->evtq, 352 SMMU_EVENTQ_PROD, SMMU_EVENTQ_CONS, EVTQ_ENTRY_DWORDS); 353 if (err) 354 return (ENXIO); 355 356 if (!(sc->features & SMMU_FEATURE_PRI)) 357 return (0); 358 359 /* PRI queue. */ 360 err = smmu_init_queue(sc, &sc->priq, 361 SMMU_PRIQ_PROD, SMMU_PRIQ_CONS, PRIQ_ENTRY_DWORDS); 362 if (err) 363 return (ENXIO); 364 365 return (0); 366 } 367 368 /* 369 * Dump 2LVL or linear STE. 370 */ 371 static void 372 smmu_dump_ste(struct smmu_softc *sc, int sid) 373 { 374 struct smmu_strtab *strtab; 375 struct l1_desc *l1_desc; 376 uint64_t *ste, *l1; 377 int i; 378 379 strtab = &sc->strtab; 380 381 if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) { 382 i = sid >> STRTAB_SPLIT; 383 l1 = (void *)((uint64_t)strtab->vaddr + 384 STRTAB_L1_DESC_DWORDS * 8 * i); 385 device_printf(sc->dev, "L1 ste == %lx\n", l1[0]); 386 387 l1_desc = &strtab->l1[i]; 388 ste = l1_desc->va; 389 if (ste == NULL) /* L2 is not initialized */ 390 return; 391 } else { 392 ste = (void *)((uint64_t)strtab->vaddr + 393 sid * (STRTAB_STE_DWORDS << 3)); 394 } 395 396 /* Dump L2 or linear STE. */ 397 for (i = 0; i < STRTAB_STE_DWORDS; i++) 398 device_printf(sc->dev, "ste[%d] == %lx\n", i, ste[i]); 399 } 400 401 static void __unused 402 smmu_dump_cd(struct smmu_softc *sc, struct smmu_cd *cd) 403 { 404 uint64_t *vaddr; 405 int i; 406 407 device_printf(sc->dev, "%s\n", __func__); 408 409 vaddr = cd->vaddr; 410 for (i = 0; i < CD_DWORDS; i++) 411 device_printf(sc->dev, "cd[%d] == %lx\n", i, vaddr[i]); 412 } 413 414 static void 415 smmu_evtq_dequeue(struct smmu_softc *sc, uint32_t *evt) 416 { 417 struct smmu_queue *evtq; 418 void *entry_addr; 419 420 evtq = &sc->evtq; 421 422 evtq->lc.val = bus_read_8(sc->res[0], evtq->prod_off); 423 entry_addr = (void *)((uint64_t)evtq->vaddr + 424 evtq->lc.cons * EVTQ_ENTRY_DWORDS * 8); 425 memcpy(evt, entry_addr, EVTQ_ENTRY_DWORDS * 8); 426 evtq->lc.cons = smmu_q_inc_cons(evtq); 427 bus_write_4(sc->res[0], evtq->cons_off, evtq->lc.cons); 428 } 429 430 static void 431 smmu_print_event(struct smmu_softc *sc, uint32_t *evt) 432 { 433 struct smmu_event *ev; 434 uintptr_t input_addr; 435 uint8_t event_id; 436 device_t dev; 437 int sid; 438 int i; 439 440 dev = sc->dev; 441 442 ev = NULL; 443 event_id = evt[0] & 0xff; 444 for (i = 0; events[i].ident != 0; i++) { 445 if (events[i].ident == event_id) { 446 ev = &events[i]; 447 break; 448 } 449 } 450 451 sid = evt[1]; 452 input_addr = evt[5]; 453 input_addr <<= 32; 454 input_addr |= evt[4]; 455 456 if (smmu_quirks_check(dev, sid, event_id, input_addr)) { 457 /* The event is known. Don't print anything. */ 458 return; 459 } 460 461 if (ev) { 462 device_printf(sc->dev, 463 "Event %s (%s) received.\n", ev->str, ev->msg); 464 } else 465 device_printf(sc->dev, "Event 0x%x received\n", event_id); 466 467 device_printf(sc->dev, "SID %x, Input Address: %jx\n", 468 sid, input_addr); 469 470 for (i = 0; i < 8; i++) 471 device_printf(sc->dev, "evt[%d] %x\n", i, evt[i]); 472 473 smmu_dump_ste(sc, sid); 474 } 475 476 static void 477 make_cmd(struct smmu_softc *sc, uint64_t *cmd, 478 struct smmu_cmdq_entry *entry) 479 { 480 481 memset(cmd, 0, CMDQ_ENTRY_DWORDS * 8); 482 cmd[0] = entry->opcode << CMD_QUEUE_OPCODE_S; 483 484 switch (entry->opcode) { 485 case CMD_TLBI_NH_VA: 486 cmd[0] |= (uint64_t)entry->tlbi.asid << TLBI_0_ASID_S; 487 cmd[1] = entry->tlbi.addr & TLBI_1_ADDR_M; 488 if (entry->tlbi.leaf) { 489 /* 490 * Leaf flag means that only cached entries 491 * for the last level of translation table walk 492 * are required to be invalidated. 493 */ 494 cmd[1] |= TLBI_1_LEAF; 495 } 496 break; 497 case CMD_TLBI_NH_ASID: 498 cmd[0] |= (uint64_t)entry->tlbi.asid << TLBI_0_ASID_S; 499 break; 500 case CMD_TLBI_NSNH_ALL: 501 case CMD_TLBI_NH_ALL: 502 case CMD_TLBI_EL2_ALL: 503 break; 504 case CMD_CFGI_CD: 505 cmd[0] |= ((uint64_t)entry->cfgi.ssid << CFGI_0_SSID_S); 506 /* FALLTROUGH */ 507 case CMD_CFGI_STE: 508 cmd[0] |= ((uint64_t)entry->cfgi.sid << CFGI_0_STE_SID_S); 509 cmd[1] |= ((uint64_t)entry->cfgi.leaf << CFGI_1_LEAF_S); 510 break; 511 case CMD_CFGI_STE_RANGE: 512 cmd[1] = (31 << CFGI_1_STE_RANGE_S); 513 break; 514 case CMD_SYNC: 515 cmd[0] |= SYNC_0_MSH_IS | SYNC_0_MSIATTR_OIWB; 516 if (entry->sync.msiaddr) { 517 cmd[0] |= SYNC_0_CS_SIG_IRQ; 518 cmd[1] |= (entry->sync.msiaddr & SYNC_1_MSIADDRESS_M); 519 } else 520 cmd[0] |= SYNC_0_CS_SIG_SEV; 521 break; 522 case CMD_PREFETCH_CONFIG: 523 cmd[0] |= ((uint64_t)entry->prefetch.sid << PREFETCH_0_SID_S); 524 break; 525 }; 526 } 527 528 static void 529 smmu_cmdq_enqueue_cmd(struct smmu_softc *sc, struct smmu_cmdq_entry *entry) 530 { 531 uint64_t cmd[CMDQ_ENTRY_DWORDS]; 532 struct smmu_queue *cmdq; 533 void *entry_addr; 534 535 cmdq = &sc->cmdq; 536 537 make_cmd(sc, cmd, entry); 538 539 SMMU_LOCK(sc); 540 541 /* Ensure that a space is available. */ 542 do { 543 cmdq->lc.cons = bus_read_4(sc->res[0], cmdq->cons_off); 544 } while (smmu_q_has_space(cmdq) == 0); 545 546 /* Write the command to the current prod entry. */ 547 entry_addr = (void *)((uint64_t)cmdq->vaddr + 548 Q_IDX(cmdq, cmdq->lc.prod) * CMDQ_ENTRY_DWORDS * 8); 549 memcpy(entry_addr, cmd, CMDQ_ENTRY_DWORDS * 8); 550 551 /* Increment prod index. */ 552 cmdq->lc.prod = smmu_q_inc_prod(cmdq); 553 bus_write_4(sc->res[0], cmdq->prod_off, cmdq->lc.prod); 554 555 SMMU_UNLOCK(sc); 556 } 557 558 static void __unused 559 smmu_poll_until_consumed(struct smmu_softc *sc, struct smmu_queue *q) 560 { 561 562 while (1) { 563 q->lc.val = bus_read_8(sc->res[0], q->prod_off); 564 if (smmu_q_empty(q)) 565 break; 566 cpu_spinwait(); 567 } 568 } 569 570 static int 571 smmu_sync(struct smmu_softc *sc) 572 { 573 struct smmu_cmdq_entry cmd; 574 struct smmu_queue *q; 575 uint32_t *base; 576 int timeout; 577 int prod; 578 579 q = &sc->cmdq; 580 prod = q->lc.prod; 581 582 /* Enqueue sync command. */ 583 cmd.opcode = CMD_SYNC; 584 cmd.sync.msiaddr = q->paddr + Q_IDX(q, prod) * CMDQ_ENTRY_DWORDS * 8; 585 smmu_cmdq_enqueue_cmd(sc, &cmd); 586 587 /* Wait for the sync completion. */ 588 base = (void *)((uint64_t)q->vaddr + 589 Q_IDX(q, prod) * CMDQ_ENTRY_DWORDS * 8); 590 591 /* 592 * It takes around 200 loops (6 instructions each) 593 * on Neoverse N1 to complete the sync. 594 */ 595 timeout = 10000; 596 597 do { 598 if (*base == 0) { 599 /* MSI write completed. */ 600 break; 601 } 602 cpu_spinwait(); 603 } while (timeout--); 604 605 if (timeout < 0) 606 device_printf(sc->dev, "Failed to sync\n"); 607 608 return (0); 609 } 610 611 static int 612 smmu_sync_cd(struct smmu_softc *sc, int sid, int ssid, bool leaf) 613 { 614 struct smmu_cmdq_entry cmd; 615 616 cmd.opcode = CMD_CFGI_CD; 617 cmd.cfgi.sid = sid; 618 cmd.cfgi.ssid = ssid; 619 cmd.cfgi.leaf = leaf; 620 smmu_cmdq_enqueue_cmd(sc, &cmd); 621 622 return (0); 623 } 624 625 static void 626 smmu_invalidate_all_sid(struct smmu_softc *sc) 627 { 628 struct smmu_cmdq_entry cmd; 629 630 /* Invalidate cached config */ 631 cmd.opcode = CMD_CFGI_STE_RANGE; 632 smmu_cmdq_enqueue_cmd(sc, &cmd); 633 smmu_sync(sc); 634 } 635 636 static void 637 smmu_tlbi_all(struct smmu_softc *sc) 638 { 639 struct smmu_cmdq_entry cmd; 640 641 /* Invalidate entire TLB */ 642 cmd.opcode = CMD_TLBI_NSNH_ALL; 643 smmu_cmdq_enqueue_cmd(sc, &cmd); 644 smmu_sync(sc); 645 } 646 647 static void 648 smmu_tlbi_asid(struct smmu_softc *sc, uint16_t asid) 649 { 650 struct smmu_cmdq_entry cmd; 651 652 /* Invalidate TLB for an ASID. */ 653 cmd.opcode = CMD_TLBI_NH_ASID; 654 cmd.tlbi.asid = asid; 655 smmu_cmdq_enqueue_cmd(sc, &cmd); 656 smmu_sync(sc); 657 } 658 659 static void 660 smmu_tlbi_va(struct smmu_softc *sc, vm_offset_t va, uint16_t asid) 661 { 662 struct smmu_cmdq_entry cmd; 663 664 /* Invalidate specific range */ 665 cmd.opcode = CMD_TLBI_NH_VA; 666 cmd.tlbi.asid = asid; 667 cmd.tlbi.vmid = 0; 668 cmd.tlbi.leaf = true; /* We change only L3. */ 669 cmd.tlbi.addr = va; 670 smmu_cmdq_enqueue_cmd(sc, &cmd); 671 } 672 673 static void 674 smmu_invalidate_sid(struct smmu_softc *sc, uint32_t sid) 675 { 676 struct smmu_cmdq_entry cmd; 677 678 /* Invalidate cached config */ 679 cmd.opcode = CMD_CFGI_STE; 680 cmd.cfgi.sid = sid; 681 smmu_cmdq_enqueue_cmd(sc, &cmd); 682 smmu_sync(sc); 683 } 684 685 static void 686 smmu_prefetch_sid(struct smmu_softc *sc, uint32_t sid) 687 { 688 struct smmu_cmdq_entry cmd; 689 690 cmd.opcode = CMD_PREFETCH_CONFIG; 691 cmd.prefetch.sid = sid; 692 smmu_cmdq_enqueue_cmd(sc, &cmd); 693 smmu_sync(sc); 694 } 695 696 /* 697 * Init STE in bypass mode. Traffic is not translated for the sid. 698 */ 699 static void 700 smmu_init_ste_bypass(struct smmu_softc *sc, uint32_t sid, uint64_t *ste) 701 { 702 uint64_t val; 703 704 val = STE0_VALID | STE0_CONFIG_BYPASS; 705 706 ste[1] = STE1_SHCFG_INCOMING | STE1_EATS_FULLATS; 707 ste[2] = 0; 708 ste[3] = 0; 709 ste[4] = 0; 710 ste[5] = 0; 711 ste[6] = 0; 712 ste[7] = 0; 713 714 smmu_invalidate_sid(sc, sid); 715 ste[0] = val; 716 dsb(sy); 717 smmu_invalidate_sid(sc, sid); 718 719 smmu_prefetch_sid(sc, sid); 720 } 721 722 /* 723 * Enable Stage1 (S1) translation for the sid. 724 */ 725 static int 726 smmu_init_ste_s1(struct smmu_softc *sc, struct smmu_cd *cd, 727 uint32_t sid, uint64_t *ste) 728 { 729 uint64_t val; 730 731 val = STE0_VALID; 732 733 /* S1 */ 734 ste[1] = STE1_EATS_FULLATS | 735 STE1_S1CSH_IS | 736 STE1_S1CIR_WBRA | 737 STE1_S1COR_WBRA | 738 STE1_STRW_NS_EL1; 739 ste[2] = 0; 740 ste[3] = 0; 741 ste[4] = 0; 742 ste[5] = 0; 743 ste[6] = 0; 744 ste[7] = 0; 745 746 if (sc->features & SMMU_FEATURE_STALL && 747 ((sc->features & SMMU_FEATURE_STALL_FORCE) == 0)) 748 ste[1] |= STE1_S1STALLD; 749 750 /* Configure STE */ 751 val |= (cd->paddr & STE0_S1CONTEXTPTR_M); 752 val |= STE0_CONFIG_S1_TRANS; 753 754 smmu_invalidate_sid(sc, sid); 755 756 /* The STE[0] has to be written in a single blast, last of all. */ 757 ste[0] = val; 758 dsb(sy); 759 760 smmu_invalidate_sid(sc, sid); 761 smmu_sync_cd(sc, sid, 0, true); 762 smmu_invalidate_sid(sc, sid); 763 764 /* The sid will be used soon most likely. */ 765 smmu_prefetch_sid(sc, sid); 766 767 return (0); 768 } 769 770 static uint64_t * 771 smmu_get_ste_addr(struct smmu_softc *sc, int sid) 772 { 773 struct smmu_strtab *strtab; 774 struct l1_desc *l1_desc; 775 uint64_t *addr; 776 777 strtab = &sc->strtab; 778 779 if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) { 780 l1_desc = &strtab->l1[sid >> STRTAB_SPLIT]; 781 addr = l1_desc->va; 782 addr += (sid & ((1 << STRTAB_SPLIT) - 1)) * STRTAB_STE_DWORDS; 783 } else { 784 addr = (void *)((uint64_t)strtab->vaddr + 785 STRTAB_STE_DWORDS * 8 * sid); 786 }; 787 788 return (addr); 789 } 790 791 static int 792 smmu_init_ste(struct smmu_softc *sc, struct smmu_cd *cd, int sid, bool bypass) 793 { 794 uint64_t *addr; 795 796 addr = smmu_get_ste_addr(sc, sid); 797 798 if (bypass) 799 smmu_init_ste_bypass(sc, sid, addr); 800 else 801 smmu_init_ste_s1(sc, cd, sid, addr); 802 803 smmu_sync(sc); 804 805 return (0); 806 } 807 808 static void 809 smmu_deinit_ste(struct smmu_softc *sc, int sid) 810 { 811 uint64_t *ste; 812 813 ste = smmu_get_ste_addr(sc, sid); 814 ste[0] = 0; 815 816 smmu_invalidate_sid(sc, sid); 817 smmu_sync_cd(sc, sid, 0, true); 818 smmu_invalidate_sid(sc, sid); 819 820 smmu_sync(sc); 821 } 822 823 static int 824 smmu_init_cd(struct smmu_softc *sc, struct smmu_domain *domain) 825 { 826 vm_paddr_t paddr; 827 uint64_t *ptr; 828 uint64_t val; 829 vm_size_t size; 830 struct smmu_cd *cd; 831 struct smmu_pmap *p; 832 833 size = 1 * (CD_DWORDS << 3); 834 835 p = &domain->p; 836 cd = domain->cd = malloc(sizeof(struct smmu_cd), 837 M_SMMU, M_WAITOK | M_ZERO); 838 839 cd->vaddr = contigmalloc(size, M_SMMU, 840 M_WAITOK | M_ZERO, /* flags */ 841 0, /* low */ 842 (1ul << 40) - 1, /* high */ 843 size, /* alignment */ 844 0); /* boundary */ 845 if (cd->vaddr == NULL) { 846 device_printf(sc->dev, "Failed to allocate CD\n"); 847 return (ENXIO); 848 } 849 850 cd->size = size; 851 cd->paddr = vtophys(cd->vaddr); 852 853 ptr = cd->vaddr; 854 855 val = CD0_VALID; 856 val |= CD0_AA64; 857 val |= CD0_R; 858 val |= CD0_A; 859 val |= CD0_ASET; 860 val |= (uint64_t)domain->asid << CD0_ASID_S; 861 val |= CD0_TG0_4KB; 862 val |= CD0_EPD1; /* Disable TT1 */ 863 val |= ((64 - sc->ias) << CD0_T0SZ_S); 864 val |= CD0_IPS_48BITS; 865 866 paddr = p->sp_l0_paddr & CD1_TTB0_M; 867 KASSERT(paddr == p->sp_l0_paddr, ("bad allocation 1")); 868 869 ptr[1] = paddr; 870 ptr[2] = 0; 871 ptr[3] = MAIR_ATTR(MAIR_DEVICE_nGnRnE, VM_MEMATTR_DEVICE) | 872 MAIR_ATTR(MAIR_NORMAL_NC, VM_MEMATTR_UNCACHEABLE) | 873 MAIR_ATTR(MAIR_NORMAL_WB, VM_MEMATTR_WRITE_BACK) | 874 MAIR_ATTR(MAIR_NORMAL_WT, VM_MEMATTR_WRITE_THROUGH); 875 876 /* Install the CD. */ 877 ptr[0] = val; 878 879 return (0); 880 } 881 882 static int 883 smmu_init_strtab_linear(struct smmu_softc *sc) 884 { 885 struct smmu_strtab *strtab; 886 vm_paddr_t base; 887 uint32_t size; 888 uint64_t reg; 889 890 strtab = &sc->strtab; 891 strtab->num_l1_entries = (1 << sc->sid_bits); 892 893 size = strtab->num_l1_entries * (STRTAB_STE_DWORDS << 3); 894 895 if (bootverbose) 896 device_printf(sc->dev, 897 "%s: linear strtab size %d, num_l1_entries %d\n", 898 __func__, size, strtab->num_l1_entries); 899 900 strtab->vaddr = contigmalloc(size, M_SMMU, 901 M_WAITOK | M_ZERO, /* flags */ 902 0, /* low */ 903 (1ul << 48) - 1, /* high */ 904 size, /* alignment */ 905 0); /* boundary */ 906 if (strtab->vaddr == NULL) { 907 device_printf(sc->dev, "failed to allocate strtab\n"); 908 return (ENXIO); 909 } 910 911 reg = STRTAB_BASE_CFG_FMT_LINEAR; 912 reg |= sc->sid_bits << STRTAB_BASE_CFG_LOG2SIZE_S; 913 strtab->base_cfg = (uint32_t)reg; 914 915 base = vtophys(strtab->vaddr); 916 917 reg = base & STRTAB_BASE_ADDR_M; 918 KASSERT(reg == base, ("bad allocation 2")); 919 reg |= STRTAB_BASE_RA; 920 strtab->base = reg; 921 922 return (0); 923 } 924 925 static int 926 smmu_init_strtab_2lvl(struct smmu_softc *sc) 927 { 928 struct smmu_strtab *strtab; 929 vm_paddr_t base; 930 uint64_t reg_base; 931 uint32_t l1size; 932 uint32_t size; 933 uint32_t reg; 934 int sz; 935 936 strtab = &sc->strtab; 937 938 size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3); 939 size = min(size, sc->sid_bits - STRTAB_SPLIT); 940 strtab->num_l1_entries = (1 << size); 941 size += STRTAB_SPLIT; 942 943 l1size = strtab->num_l1_entries * (STRTAB_L1_DESC_DWORDS << 3); 944 945 if (bootverbose) 946 device_printf(sc->dev, 947 "%s: size %d, l1 entries %d, l1size %d\n", 948 __func__, size, strtab->num_l1_entries, l1size); 949 950 strtab->vaddr = contigmalloc(l1size, M_SMMU, 951 M_WAITOK | M_ZERO, /* flags */ 952 0, /* low */ 953 (1ul << 48) - 1, /* high */ 954 l1size, /* alignment */ 955 0); /* boundary */ 956 if (strtab->vaddr == NULL) { 957 device_printf(sc->dev, "Failed to allocate 2lvl strtab.\n"); 958 return (ENOMEM); 959 } 960 961 sz = strtab->num_l1_entries * sizeof(struct l1_desc); 962 963 strtab->l1 = malloc(sz, M_SMMU, M_WAITOK | M_ZERO); 964 if (strtab->l1 == NULL) { 965 contigfree(strtab->vaddr, l1size, M_SMMU); 966 return (ENOMEM); 967 } 968 969 reg = STRTAB_BASE_CFG_FMT_2LVL; 970 reg |= size << STRTAB_BASE_CFG_LOG2SIZE_S; 971 reg |= STRTAB_SPLIT << STRTAB_BASE_CFG_SPLIT_S; 972 strtab->base_cfg = (uint32_t)reg; 973 974 base = vtophys(strtab->vaddr); 975 976 reg_base = base & STRTAB_BASE_ADDR_M; 977 KASSERT(reg_base == base, ("bad allocation 3")); 978 reg_base |= STRTAB_BASE_RA; 979 strtab->base = reg_base; 980 981 return (0); 982 } 983 984 static int 985 smmu_init_strtab(struct smmu_softc *sc) 986 { 987 int error; 988 989 if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) 990 error = smmu_init_strtab_2lvl(sc); 991 else 992 error = smmu_init_strtab_linear(sc); 993 994 return (error); 995 } 996 997 static int 998 smmu_init_l1_entry(struct smmu_softc *sc, int sid) 999 { 1000 struct smmu_strtab *strtab; 1001 struct l1_desc *l1_desc; 1002 uint64_t *addr; 1003 uint64_t val; 1004 size_t size; 1005 int i; 1006 1007 strtab = &sc->strtab; 1008 l1_desc = &strtab->l1[sid >> STRTAB_SPLIT]; 1009 if (l1_desc->va) { 1010 /* Already allocated. */ 1011 return (0); 1012 } 1013 1014 size = 1 << (STRTAB_SPLIT + ilog2(STRTAB_STE_DWORDS) + 3); 1015 1016 l1_desc->span = STRTAB_SPLIT + 1; 1017 l1_desc->size = size; 1018 l1_desc->va = contigmalloc(size, M_SMMU, 1019 M_WAITOK | M_ZERO, /* flags */ 1020 0, /* low */ 1021 (1ul << 48) - 1, /* high */ 1022 size, /* alignment */ 1023 0); /* boundary */ 1024 if (l1_desc->va == NULL) { 1025 device_printf(sc->dev, "failed to allocate l2 entry\n"); 1026 return (ENXIO); 1027 } 1028 1029 l1_desc->pa = vtophys(l1_desc->va); 1030 1031 i = sid >> STRTAB_SPLIT; 1032 addr = (void *)((uint64_t)strtab->vaddr + 1033 STRTAB_L1_DESC_DWORDS * 8 * i); 1034 1035 /* Install the L1 entry. */ 1036 val = l1_desc->pa & STRTAB_L1_DESC_L2PTR_M; 1037 KASSERT(val == l1_desc->pa, ("bad allocation 4")); 1038 val |= l1_desc->span; 1039 *addr = val; 1040 1041 return (0); 1042 } 1043 1044 static void __unused 1045 smmu_deinit_l1_entry(struct smmu_softc *sc, int sid) 1046 { 1047 struct smmu_strtab *strtab; 1048 struct l1_desc *l1_desc; 1049 uint64_t *addr; 1050 int i; 1051 1052 strtab = &sc->strtab; 1053 1054 i = sid >> STRTAB_SPLIT; 1055 addr = (void *)((uint64_t)strtab->vaddr + 1056 STRTAB_L1_DESC_DWORDS * 8 * i); 1057 *addr = 0; 1058 1059 l1_desc = &strtab->l1[sid >> STRTAB_SPLIT]; 1060 contigfree(l1_desc->va, l1_desc->size, M_SMMU); 1061 } 1062 1063 static int 1064 smmu_disable(struct smmu_softc *sc) 1065 { 1066 uint32_t reg; 1067 int error; 1068 1069 /* Disable SMMU */ 1070 reg = bus_read_4(sc->res[0], SMMU_CR0); 1071 reg &= ~CR0_SMMUEN; 1072 error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg); 1073 if (error) 1074 device_printf(sc->dev, "Could not disable SMMU.\n"); 1075 1076 return (0); 1077 } 1078 1079 static int 1080 smmu_event_intr(void *arg) 1081 { 1082 uint32_t evt[EVTQ_ENTRY_DWORDS * 2]; 1083 struct smmu_softc *sc; 1084 1085 sc = arg; 1086 1087 do { 1088 smmu_evtq_dequeue(sc, evt); 1089 smmu_print_event(sc, evt); 1090 } while (!smmu_q_empty(&sc->evtq)); 1091 1092 return (FILTER_HANDLED); 1093 } 1094 1095 static int __unused 1096 smmu_sync_intr(void *arg) 1097 { 1098 struct smmu_softc *sc; 1099 1100 sc = arg; 1101 1102 device_printf(sc->dev, "%s\n", __func__); 1103 1104 return (FILTER_HANDLED); 1105 } 1106 1107 static int 1108 smmu_gerr_intr(void *arg) 1109 { 1110 struct smmu_softc *sc; 1111 1112 sc = arg; 1113 1114 device_printf(sc->dev, "SMMU Global Error\n"); 1115 1116 return (FILTER_HANDLED); 1117 } 1118 1119 static int 1120 smmu_enable_interrupts(struct smmu_softc *sc) 1121 { 1122 uint32_t reg; 1123 int error; 1124 1125 /* Disable MSI. */ 1126 bus_write_8(sc->res[0], SMMU_GERROR_IRQ_CFG0, 0); 1127 bus_write_4(sc->res[0], SMMU_GERROR_IRQ_CFG1, 0); 1128 bus_write_4(sc->res[0], SMMU_GERROR_IRQ_CFG2, 0); 1129 1130 bus_write_8(sc->res[0], SMMU_EVENTQ_IRQ_CFG0, 0); 1131 bus_write_4(sc->res[0], SMMU_EVENTQ_IRQ_CFG1, 0); 1132 bus_write_4(sc->res[0], SMMU_EVENTQ_IRQ_CFG2, 0); 1133 1134 if (sc->features & CR0_PRIQEN) { 1135 bus_write_8(sc->res[0], SMMU_PRIQ_IRQ_CFG0, 0); 1136 bus_write_4(sc->res[0], SMMU_PRIQ_IRQ_CFG1, 0); 1137 bus_write_4(sc->res[0], SMMU_PRIQ_IRQ_CFG2, 0); 1138 } 1139 1140 /* Disable any interrupts. */ 1141 error = smmu_write_ack(sc, SMMU_IRQ_CTRL, SMMU_IRQ_CTRLACK, 0); 1142 if (error) { 1143 device_printf(sc->dev, "Could not disable interrupts.\n"); 1144 return (ENXIO); 1145 } 1146 1147 /* Enable interrupts. */ 1148 reg = IRQ_CTRL_EVENTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN; 1149 if (sc->features & SMMU_FEATURE_PRI) 1150 reg |= IRQ_CTRL_PRIQ_IRQEN; 1151 1152 error = smmu_write_ack(sc, SMMU_IRQ_CTRL, SMMU_IRQ_CTRLACK, reg); 1153 if (error) { 1154 device_printf(sc->dev, "Could not enable interrupts.\n"); 1155 return (ENXIO); 1156 } 1157 1158 return (0); 1159 } 1160 1161 #ifdef DEV_ACPI 1162 static void 1163 smmu_configure_intr(struct smmu_softc *sc, struct resource *res) 1164 { 1165 struct intr_map_data_acpi *ad; 1166 struct intr_map_data *data; 1167 1168 data = rman_get_virtual(res); 1169 KASSERT(data != NULL, ("data is NULL")); 1170 1171 if (data->type == INTR_MAP_DATA_ACPI) { 1172 ad = (struct intr_map_data_acpi *)data; 1173 ad->trig = INTR_TRIGGER_EDGE; 1174 ad->pol = INTR_POLARITY_HIGH; 1175 } 1176 } 1177 #endif 1178 1179 static int 1180 smmu_setup_interrupts(struct smmu_softc *sc) 1181 { 1182 device_t dev; 1183 int error; 1184 1185 dev = sc->dev; 1186 1187 #ifdef DEV_ACPI 1188 /* 1189 * Configure SMMU interrupts as EDGE triggered manually 1190 * as ACPI tables carries no information for that. 1191 */ 1192 smmu_configure_intr(sc, sc->res[1]); 1193 /* PRIQ is not in use. */ 1194 smmu_configure_intr(sc, sc->res[3]); 1195 smmu_configure_intr(sc, sc->res[4]); 1196 #endif 1197 1198 error = bus_setup_intr(dev, sc->res[1], INTR_TYPE_MISC, 1199 smmu_event_intr, NULL, sc, &sc->intr_cookie[0]); 1200 if (error) { 1201 device_printf(dev, "Couldn't setup Event interrupt handler\n"); 1202 return (ENXIO); 1203 } 1204 1205 error = bus_setup_intr(dev, sc->res[4], INTR_TYPE_MISC, 1206 smmu_gerr_intr, NULL, sc, &sc->intr_cookie[2]); 1207 if (error) { 1208 device_printf(dev, "Couldn't setup Gerr interrupt handler\n"); 1209 return (ENXIO); 1210 } 1211 1212 return (0); 1213 } 1214 1215 static int 1216 smmu_reset(struct smmu_softc *sc) 1217 { 1218 struct smmu_cmdq_entry cmd; 1219 struct smmu_strtab *strtab; 1220 int error; 1221 int reg; 1222 1223 reg = bus_read_4(sc->res[0], SMMU_CR0); 1224 1225 if (reg & CR0_SMMUEN) 1226 device_printf(sc->dev, 1227 "%s: Warning: SMMU is enabled\n", __func__); 1228 1229 error = smmu_disable(sc); 1230 if (error) 1231 device_printf(sc->dev, 1232 "%s: Could not disable SMMU.\n", __func__); 1233 1234 if (smmu_enable_interrupts(sc) != 0) { 1235 device_printf(sc->dev, "Could not enable interrupts.\n"); 1236 return (ENXIO); 1237 } 1238 1239 reg = CR1_TABLE_SH_IS | 1240 CR1_TABLE_OC_WBC | 1241 CR1_TABLE_IC_WBC | 1242 CR1_QUEUE_SH_IS | 1243 CR1_QUEUE_OC_WBC | 1244 CR1_QUEUE_IC_WBC; 1245 bus_write_4(sc->res[0], SMMU_CR1, reg); 1246 1247 reg = CR2_PTM | CR2_RECINVSID | CR2_E2H; 1248 bus_write_4(sc->res[0], SMMU_CR2, reg); 1249 1250 /* Stream table. */ 1251 strtab = &sc->strtab; 1252 bus_write_8(sc->res[0], SMMU_STRTAB_BASE, strtab->base); 1253 bus_write_4(sc->res[0], SMMU_STRTAB_BASE_CFG, strtab->base_cfg); 1254 1255 /* Command queue. */ 1256 bus_write_8(sc->res[0], SMMU_CMDQ_BASE, sc->cmdq.base); 1257 bus_write_4(sc->res[0], SMMU_CMDQ_PROD, sc->cmdq.lc.prod); 1258 bus_write_4(sc->res[0], SMMU_CMDQ_CONS, sc->cmdq.lc.cons); 1259 1260 reg = CR0_CMDQEN; 1261 error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg); 1262 if (error) { 1263 device_printf(sc->dev, "Could not enable command queue\n"); 1264 return (ENXIO); 1265 } 1266 1267 /* Invalidate cached configuration. */ 1268 smmu_invalidate_all_sid(sc); 1269 1270 if (sc->features & SMMU_FEATURE_HYP) { 1271 cmd.opcode = CMD_TLBI_EL2_ALL; 1272 smmu_cmdq_enqueue_cmd(sc, &cmd); 1273 }; 1274 1275 /* Invalidate TLB. */ 1276 smmu_tlbi_all(sc); 1277 1278 /* Event queue */ 1279 bus_write_8(sc->res[0], SMMU_EVENTQ_BASE, sc->evtq.base); 1280 bus_write_4(sc->res[0], SMMU_EVENTQ_PROD, sc->evtq.lc.prod); 1281 bus_write_4(sc->res[0], SMMU_EVENTQ_CONS, sc->evtq.lc.cons); 1282 1283 reg |= CR0_EVENTQEN; 1284 error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg); 1285 if (error) { 1286 device_printf(sc->dev, "Could not enable event queue\n"); 1287 return (ENXIO); 1288 } 1289 1290 if (sc->features & SMMU_FEATURE_PRI) { 1291 /* PRI queue */ 1292 bus_write_8(sc->res[0], SMMU_PRIQ_BASE, sc->priq.base); 1293 bus_write_4(sc->res[0], SMMU_PRIQ_PROD, sc->priq.lc.prod); 1294 bus_write_4(sc->res[0], SMMU_PRIQ_CONS, sc->priq.lc.cons); 1295 1296 reg |= CR0_PRIQEN; 1297 error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg); 1298 if (error) { 1299 device_printf(sc->dev, "Could not enable PRI queue\n"); 1300 return (ENXIO); 1301 } 1302 } 1303 1304 if (sc->features & SMMU_FEATURE_ATS) { 1305 reg |= CR0_ATSCHK; 1306 error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg); 1307 if (error) { 1308 device_printf(sc->dev, "Could not enable ATS check.\n"); 1309 return (ENXIO); 1310 } 1311 } 1312 1313 reg |= CR0_SMMUEN; 1314 error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg); 1315 if (error) { 1316 device_printf(sc->dev, "Could not enable SMMU.\n"); 1317 return (ENXIO); 1318 } 1319 1320 return (0); 1321 } 1322 1323 static int 1324 smmu_check_features(struct smmu_softc *sc) 1325 { 1326 uint32_t reg; 1327 uint32_t val; 1328 1329 sc->features = 0; 1330 1331 reg = bus_read_4(sc->res[0], SMMU_IDR0); 1332 1333 if (reg & IDR0_ST_LVL_2) { 1334 if (bootverbose) 1335 device_printf(sc->dev, 1336 "2-level stream table supported.\n"); 1337 sc->features |= SMMU_FEATURE_2_LVL_STREAM_TABLE; 1338 } 1339 1340 if (reg & IDR0_CD2L) { 1341 if (bootverbose) 1342 device_printf(sc->dev, 1343 "2-level CD table supported.\n"); 1344 sc->features |= SMMU_FEATURE_2_LVL_CD; 1345 } 1346 1347 switch (reg & IDR0_TTENDIAN_M) { 1348 case IDR0_TTENDIAN_MIXED: 1349 if (bootverbose) 1350 device_printf(sc->dev, "Mixed endianness supported.\n"); 1351 sc->features |= SMMU_FEATURE_TT_LE; 1352 sc->features |= SMMU_FEATURE_TT_BE; 1353 break; 1354 case IDR0_TTENDIAN_LITTLE: 1355 if (bootverbose) 1356 device_printf(sc->dev, 1357 "Little endian supported only.\n"); 1358 sc->features |= SMMU_FEATURE_TT_LE; 1359 break; 1360 case IDR0_TTENDIAN_BIG: 1361 if (bootverbose) 1362 device_printf(sc->dev, "Big endian supported only.\n"); 1363 sc->features |= SMMU_FEATURE_TT_BE; 1364 break; 1365 default: 1366 device_printf(sc->dev, "Unsupported endianness.\n"); 1367 return (ENXIO); 1368 } 1369 1370 if (reg & IDR0_SEV) 1371 sc->features |= SMMU_FEATURE_SEV; 1372 1373 if (reg & IDR0_MSI) { 1374 if (bootverbose) 1375 device_printf(sc->dev, "MSI feature present.\n"); 1376 sc->features |= SMMU_FEATURE_MSI; 1377 } 1378 1379 if (reg & IDR0_HYP) { 1380 if (bootverbose) 1381 device_printf(sc->dev, "HYP feature present.\n"); 1382 sc->features |= SMMU_FEATURE_HYP; 1383 } 1384 1385 if (reg & IDR0_ATS) 1386 sc->features |= SMMU_FEATURE_ATS; 1387 1388 if (reg & IDR0_PRI) 1389 sc->features |= SMMU_FEATURE_PRI; 1390 1391 switch (reg & IDR0_STALL_MODEL_M) { 1392 case IDR0_STALL_MODEL_FORCE: 1393 /* Stall is forced. */ 1394 sc->features |= SMMU_FEATURE_STALL_FORCE; 1395 /* FALLTHROUGH */ 1396 case IDR0_STALL_MODEL_STALL: 1397 sc->features |= SMMU_FEATURE_STALL; 1398 break; 1399 } 1400 1401 /* Grab translation stages supported. */ 1402 if (reg & IDR0_S1P) { 1403 if (bootverbose) 1404 device_printf(sc->dev, 1405 "Stage 1 translation supported.\n"); 1406 sc->features |= SMMU_FEATURE_S1P; 1407 } 1408 if (reg & IDR0_S2P) { 1409 if (bootverbose) 1410 device_printf(sc->dev, 1411 "Stage 2 translation supported.\n"); 1412 sc->features |= SMMU_FEATURE_S2P; 1413 } 1414 1415 switch (reg & IDR0_TTF_M) { 1416 case IDR0_TTF_ALL: 1417 case IDR0_TTF_AA64: 1418 sc->ias = 40; 1419 break; 1420 default: 1421 device_printf(sc->dev, "No AArch64 table format support.\n"); 1422 return (ENXIO); 1423 } 1424 1425 if (reg & IDR0_ASID16) 1426 sc->asid_bits = 16; 1427 else 1428 sc->asid_bits = 8; 1429 1430 if (bootverbose) 1431 device_printf(sc->dev, "ASID bits %d\n", sc->asid_bits); 1432 1433 if (reg & IDR0_VMID16) 1434 sc->vmid_bits = 16; 1435 else 1436 sc->vmid_bits = 8; 1437 1438 reg = bus_read_4(sc->res[0], SMMU_IDR1); 1439 1440 if (reg & (IDR1_TABLES_PRESET | IDR1_QUEUES_PRESET | IDR1_REL)) { 1441 device_printf(sc->dev, 1442 "Embedded implementations not supported by this driver.\n"); 1443 return (ENXIO); 1444 } 1445 1446 val = (reg & IDR1_CMDQS_M) >> IDR1_CMDQS_S; 1447 sc->cmdq.size_log2 = val; 1448 if (bootverbose) 1449 device_printf(sc->dev, "CMD queue bits %d\n", val); 1450 1451 val = (reg & IDR1_EVENTQS_M) >> IDR1_EVENTQS_S; 1452 sc->evtq.size_log2 = val; 1453 if (bootverbose) 1454 device_printf(sc->dev, "EVENT queue bits %d\n", val); 1455 1456 if (sc->features & SMMU_FEATURE_PRI) { 1457 val = (reg & IDR1_PRIQS_M) >> IDR1_PRIQS_S; 1458 sc->priq.size_log2 = val; 1459 if (bootverbose) 1460 device_printf(sc->dev, "PRI queue bits %d\n", val); 1461 } 1462 1463 sc->ssid_bits = (reg & IDR1_SSIDSIZE_M) >> IDR1_SSIDSIZE_S; 1464 sc->sid_bits = (reg & IDR1_SIDSIZE_M) >> IDR1_SIDSIZE_S; 1465 1466 if (sc->sid_bits <= STRTAB_SPLIT) 1467 sc->features &= ~SMMU_FEATURE_2_LVL_STREAM_TABLE; 1468 1469 if (bootverbose) { 1470 device_printf(sc->dev, "SSID bits %d\n", sc->ssid_bits); 1471 device_printf(sc->dev, "SID bits %d\n", sc->sid_bits); 1472 } 1473 1474 /* IDR3 */ 1475 reg = bus_read_4(sc->res[0], SMMU_IDR3); 1476 if (reg & IDR3_RIL) 1477 sc->features |= SMMU_FEATURE_RANGE_INV; 1478 1479 /* IDR5 */ 1480 reg = bus_read_4(sc->res[0], SMMU_IDR5); 1481 1482 switch (reg & IDR5_OAS_M) { 1483 case IDR5_OAS_32: 1484 sc->oas = 32; 1485 break; 1486 case IDR5_OAS_36: 1487 sc->oas = 36; 1488 break; 1489 case IDR5_OAS_40: 1490 sc->oas = 40; 1491 break; 1492 case IDR5_OAS_42: 1493 sc->oas = 42; 1494 break; 1495 case IDR5_OAS_44: 1496 sc->oas = 44; 1497 break; 1498 case IDR5_OAS_48: 1499 sc->oas = 48; 1500 break; 1501 case IDR5_OAS_52: 1502 sc->oas = 52; 1503 break; 1504 } 1505 1506 sc->pgsizes = 0; 1507 if (reg & IDR5_GRAN64K) 1508 sc->pgsizes |= 64 * 1024; 1509 if (reg & IDR5_GRAN16K) 1510 sc->pgsizes |= 16 * 1024; 1511 if (reg & IDR5_GRAN4K) 1512 sc->pgsizes |= 4 * 1024; 1513 1514 if ((reg & IDR5_VAX_M) == IDR5_VAX_52) 1515 sc->features |= SMMU_FEATURE_VAX; 1516 1517 return (0); 1518 } 1519 1520 static void 1521 smmu_init_asids(struct smmu_softc *sc) 1522 { 1523 1524 sc->asid_set_size = (1 << sc->asid_bits); 1525 sc->asid_set = bit_alloc(sc->asid_set_size, M_SMMU, M_WAITOK); 1526 mtx_init(&sc->asid_set_mutex, "asid set", NULL, MTX_SPIN); 1527 } 1528 1529 static int 1530 smmu_asid_alloc(struct smmu_softc *sc, int *new_asid) 1531 { 1532 1533 mtx_lock_spin(&sc->asid_set_mutex); 1534 bit_ffc(sc->asid_set, sc->asid_set_size, new_asid); 1535 if (*new_asid == -1) { 1536 mtx_unlock_spin(&sc->asid_set_mutex); 1537 return (ENOMEM); 1538 } 1539 bit_set(sc->asid_set, *new_asid); 1540 mtx_unlock_spin(&sc->asid_set_mutex); 1541 1542 return (0); 1543 } 1544 1545 static void 1546 smmu_asid_free(struct smmu_softc *sc, int asid) 1547 { 1548 1549 mtx_lock_spin(&sc->asid_set_mutex); 1550 bit_clear(sc->asid_set, asid); 1551 mtx_unlock_spin(&sc->asid_set_mutex); 1552 } 1553 1554 /* 1555 * Device interface. 1556 */ 1557 int 1558 smmu_attach(device_t dev) 1559 { 1560 struct smmu_softc *sc; 1561 int error; 1562 1563 sc = device_get_softc(dev); 1564 sc->dev = dev; 1565 1566 mtx_init(&sc->sc_mtx, device_get_nameunit(sc->dev), "smmu", MTX_DEF); 1567 1568 error = smmu_setup_interrupts(sc); 1569 if (error) { 1570 bus_release_resources(dev, smmu_spec, sc->res); 1571 return (ENXIO); 1572 } 1573 1574 error = smmu_check_features(sc); 1575 if (error) { 1576 device_printf(dev, "Some features are required " 1577 "but not supported by hardware.\n"); 1578 return (ENXIO); 1579 } 1580 1581 smmu_init_asids(sc); 1582 1583 error = smmu_init_queues(sc); 1584 if (error) { 1585 device_printf(dev, "Couldn't allocate queues.\n"); 1586 return (ENXIO); 1587 } 1588 1589 error = smmu_init_strtab(sc); 1590 if (error) { 1591 device_printf(dev, "Couldn't allocate strtab.\n"); 1592 return (ENXIO); 1593 } 1594 1595 error = smmu_reset(sc); 1596 if (error) { 1597 device_printf(dev, "Couldn't reset SMMU.\n"); 1598 return (ENXIO); 1599 } 1600 1601 return (0); 1602 } 1603 1604 int 1605 smmu_detach(device_t dev) 1606 { 1607 struct smmu_softc *sc; 1608 1609 sc = device_get_softc(dev); 1610 1611 bus_release_resources(dev, smmu_spec, sc->res); 1612 1613 return (0); 1614 } 1615 1616 static int 1617 smmu_read_ivar(device_t dev, device_t child, int which, uintptr_t *result) 1618 { 1619 struct smmu_softc *sc; 1620 1621 sc = device_get_softc(dev); 1622 1623 device_printf(sc->dev, "%s\n", __func__); 1624 1625 return (ENOENT); 1626 } 1627 1628 static int 1629 smmu_unmap(device_t dev, struct iommu_domain *iodom, 1630 vm_offset_t va, bus_size_t size) 1631 { 1632 struct smmu_domain *domain; 1633 struct smmu_softc *sc; 1634 int err; 1635 int i; 1636 1637 sc = device_get_softc(dev); 1638 1639 domain = (struct smmu_domain *)iodom; 1640 1641 err = 0; 1642 1643 dprintf("%s: %lx, %ld, domain %d\n", __func__, va, size, domain->asid); 1644 1645 for (i = 0; i < size; i += PAGE_SIZE) { 1646 if (smmu_pmap_remove(&domain->p, va) == 0) { 1647 /* pmap entry removed, invalidate TLB. */ 1648 smmu_tlbi_va(sc, va, domain->asid); 1649 } else { 1650 err = ENOENT; 1651 break; 1652 } 1653 va += PAGE_SIZE; 1654 } 1655 1656 smmu_sync(sc); 1657 1658 return (err); 1659 } 1660 1661 static int 1662 smmu_map(device_t dev, struct iommu_domain *iodom, 1663 vm_offset_t va, vm_page_t *ma, vm_size_t size, 1664 vm_prot_t prot) 1665 { 1666 struct smmu_domain *domain; 1667 struct smmu_softc *sc; 1668 vm_paddr_t pa; 1669 int error; 1670 int i; 1671 1672 sc = device_get_softc(dev); 1673 1674 domain = (struct smmu_domain *)iodom; 1675 1676 dprintf("%s: %lx -> %lx, %ld, domain %d\n", __func__, va, pa, size, 1677 domain->asid); 1678 1679 for (i = 0; size > 0; size -= PAGE_SIZE) { 1680 pa = VM_PAGE_TO_PHYS(ma[i++]); 1681 error = smmu_pmap_enter(&domain->p, va, pa, prot, 0); 1682 if (error) 1683 return (error); 1684 smmu_tlbi_va(sc, va, domain->asid); 1685 va += PAGE_SIZE; 1686 } 1687 1688 smmu_sync(sc); 1689 1690 return (0); 1691 } 1692 1693 static struct iommu_domain * 1694 smmu_domain_alloc(device_t dev, struct iommu_unit *iommu) 1695 { 1696 struct iommu_domain *iodom; 1697 struct smmu_domain *domain; 1698 struct smmu_unit *unit; 1699 struct smmu_softc *sc; 1700 int error; 1701 int new_asid; 1702 1703 sc = device_get_softc(dev); 1704 1705 unit = (struct smmu_unit *)iommu; 1706 1707 domain = malloc(sizeof(*domain), M_SMMU, M_WAITOK | M_ZERO); 1708 1709 error = smmu_asid_alloc(sc, &new_asid); 1710 if (error) { 1711 free(domain, M_SMMU); 1712 device_printf(sc->dev, 1713 "Could not allocate ASID for a new domain.\n"); 1714 return (NULL); 1715 } 1716 1717 domain->asid = (uint16_t)new_asid; 1718 1719 smmu_pmap_pinit(&domain->p); 1720 1721 error = smmu_init_cd(sc, domain); 1722 if (error) { 1723 free(domain, M_SMMU); 1724 device_printf(sc->dev, "Could not initialize CD\n"); 1725 return (NULL); 1726 } 1727 1728 smmu_tlbi_asid(sc, domain->asid); 1729 1730 LIST_INIT(&domain->ctx_list); 1731 1732 IOMMU_LOCK(iommu); 1733 LIST_INSERT_HEAD(&unit->domain_list, domain, next); 1734 IOMMU_UNLOCK(iommu); 1735 1736 iodom = &domain->iodom; 1737 1738 /* 1739 * Use 48-bit address space regardless of VAX bit 1740 * as we need 64k IOMMU_PAGE_SIZE for 52-bit space. 1741 */ 1742 iodom->end = MAXADDR_48BIT; 1743 1744 return (iodom); 1745 } 1746 1747 static void 1748 smmu_domain_free(device_t dev, struct iommu_domain *iodom) 1749 { 1750 struct smmu_domain *domain; 1751 struct smmu_softc *sc; 1752 struct smmu_cd *cd; 1753 1754 sc = device_get_softc(dev); 1755 1756 domain = (struct smmu_domain *)iodom; 1757 1758 LIST_REMOVE(domain, next); 1759 1760 cd = domain->cd; 1761 1762 smmu_pmap_remove_pages(&domain->p); 1763 smmu_pmap_release(&domain->p); 1764 1765 smmu_tlbi_asid(sc, domain->asid); 1766 smmu_asid_free(sc, domain->asid); 1767 1768 contigfree(cd->vaddr, cd->size, M_SMMU); 1769 free(cd, M_SMMU); 1770 1771 free(domain, M_SMMU); 1772 } 1773 1774 static int 1775 smmu_set_buswide(device_t dev, struct smmu_domain *domain, 1776 struct smmu_ctx *ctx) 1777 { 1778 struct smmu_softc *sc; 1779 int i; 1780 1781 sc = device_get_softc(dev); 1782 1783 for (i = 0; i < PCI_SLOTMAX; i++) 1784 smmu_init_ste(sc, domain->cd, (ctx->sid | i), ctx->bypass); 1785 1786 return (0); 1787 } 1788 1789 static int 1790 smmu_pci_get_sid(device_t child, u_int *xref0, u_int *sid0) 1791 { 1792 struct pci_id_ofw_iommu pi; 1793 int err; 1794 1795 err = pci_get_id(child, PCI_ID_OFW_IOMMU, (uintptr_t *)&pi); 1796 if (err == 0) { 1797 if (sid0) 1798 *sid0 = pi.id; 1799 if (xref0) 1800 *xref0 = pi.xref; 1801 } 1802 1803 return (err); 1804 } 1805 1806 static struct iommu_ctx * 1807 smmu_ctx_alloc(device_t dev, struct iommu_domain *iodom, device_t child, 1808 bool disabled) 1809 { 1810 struct smmu_domain *domain; 1811 struct smmu_ctx *ctx; 1812 1813 domain = (struct smmu_domain *)iodom; 1814 1815 ctx = malloc(sizeof(struct smmu_ctx), M_SMMU, M_WAITOK | M_ZERO); 1816 ctx->dev = child; 1817 ctx->domain = domain; 1818 if (disabled) 1819 ctx->bypass = true; 1820 1821 IOMMU_DOMAIN_LOCK(iodom); 1822 LIST_INSERT_HEAD(&domain->ctx_list, ctx, next); 1823 IOMMU_DOMAIN_UNLOCK(iodom); 1824 1825 return (&ctx->ioctx); 1826 } 1827 1828 static int 1829 smmu_ctx_init(device_t dev, struct iommu_ctx *ioctx) 1830 { 1831 struct smmu_domain *domain; 1832 struct iommu_domain *iodom; 1833 struct smmu_softc *sc; 1834 struct smmu_ctx *ctx; 1835 devclass_t pci_class; 1836 u_int sid; 1837 int err; 1838 1839 ctx = (struct smmu_ctx *)ioctx; 1840 1841 sc = device_get_softc(dev); 1842 1843 domain = ctx->domain; 1844 iodom = (struct iommu_domain *)domain; 1845 1846 pci_class = devclass_find("pci"); 1847 if (device_get_devclass(device_get_parent(ctx->dev)) == pci_class) { 1848 err = smmu_pci_get_sid(ctx->dev, NULL, &sid); 1849 if (err) 1850 return (err); 1851 1852 ioctx->rid = pci_get_rid(dev); 1853 ctx->sid = sid; 1854 ctx->vendor = pci_get_vendor(ctx->dev); 1855 ctx->device = pci_get_device(ctx->dev); 1856 } 1857 1858 if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) { 1859 err = smmu_init_l1_entry(sc, ctx->sid); 1860 if (err) 1861 return (err); 1862 } 1863 1864 /* 1865 * Neoverse N1 SDP: 1866 * 0x800 xhci 1867 * 0x700 re 1868 * 0x600 sata 1869 */ 1870 1871 smmu_init_ste(sc, domain->cd, ctx->sid, ctx->bypass); 1872 1873 if (device_get_devclass(device_get_parent(ctx->dev)) == pci_class) 1874 if (iommu_is_buswide_ctx(iodom->iommu, pci_get_bus(ctx->dev))) 1875 smmu_set_buswide(dev, domain, ctx); 1876 1877 return (0); 1878 } 1879 1880 static void 1881 smmu_ctx_free(device_t dev, struct iommu_ctx *ioctx) 1882 { 1883 struct smmu_softc *sc; 1884 struct smmu_ctx *ctx; 1885 1886 IOMMU_ASSERT_LOCKED(ioctx->domain->iommu); 1887 1888 sc = device_get_softc(dev); 1889 ctx = (struct smmu_ctx *)ioctx; 1890 1891 smmu_deinit_ste(sc, ctx->sid); 1892 1893 LIST_REMOVE(ctx, next); 1894 1895 free(ctx, M_SMMU); 1896 } 1897 1898 struct smmu_ctx * 1899 smmu_ctx_lookup_by_sid(device_t dev, u_int sid) 1900 { 1901 struct smmu_softc *sc; 1902 struct smmu_domain *domain; 1903 struct smmu_unit *unit; 1904 struct smmu_ctx *ctx; 1905 1906 sc = device_get_softc(dev); 1907 1908 unit = &sc->unit; 1909 1910 LIST_FOREACH(domain, &unit->domain_list, next) { 1911 LIST_FOREACH(ctx, &domain->ctx_list, next) { 1912 if (ctx->sid == sid) 1913 return (ctx); 1914 } 1915 } 1916 1917 return (NULL); 1918 } 1919 1920 static struct iommu_ctx * 1921 smmu_ctx_lookup(device_t dev, device_t child) 1922 { 1923 struct iommu_unit *iommu __diagused; 1924 struct smmu_softc *sc; 1925 struct smmu_domain *domain; 1926 struct smmu_unit *unit; 1927 struct smmu_ctx *ctx; 1928 1929 sc = device_get_softc(dev); 1930 1931 unit = &sc->unit; 1932 iommu = &unit->iommu; 1933 1934 IOMMU_ASSERT_LOCKED(iommu); 1935 1936 LIST_FOREACH(domain, &unit->domain_list, next) { 1937 IOMMU_DOMAIN_LOCK(&domain->iodom); 1938 LIST_FOREACH(ctx, &domain->ctx_list, next) { 1939 if (ctx->dev == child) { 1940 IOMMU_DOMAIN_UNLOCK(&domain->iodom); 1941 return (&ctx->ioctx); 1942 } 1943 } 1944 IOMMU_DOMAIN_UNLOCK(&domain->iodom); 1945 } 1946 1947 return (NULL); 1948 } 1949 1950 static int 1951 smmu_find(device_t dev, device_t child) 1952 { 1953 struct smmu_softc *sc; 1954 u_int xref; 1955 int err; 1956 1957 sc = device_get_softc(dev); 1958 1959 err = smmu_pci_get_sid(child, &xref, NULL); 1960 if (err) 1961 return (ENOENT); 1962 1963 /* Check if xref is ours. */ 1964 if (xref != sc->xref) 1965 return (EFAULT); 1966 1967 return (0); 1968 } 1969 1970 #ifdef FDT 1971 static int 1972 smmu_ofw_md_data(device_t dev, struct iommu_ctx *ioctx, pcell_t *cells, 1973 int ncells) 1974 { 1975 struct smmu_ctx *ctx; 1976 1977 ctx = (struct smmu_ctx *)ioctx; 1978 1979 if (ncells != 1) 1980 return (-1); 1981 1982 ctx->sid = cells[0]; 1983 1984 return (0); 1985 } 1986 #endif 1987 1988 static device_method_t smmu_methods[] = { 1989 /* Device interface */ 1990 DEVMETHOD(device_detach, smmu_detach), 1991 1992 /* SMMU interface */ 1993 DEVMETHOD(iommu_find, smmu_find), 1994 DEVMETHOD(iommu_map, smmu_map), 1995 DEVMETHOD(iommu_unmap, smmu_unmap), 1996 DEVMETHOD(iommu_domain_alloc, smmu_domain_alloc), 1997 DEVMETHOD(iommu_domain_free, smmu_domain_free), 1998 DEVMETHOD(iommu_ctx_alloc, smmu_ctx_alloc), 1999 DEVMETHOD(iommu_ctx_init, smmu_ctx_init), 2000 DEVMETHOD(iommu_ctx_free, smmu_ctx_free), 2001 DEVMETHOD(iommu_ctx_lookup, smmu_ctx_lookup), 2002 #ifdef FDT 2003 DEVMETHOD(iommu_ofw_md_data, smmu_ofw_md_data), 2004 #endif 2005 2006 /* Bus interface */ 2007 DEVMETHOD(bus_read_ivar, smmu_read_ivar), 2008 2009 /* End */ 2010 DEVMETHOD_END 2011 }; 2012 2013 DEFINE_CLASS_0(smmu, smmu_driver, smmu_methods, sizeof(struct smmu_softc)); 2014