1 /*- 2 * Copyright (c) 2021 Andrew Turner 3 * Copyright (c) 2022 Michael J. Karels <karels@freebsd.org> 4 * Copyright (c) 2022 Kyle Evans <kevans@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include "opt_platform.h" 29 30 #include <sys/param.h> 31 #include <sys/systm.h> 32 #include <sys/bus.h> 33 #include <sys/kernel.h> 34 #include <sys/module.h> 35 #include <sys/proc.h> 36 #include <sys/rman.h> 37 #include <sys/smp.h> 38 39 #include <machine/bus.h> 40 #include <machine/machdep.h> 41 #ifdef SMP 42 #include <machine/intr.h> 43 #include <machine/smp.h> 44 #endif 45 46 #include <dev/fdt/fdt_intr.h> 47 48 #include <dev/ofw/openfirm.h> 49 #include <dev/ofw/ofw_bus.h> 50 #include <dev/ofw/ofw_bus_subr.h> 51 52 #include <dt-bindings/interrupt-controller/apple-aic.h> 53 54 #include "pic_if.h" 55 56 #define AIC_INFO 0x0004 57 #define AIC_INFO_NDIE(val) (((val) >> 24) & 0xf) 58 #define AIC_INFO_NIRQS(val) ((val) & 0x0000ffff) 59 60 #define AIC_WHOAMI 0x2000 61 #define AIC_EVENT 0x2004 62 #define AIC_EVENT_DIE(val) (((val) >> 24) & 0xff) 63 #define AIC_EVENT_TYPE(val) (((val) >> 16) & 0xff) 64 #define AIC_EVENT_TYPE_NONE 0 65 #define AIC_EVENT_TYPE_IRQ 1 66 #define AIC_EVENT_TYPE_IPI 4 67 #define AIC_EVENT_IRQ(val) ((val) & 0xffff) 68 #define AIC_EVENT_IPI_OTHER 1 69 #define AIC_EVENT_IPI_SELF 2 70 #define AIC_IPI_SEND 0x2008 71 #define AIC_IPI_ACK 0x200c 72 #define AIC_IPI_MASK_SET 0x2024 73 #define AIC_IPI_MASK_CLR 0x2028 74 #define AIC_IPI_OTHER 0x00000001 75 #define AIC_IPI_SELF 0x80000000 76 #define AIC_TARGET_CPU(irq) (0x3000 + (irq) * 4) 77 #define AIC_SW_SET(irq) (0x4000 + (((irq) >> 5) * 4)) 78 #define AIC_SW_CLEAR(irq) (0x4080 + (((irq) >> 5) * 4)) 79 #define AIC_MASK_SET(irq) (0x4100 + (((irq) >> 5) * 4)) 80 #define AIC_MASK_CLEAR(irq) (0x4180 + (((irq) >> 5) * 4)) 81 #define AIC_IRQ_MASK(irq) (1u << ((irq) & 0x1f)) 82 83 #define AIC_IPI_LOCAL_RR_EL1 s3_5_c15_c0_0 84 #define AIC_IPI_GLOBAL_RR_EL1 s3_5_c15_c0_1 85 86 #define AIC_IPI_SR_EL1 s3_5_c15_c1_1 87 #define AIC_IPI_SR_EL1_PENDING (1 << 0) 88 89 #define AIC_FIQ_VM_TIMER s3_5_c15_c1_3 90 #define AIC_FIQ_VM_TIMER_VEN (1 << 0) 91 #define AIC_FIQ_VM_TIMER_PEN (1 << 1) 92 #define AIC_FIQ_VM_TIMER_BITS (AIC_FIQ_VM_TIMER_VEN | AIC_FIQ_VM_TIMER_PEN) 93 94 #define CNTV_CTL_ENABLE (1 << 0) 95 #define CNTV_CTL_IMASK (1 << 1) 96 #define CNTV_CTL_ISTATUS (1 << 2) 97 #define CNTV_CTL_BITS \ 98 (CNTV_CTL_ENABLE | CNTV_CTL_IMASK | CNTV_CTL_ISTATUS) 99 100 #define AIC_MAXCPUS 32 101 #define AIC_MAXDIES 4 102 103 static struct ofw_compat_data compat_data[] = { 104 { "apple,aic", 1 }, 105 { NULL, 0 } 106 }; 107 108 enum apple_aic_irq_type { 109 AIC_TYPE_INVAL, 110 AIC_TYPE_IRQ, 111 AIC_TYPE_FIQ, 112 AIC_TYPE_IPI, 113 }; 114 115 struct apple_aic_irqsrc { 116 struct intr_irqsrc ai_isrc; 117 enum apple_aic_irq_type ai_type; 118 struct { 119 /* AIC_TYPE_IRQ */ 120 enum intr_polarity ai_pol; 121 enum intr_trigger ai_trig; 122 u_int ai_irq; 123 }; 124 }; 125 126 #ifdef SMP 127 #define AIC_NIPIS INTR_IPI_COUNT 128 #endif 129 130 struct apple_aic_softc { 131 device_t sc_dev; 132 struct resource *sc_mem; 133 struct apple_aic_irqsrc *sc_isrcs[AIC_MAXDIES]; 134 u_int sc_nirqs; 135 u_int sc_ndie; 136 #ifdef SMP 137 struct apple_aic_irqsrc sc_ipi_srcs[AIC_NIPIS]; 138 u_int *sc_cpuids; /* cpu index to AIC CPU ID */ 139 uint32_t *sc_ipimasks; 140 #endif 141 }; 142 143 static u_int aic_next_cpu; 144 145 static device_probe_t apple_aic_probe; 146 static device_attach_t apple_aic_attach; 147 148 static pic_disable_intr_t apple_aic_disable_intr; 149 static pic_enable_intr_t apple_aic_enable_intr; 150 static pic_map_intr_t apple_aic_map_intr; 151 static pic_setup_intr_t apple_aic_setup_intr; 152 static pic_teardown_intr_t apple_aic_teardown_intr; 153 static pic_post_filter_t apple_aic_post_filter; 154 static pic_post_ithread_t apple_aic_post_ithread; 155 static pic_pre_ithread_t apple_aic_pre_ithread; 156 #ifdef SMP 157 static pic_bind_intr_t apple_aic_bind_intr; 158 static pic_init_secondary_t apple_aic_init_secondary; 159 static pic_ipi_send_t apple_aic_ipi_send; 160 static pic_ipi_setup_t apple_aic_ipi_setup; 161 #endif 162 163 static int apple_aic_irq(void *); 164 static int apple_aic_fiq(void *); 165 166 static int 167 apple_aic_probe(device_t dev) 168 { 169 170 if (!ofw_bus_status_okay(dev)) 171 return (ENXIO); 172 173 if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0) 174 return (ENXIO); 175 176 device_set_desc(dev, "Apple Interrupt Controller"); 177 return (BUS_PROBE_DEFAULT); 178 } 179 180 static int 181 apple_aic_attach(device_t dev) 182 { 183 struct apple_aic_softc *sc; 184 struct intr_irqsrc *isrc; 185 const char *name; 186 intptr_t xref; 187 int error, rid; 188 u_int i, cpu, j, info; 189 190 sc = device_get_softc(dev); 191 sc->sc_dev = dev; 192 193 rid = 0; 194 sc->sc_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 195 RF_ACTIVE); 196 if (sc->sc_mem == NULL) { 197 device_printf(dev, "Unable to allocate memory\n"); 198 return (ENXIO); 199 } 200 201 info = bus_read_4(sc->sc_mem, AIC_INFO); 202 sc->sc_nirqs = AIC_INFO_NIRQS(info); 203 sc->sc_ndie = AIC_INFO_NDIE(info) + 1; 204 if (bootverbose) 205 device_printf(dev, "Found %d interrupts, %d die\n", 206 sc->sc_nirqs, sc->sc_ndie); 207 208 for (i = 0; i < sc->sc_ndie; i++) { 209 sc->sc_isrcs[i] = mallocarray(sc->sc_nirqs, 210 sizeof(**sc->sc_isrcs), M_DEVBUF, M_WAITOK | M_ZERO); 211 } 212 213 #ifdef SMP 214 sc->sc_ipimasks = malloc(sizeof(*sc->sc_ipimasks) * mp_maxid + 1, 215 M_DEVBUF, M_WAITOK | M_ZERO); 216 sc->sc_cpuids = malloc(sizeof(*sc->sc_cpuids) * mp_maxid + 1, 217 M_DEVBUF, M_WAITOK | M_ZERO); 218 219 cpu = PCPU_GET(cpuid); 220 sc->sc_cpuids[cpu] = bus_read_4(sc->sc_mem, AIC_WHOAMI); 221 if (bootverbose) 222 device_printf(dev, "BSP CPU %d: whoami %x\n", cpu, 223 sc->sc_cpuids[cpu]); 224 #endif 225 226 227 name = device_get_nameunit(dev); 228 for (i = 0; i < sc->sc_ndie; i++) { 229 struct apple_aic_irqsrc *die_isrcs; 230 231 die_isrcs = sc->sc_isrcs[i]; 232 for (j = 0; j < sc->sc_nirqs; j++) { 233 isrc = &die_isrcs[j].ai_isrc; 234 die_isrcs[j].ai_pol = INTR_POLARITY_CONFORM; 235 die_isrcs[j].ai_trig = INTR_TRIGGER_CONFORM; 236 die_isrcs[j].ai_type = AIC_TYPE_INVAL; 237 die_isrcs[j].ai_irq = j; 238 239 error = intr_isrc_register(isrc, dev, 0, "%s,d%us%u", name, 240 i, j); 241 if (error != 0) { 242 device_printf(dev, "Unable to register irq %u:%u\n", 243 i, j); 244 return (error); 245 } 246 } 247 } 248 249 xref = OF_xref_from_node(ofw_bus_get_node(dev)); 250 if (intr_pic_register(dev, xref) == NULL) { 251 device_printf(dev, "Unable to register interrupt handler\n"); 252 return (ENXIO); 253 } 254 255 if (intr_pic_claim_root(dev, xref, apple_aic_irq, sc, 256 INTR_ROOT_IRQ) != 0) { 257 device_printf(dev, 258 "Unable to set root interrupt controller\n"); 259 intr_pic_deregister(dev, xref); 260 return (ENXIO); 261 } 262 263 if (intr_pic_claim_root(dev, xref, apple_aic_fiq, sc, 264 INTR_ROOT_FIQ) != 0) { 265 device_printf(dev, 266 "Unable to set root fiq controller\n"); 267 intr_pic_deregister(dev, xref); 268 return (ENXIO); 269 } 270 271 #ifdef SMP 272 if (intr_ipi_pic_register(dev, 0) != 0) { 273 device_printf(dev, "could not register for IPIs\n"); 274 return (ENXIO); 275 } 276 #endif 277 278 OF_device_register_xref(xref, dev); 279 280 return (0); 281 } 282 283 static int 284 apple_aic_map_intr_fdt(struct apple_aic_softc *sc, 285 struct intr_map_data_fdt *data, u_int *irq, enum apple_aic_irq_type *typep, 286 enum intr_polarity *polp, enum intr_trigger *trigp, u_int *die) 287 { 288 if (data->ncells != 3) 289 return (EINVAL); 290 291 /* XXX AIC2 */ 292 *die = 0; 293 294 /* 295 * The first cell is the interrupt type: 296 * 0 = IRQ 297 * 1 = FIQ 298 * The second cell is the interrupt number 299 * The third cell is the flags 300 */ 301 switch(data->cells[0]) { 302 case 0: 303 if (typep != NULL) 304 *typep = AIC_TYPE_IRQ; 305 break; 306 case 1: 307 if (typep != NULL) 308 *typep = AIC_TYPE_FIQ; 309 break; 310 default: 311 return (EINVAL); 312 } 313 314 *irq = data->cells[1]; 315 if (*irq > sc->sc_nirqs) 316 return (EINVAL); 317 318 if (trigp != NULL) { 319 if ((data->cells[2] & FDT_INTR_EDGE_MASK) != 0) 320 *trigp = INTR_TRIGGER_EDGE; 321 else 322 *trigp = INTR_TRIGGER_LEVEL; 323 } 324 if (polp != NULL) { 325 if ((data->cells[2] & FDT_INTR_LEVEL_HIGH) != 0) 326 *polp = INTR_POLARITY_HIGH; 327 else 328 *polp = INTR_POLARITY_LOW; 329 } 330 331 return (0); 332 } 333 334 static int 335 apple_aic_map_intr(device_t dev, struct intr_map_data *data, 336 struct intr_irqsrc **isrcp) 337 { 338 struct apple_aic_softc *sc; 339 int error; 340 u_int irq; 341 u_int die; 342 343 sc = device_get_softc(dev); 344 345 error = 0; 346 switch(data->type) { 347 case INTR_MAP_DATA_FDT: 348 error = apple_aic_map_intr_fdt(sc, 349 (struct intr_map_data_fdt *)data, &irq, NULL, NULL, NULL, 350 &die); 351 if (error == 0) 352 *isrcp = &sc->sc_isrcs[0 /* XXX */][irq].ai_isrc; 353 break; 354 default: 355 return (ENOTSUP); 356 } 357 358 return (error); 359 } 360 361 static int 362 apple_aic_setup_intr(device_t dev, struct intr_irqsrc *isrc, 363 struct resource *res, struct intr_map_data *data) 364 { 365 struct apple_aic_softc *sc; 366 enum apple_aic_irq_type type; 367 struct apple_aic_irqsrc *ai; 368 enum intr_trigger trig; 369 enum intr_polarity pol; 370 int error; 371 u_int die, irq; 372 373 sc = device_get_softc(dev); 374 ai = (struct apple_aic_irqsrc *)isrc; 375 376 if (data != NULL) { 377 KASSERT(data->type == INTR_MAP_DATA_FDT, 378 ("%s: Only FDT data is supported (got %#x)", __func__, 379 data->type)); 380 error = apple_aic_map_intr_fdt(sc, 381 (struct intr_map_data_fdt *)data, &irq, &type, &pol, &trig, 382 &die); 383 if (error != 0) 384 return (error); 385 } else { 386 pol = INTR_POLARITY_CONFORM; 387 trig = INTR_TRIGGER_CONFORM; 388 } 389 390 if (isrc->isrc_handlers != 0) { 391 /* TODO */ 392 return (0); 393 } 394 395 if (pol == INTR_POLARITY_CONFORM) 396 pol = INTR_POLARITY_LOW; 397 if (trig == INTR_TRIGGER_CONFORM) 398 trig = INTR_TRIGGER_EDGE; 399 400 ai->ai_pol = pol; 401 ai->ai_trig = trig; 402 ai->ai_type = type; 403 404 /* 405 * Only the timer uses FIQs. These could be sent to any CPU. 406 */ 407 switch (type) { 408 case AIC_TYPE_IRQ: 409 /* XXX die sensitive? */ 410 aic_next_cpu = intr_irq_next_cpu(aic_next_cpu, &all_cpus); 411 bus_write_4(sc->sc_mem, AIC_TARGET_CPU(irq), 412 1 << sc->sc_cpuids[aic_next_cpu]); 413 break; 414 case AIC_TYPE_FIQ: 415 isrc->isrc_flags |= INTR_ISRCF_PPI; 416 break; 417 default: 418 return (EINVAL); 419 } 420 421 return (0); 422 } 423 424 static int 425 apple_aic_teardown_intr(device_t dev, struct intr_irqsrc *isrc, 426 struct resource *res, struct intr_map_data *data) 427 { 428 panic("%s\n", __func__); 429 } 430 431 static void 432 apple_aic_enable_intr(device_t dev, struct intr_irqsrc *isrc) 433 { 434 struct apple_aic_irqsrc *ai; 435 struct apple_aic_softc *sc; 436 u_int irq; 437 438 ai = (struct apple_aic_irqsrc *)isrc; 439 irq = ai->ai_irq; 440 switch(ai->ai_type) { 441 case AIC_TYPE_IRQ: 442 sc = device_get_softc(dev); 443 bus_write_4(sc->sc_mem, AIC_MASK_CLEAR(irq), AIC_IRQ_MASK(irq)); 444 break; 445 case AIC_TYPE_IPI: 446 /* Nothing needed here. */ 447 break; 448 case AIC_TYPE_FIQ: 449 /* TODO */ 450 break; 451 default: 452 panic("%s: %x\n", __func__, ai->ai_type); 453 } 454 } 455 456 static void 457 apple_aic_disable_intr(device_t dev, struct intr_irqsrc *isrc) 458 { 459 struct apple_aic_irqsrc *ai; 460 struct apple_aic_softc *sc; 461 u_int irq; 462 463 ai = (struct apple_aic_irqsrc *)isrc; 464 irq = ai->ai_irq; 465 switch(ai->ai_type) { 466 case AIC_TYPE_IRQ: 467 sc = device_get_softc(dev); 468 bus_write_4(sc->sc_mem, AIC_MASK_SET(irq), AIC_IRQ_MASK(irq)); 469 break; 470 case AIC_TYPE_IPI: 471 /* Nothing needed here. */ 472 break; 473 case AIC_TYPE_FIQ: 474 /* TODO */ 475 break; 476 default: 477 panic("%s: %x\n", __func__, ai->ai_type); 478 } 479 } 480 481 static void 482 apple_aic_post_filter(device_t dev, struct intr_irqsrc *isrc) 483 { 484 struct apple_aic_softc *sc; 485 struct apple_aic_irqsrc *ai; 486 int irq; 487 488 ai = (struct apple_aic_irqsrc *)isrc; 489 irq = ai->ai_irq; 490 switch(ai->ai_type) { 491 case AIC_TYPE_IRQ: 492 sc = device_get_softc(dev); 493 bus_write_4(sc->sc_mem, AIC_SW_CLEAR(irq), AIC_IRQ_MASK(irq)); 494 bus_write_4(sc->sc_mem, AIC_MASK_CLEAR(irq), AIC_IRQ_MASK(irq)); 495 break; 496 case AIC_TYPE_FIQ: 497 /* TODO */ 498 break; 499 default: 500 panic("%s: %x\n", __func__, ai->ai_type); 501 } 502 } 503 504 static void 505 apple_aic_pre_ithread(device_t dev, struct intr_irqsrc *isrc) 506 { 507 struct apple_aic_softc *sc; 508 struct apple_aic_irqsrc *ai; 509 int irq; 510 511 ai = (struct apple_aic_irqsrc *)isrc; 512 sc = device_get_softc(dev); 513 irq = ai->ai_irq; 514 bus_write_4(sc->sc_mem, AIC_SW_CLEAR(irq), AIC_IRQ_MASK(irq)); 515 apple_aic_disable_intr(dev, isrc); 516 /* ACK IT */ 517 } 518 519 static void 520 apple_aic_post_ithread(device_t dev, struct intr_irqsrc *isrc) 521 { 522 struct apple_aic_softc *sc; 523 struct apple_aic_irqsrc *ai; 524 int irq; 525 526 ai = (struct apple_aic_irqsrc *)isrc; 527 sc = device_get_softc(dev); 528 irq = ai->ai_irq; 529 530 bus_write_4(sc->sc_mem, AIC_MASK_CLEAR(irq), AIC_IRQ_MASK(irq)); 531 apple_aic_enable_intr(dev, isrc); 532 } 533 534 #ifdef SMP 535 static void 536 apple_aic_ipi_received(struct apple_aic_softc *sc, struct trapframe *tf) 537 { 538 uint32_t mask; 539 uint32_t ipi; 540 int cpu; 541 542 cpu = PCPU_GET(cpuid); 543 544 mask = atomic_readandclear_32(&sc->sc_ipimasks[cpu]); 545 546 while (mask != 0) { 547 ipi = ffs(mask) - 1; 548 mask &= ~(1 << ipi); 549 550 intr_ipi_dispatch(ipi); 551 } 552 } 553 #endif 554 555 static int 556 apple_aic_irq(void *arg) 557 { 558 struct apple_aic_softc *sc; 559 uint32_t die, event, irq, type; 560 struct apple_aic_irqsrc *aisrc; 561 struct trapframe *tf; 562 563 sc = arg; 564 tf = curthread->td_intr_frame; 565 566 event = bus_read_4(sc->sc_mem, AIC_EVENT); 567 type = AIC_EVENT_TYPE(event); 568 569 /* If we get an IPI here, we really goofed. */ 570 MPASS(type != AIC_EVENT_TYPE_IPI); 571 572 if (type != AIC_EVENT_TYPE_IRQ) { 573 if (type != AIC_EVENT_TYPE_NONE) 574 device_printf(sc->sc_dev, "unexpected event type %d\n", 575 type); 576 return (FILTER_STRAY); 577 } 578 579 die = AIC_EVENT_DIE(event); 580 irq = AIC_EVENT_IRQ(event); 581 582 if (die >= sc->sc_ndie) 583 panic("%s: unexpected die %d", __func__, die); 584 if (irq >= sc->sc_nirqs) 585 panic("%s: unexpected irq %d", __func__, irq); 586 587 aisrc = &sc->sc_isrcs[die][irq]; 588 if (intr_isrc_dispatch(&aisrc->ai_isrc, tf) != 0) { 589 device_printf(sc->sc_dev, "Stray irq %u:%u disabled\n", 590 die, irq); 591 return (FILTER_STRAY); 592 } 593 594 return (FILTER_HANDLED); 595 } 596 597 static int 598 apple_aic_fiq(void *arg) 599 { 600 struct apple_aic_softc *sc; 601 struct apple_aic_irqsrc *isrcs; 602 struct trapframe *tf; 603 604 sc = arg; 605 tf = curthread->td_intr_frame; 606 607 #ifdef SMP 608 /* Handle IPIs. */ 609 if ((READ_SPECIALREG(AIC_IPI_SR_EL1) & AIC_IPI_SR_EL1_PENDING) != 0) { 610 WRITE_SPECIALREG(AIC_IPI_SR_EL1, AIC_IPI_SR_EL1_PENDING); 611 apple_aic_ipi_received(sc, tf); 612 } 613 #endif 614 615 /* 616 * FIQs don't store any state in the interrupt controller at all outside 617 * of IPI handling, so we have to probe around outside of AIC to 618 * determine if we might have been fired off due to a timer. 619 */ 620 isrcs = sc->sc_isrcs[0]; 621 if ((READ_SPECIALREG(cntv_ctl_el0) & CNTV_CTL_BITS) == 622 (CNTV_CTL_ENABLE | CNTV_CTL_ISTATUS)) { 623 intr_isrc_dispatch(&isrcs[AIC_TMR_GUEST_VIRT].ai_isrc, tf); 624 } 625 626 if (has_hyp()) { 627 uint64_t reg; 628 629 if ((READ_SPECIALREG(cntp_ctl_el0) & CNTV_CTL_ISTATUS) != 0) { 630 intr_isrc_dispatch(&isrcs[AIC_TMR_GUEST_PHYS].ai_isrc, 631 tf); 632 } 633 634 reg = READ_SPECIALREG(AIC_FIQ_VM_TIMER); 635 if ((reg & AIC_FIQ_VM_TIMER_PEN) != 0) { 636 intr_isrc_dispatch(&isrcs[AIC_TMR_HV_PHYS].ai_isrc, tf); 637 } 638 639 if ((reg & AIC_FIQ_VM_TIMER_VEN) != 0) { 640 intr_isrc_dispatch(&isrcs[AIC_TMR_HV_VIRT].ai_isrc, tf); 641 } 642 } 643 644 return (FILTER_HANDLED); 645 } 646 647 #ifdef SMP 648 static int 649 apple_aic_bind_intr(device_t dev, struct intr_irqsrc *isrc) 650 { 651 struct apple_aic_softc *sc = device_get_softc(dev); 652 static int aic_next_cpu; 653 uint32_t targets = 0; 654 u_int irq, cpu; 655 656 MPASS(((struct apple_aic_irqsrc *)isrc)->ai_type == AIC_TYPE_IRQ); 657 irq = ((struct apple_aic_irqsrc *)isrc)->ai_irq; 658 if (CPU_EMPTY(&isrc->isrc_cpu)) { 659 aic_next_cpu = intr_irq_next_cpu(aic_next_cpu, &all_cpus); 660 CPU_SETOF(aic_next_cpu, &isrc->isrc_cpu); 661 bus_write_4(sc->sc_mem, AIC_TARGET_CPU(irq), 662 sc->sc_cpuids[aic_next_cpu] << 1); 663 } else { 664 CPU_FOREACH_ISSET(cpu, &isrc->isrc_cpu) { 665 targets |= sc->sc_cpuids[cpu] << 1; 666 } 667 bus_write_4(sc->sc_mem, AIC_TARGET_CPU(irq), targets); 668 } 669 return (0); 670 } 671 672 static void 673 apple_aic_ipi_send(device_t dev, struct intr_irqsrc *isrc, cpuset_t cpus, 674 u_int ipi) 675 { 676 struct apple_aic_softc *sc; 677 uint64_t aff, localgrp, sendmask; 678 u_int cpu; 679 680 sc = device_get_softc(dev); 681 sendmask = 0; 682 localgrp = CPU_AFF1(CPU_AFFINITY(PCPU_GET(cpuid))); 683 684 KASSERT(isrc == &sc->sc_ipi_srcs[ipi].ai_isrc, 685 ("%s: bad ISRC %p argument", __func__, isrc)); 686 for (cpu = 0; cpu <= mp_maxid; cpu++) { 687 if (CPU_ISSET(cpu, &cpus)) { 688 aff = CPU_AFFINITY(cpu); 689 sendmask = CPU_AFF0(aff); 690 atomic_set_32(&sc->sc_ipimasks[cpu], 1 << ipi); 691 692 /* 693 * The above write to sc_ipimasks needs to be visible 694 * before we write to the ipi register to avoid the 695 * targetted CPU missing the dispatch in 696 * apple_aic_ipi_received(). Note that WRITE_SPECIALREG 697 * isn't a memory operation, so we can't relax this to a 698 * a dmb. 699 */ 700 dsb(ishst); 701 702 if (CPU_AFF1(aff) == localgrp) { 703 WRITE_SPECIALREG(AIC_IPI_LOCAL_RR_EL1, 704 sendmask); 705 } else { 706 sendmask |= CPU_AFF1(aff) << 16; 707 WRITE_SPECIALREG(AIC_IPI_GLOBAL_RR_EL1, 708 sendmask); 709 } 710 711 isb(); 712 } 713 } 714 } 715 716 static int 717 apple_aic_ipi_setup(device_t dev, u_int ipi, struct intr_irqsrc **isrcp) 718 { 719 struct apple_aic_softc *sc = device_get_softc(dev); 720 struct apple_aic_irqsrc *ai; 721 722 KASSERT(ipi < AIC_NIPIS, ("%s: ipi %u too high", __func__, ipi)); 723 724 ai = &sc->sc_ipi_srcs[ipi]; 725 ai->ai_type = AIC_TYPE_IPI; 726 727 *isrcp = &ai->ai_isrc; 728 return (0); 729 } 730 731 static void 732 apple_aic_init_secondary(device_t dev, uint32_t rootnum) 733 { 734 struct apple_aic_softc *sc = device_get_softc(dev); 735 u_int cpu = PCPU_GET(cpuid); 736 737 /* We don't need to re-initialize for the FIQ root. */ 738 if (rootnum != INTR_ROOT_IRQ) 739 return; 740 741 sc->sc_cpuids[cpu] = bus_read_4(sc->sc_mem, AIC_WHOAMI); 742 if (bootverbose) 743 device_printf(dev, "CPU %d: whoami %x\n", cpu, 744 sc->sc_cpuids[cpu]); 745 746 bus_write_4(sc->sc_mem, AIC_IPI_MASK_SET, AIC_IPI_SELF | AIC_IPI_OTHER); 747 } 748 #endif 749 750 static device_method_t apple_aic_methods[] = { 751 /* Device interface */ 752 DEVMETHOD(device_probe, apple_aic_probe), 753 DEVMETHOD(device_attach, apple_aic_attach), 754 755 /* Interrupt controller interface */ 756 DEVMETHOD(pic_disable_intr, apple_aic_disable_intr), 757 DEVMETHOD(pic_enable_intr, apple_aic_enable_intr), 758 DEVMETHOD(pic_map_intr, apple_aic_map_intr), 759 DEVMETHOD(pic_setup_intr, apple_aic_setup_intr), 760 DEVMETHOD(pic_teardown_intr, apple_aic_teardown_intr), 761 DEVMETHOD(pic_post_filter, apple_aic_post_filter), 762 DEVMETHOD(pic_post_ithread, apple_aic_post_ithread), 763 DEVMETHOD(pic_pre_ithread, apple_aic_pre_ithread), 764 #ifdef SMP 765 DEVMETHOD(pic_bind_intr, apple_aic_bind_intr), 766 DEVMETHOD(pic_init_secondary, apple_aic_init_secondary), 767 DEVMETHOD(pic_ipi_send, apple_aic_ipi_send), 768 DEVMETHOD(pic_ipi_setup, apple_aic_ipi_setup), 769 #endif 770 771 /* End */ 772 DEVMETHOD_END 773 }; 774 775 static DEFINE_CLASS_0(aic, apple_aic_driver, apple_aic_methods, 776 sizeof(struct apple_aic_softc)); 777 778 EARLY_DRIVER_MODULE(aic, simplebus, apple_aic_driver, 0, 0, 779 BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); 780