1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2021 Andrew Turner 5 * Copyright (c) 2022 Michael J. Karels <karels@freebsd.org> 6 * Copyright (c) 2022 Kyle Evans <kevans@FreeBSD.org> 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include "opt_platform.h" 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/bus.h> 35 #include <sys/kernel.h> 36 #include <sys/module.h> 37 #include <sys/proc.h> 38 #include <sys/rman.h> 39 #include <sys/smp.h> 40 41 #include <machine/bus.h> 42 #include <machine/machdep.h> 43 #ifdef SMP 44 #include <machine/intr.h> 45 #include <machine/smp.h> 46 #endif 47 48 #include <dev/fdt/fdt_intr.h> 49 50 #include <dev/ofw/openfirm.h> 51 #include <dev/ofw/ofw_bus.h> 52 #include <dev/ofw/ofw_bus_subr.h> 53 54 #include <dt-bindings/interrupt-controller/apple-aic.h> 55 56 #include "pic_if.h" 57 58 #define AIC_INFO 0x0004 59 #define AIC_INFO_NDIE(val) (((val) >> 24) & 0xf) 60 #define AIC_INFO_NIRQS(val) ((val) & 0x0000ffff) 61 62 #define AIC_WHOAMI 0x2000 63 #define AIC_EVENT 0x2004 64 #define AIC_EVENT_DIE(val) (((val) >> 24) & 0xff) 65 #define AIC_EVENT_TYPE(val) (((val) >> 16) & 0xff) 66 #define AIC_EVENT_TYPE_NONE 0 67 #define AIC_EVENT_TYPE_IRQ 1 68 #define AIC_EVENT_TYPE_IPI 4 69 #define AIC_EVENT_IRQ(val) ((val) & 0xffff) 70 #define AIC_EVENT_IPI_OTHER 1 71 #define AIC_EVENT_IPI_SELF 2 72 #define AIC_IPI_SEND 0x2008 73 #define AIC_IPI_ACK 0x200c 74 #define AIC_IPI_MASK_SET 0x2024 75 #define AIC_IPI_MASK_CLR 0x2028 76 #define AIC_IPI_OTHER 0x00000001 77 #define AIC_IPI_SELF 0x80000000 78 #define AIC_TARGET_CPU(irq) (0x3000 + (irq) * 4) 79 #define AIC_SW_SET(irq) (0x4000 + (((irq) >> 5) * 4)) 80 #define AIC_SW_CLEAR(irq) (0x4080 + (((irq) >> 5) * 4)) 81 #define AIC_MASK_SET(irq) (0x4100 + (((irq) >> 5) * 4)) 82 #define AIC_MASK_CLEAR(irq) (0x4180 + (((irq) >> 5) * 4)) 83 #define AIC_IRQ_MASK(irq) (1u << ((irq) & 0x1f)) 84 85 #define AIC_IPI_LOCAL_RR_EL1 s3_5_c15_c0_0 86 #define AIC_IPI_GLOBAL_RR_EL1 s3_5_c15_c0_1 87 88 #define AIC_IPI_SR_EL1 s3_5_c15_c1_1 89 #define AIC_IPI_SR_EL1_PENDING (1 << 0) 90 91 #define AIC_FIQ_VM_TIMER s3_5_c15_c1_3 92 #define AIC_FIQ_VM_TIMER_VEN (1 << 0) 93 #define AIC_FIQ_VM_TIMER_PEN (1 << 1) 94 #define AIC_FIQ_VM_TIMER_BITS (AIC_FIQ_VM_TIMER_VEN | AIC_FIQ_VM_TIMER_PEN) 95 96 #define CNTV_CTL_ENABLE (1 << 0) 97 #define CNTV_CTL_IMASK (1 << 1) 98 #define CNTV_CTL_ISTATUS (1 << 2) 99 #define CNTV_CTL_BITS \ 100 (CNTV_CTL_ENABLE | CNTV_CTL_IMASK | CNTV_CTL_ISTATUS) 101 102 #define AIC_MAXCPUS 32 103 #define AIC_MAXDIES 4 104 105 static struct ofw_compat_data compat_data[] = { 106 { "apple,aic", 1 }, 107 { NULL, 0 } 108 }; 109 110 enum apple_aic_irq_type { 111 AIC_TYPE_INVAL, 112 AIC_TYPE_IRQ, 113 AIC_TYPE_FIQ, 114 AIC_TYPE_IPI, 115 }; 116 117 struct apple_aic_irqsrc { 118 struct intr_irqsrc ai_isrc; 119 enum apple_aic_irq_type ai_type; 120 struct { 121 /* AIC_TYPE_IRQ */ 122 enum intr_polarity ai_pol; 123 enum intr_trigger ai_trig; 124 u_int ai_irq; 125 }; 126 }; 127 128 #ifdef SMP 129 #define AIC_NIPIS INTR_IPI_COUNT 130 #endif 131 132 struct apple_aic_softc { 133 device_t sc_dev; 134 struct resource *sc_mem; 135 struct apple_aic_irqsrc *sc_isrcs[AIC_MAXDIES]; 136 u_int sc_nirqs; 137 u_int sc_ndie; 138 #ifdef SMP 139 struct apple_aic_irqsrc sc_ipi_srcs[AIC_NIPIS]; 140 uint32_t *sc_ipimasks; 141 #endif 142 u_int *sc_cpuids; /* cpu index to AIC CPU ID */ 143 }; 144 145 static u_int aic_next_cpu; 146 147 static device_probe_t apple_aic_probe; 148 static device_attach_t apple_aic_attach; 149 150 static pic_disable_intr_t apple_aic_disable_intr; 151 static pic_enable_intr_t apple_aic_enable_intr; 152 static pic_map_intr_t apple_aic_map_intr; 153 static pic_setup_intr_t apple_aic_setup_intr; 154 static pic_teardown_intr_t apple_aic_teardown_intr; 155 static pic_post_filter_t apple_aic_post_filter; 156 static pic_post_ithread_t apple_aic_post_ithread; 157 static pic_pre_ithread_t apple_aic_pre_ithread; 158 #ifdef SMP 159 static pic_bind_intr_t apple_aic_bind_intr; 160 static pic_init_secondary_t apple_aic_init_secondary; 161 static pic_ipi_send_t apple_aic_ipi_send; 162 static pic_ipi_setup_t apple_aic_ipi_setup; 163 #endif 164 165 static int apple_aic_irq(void *); 166 static int apple_aic_fiq(void *); 167 168 static int 169 apple_aic_probe(device_t dev) 170 { 171 172 if (!ofw_bus_status_okay(dev)) 173 return (ENXIO); 174 175 if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0) 176 return (ENXIO); 177 178 device_set_desc(dev, "Apple Interrupt Controller"); 179 return (BUS_PROBE_DEFAULT); 180 } 181 182 static int 183 apple_aic_attach(device_t dev) 184 { 185 struct apple_aic_softc *sc; 186 struct intr_irqsrc *isrc; 187 const char *name; 188 intptr_t xref; 189 int error, rid; 190 u_int i, cpu, j, info; 191 192 sc = device_get_softc(dev); 193 sc->sc_dev = dev; 194 195 rid = 0; 196 sc->sc_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 197 RF_ACTIVE); 198 if (sc->sc_mem == NULL) { 199 device_printf(dev, "Unable to allocate memory\n"); 200 return (ENXIO); 201 } 202 203 info = bus_read_4(sc->sc_mem, AIC_INFO); 204 sc->sc_nirqs = AIC_INFO_NIRQS(info); 205 sc->sc_ndie = AIC_INFO_NDIE(info) + 1; 206 if (bootverbose) 207 device_printf(dev, "Found %d interrupts, %d die\n", 208 sc->sc_nirqs, sc->sc_ndie); 209 210 for (i = 0; i < sc->sc_ndie; i++) { 211 sc->sc_isrcs[i] = mallocarray(sc->sc_nirqs, 212 sizeof(**sc->sc_isrcs), M_DEVBUF, M_WAITOK | M_ZERO); 213 } 214 215 #ifdef SMP 216 sc->sc_ipimasks = malloc(sizeof(*sc->sc_ipimasks) * mp_maxid + 1, 217 M_DEVBUF, M_WAITOK | M_ZERO); 218 #endif 219 sc->sc_cpuids = malloc(sizeof(*sc->sc_cpuids) * mp_maxid + 1, 220 M_DEVBUF, M_WAITOK | M_ZERO); 221 222 cpu = PCPU_GET(cpuid); 223 sc->sc_cpuids[cpu] = bus_read_4(sc->sc_mem, AIC_WHOAMI); 224 if (bootverbose) 225 device_printf(dev, "BSP CPU %d: whoami %x\n", cpu, 226 sc->sc_cpuids[cpu]); 227 228 name = device_get_nameunit(dev); 229 for (i = 0; i < sc->sc_ndie; i++) { 230 struct apple_aic_irqsrc *die_isrcs; 231 232 die_isrcs = sc->sc_isrcs[i]; 233 for (j = 0; j < sc->sc_nirqs; j++) { 234 isrc = &die_isrcs[j].ai_isrc; 235 die_isrcs[j].ai_pol = INTR_POLARITY_CONFORM; 236 die_isrcs[j].ai_trig = INTR_TRIGGER_CONFORM; 237 die_isrcs[j].ai_type = AIC_TYPE_INVAL; 238 die_isrcs[j].ai_irq = j; 239 240 error = intr_isrc_register(isrc, dev, 0, "%s,d%us%u", name, 241 i, j); 242 if (error != 0) { 243 device_printf(dev, "Unable to register irq %u:%u\n", 244 i, j); 245 return (error); 246 } 247 } 248 } 249 250 xref = OF_xref_from_node(ofw_bus_get_node(dev)); 251 if (intr_pic_register(dev, xref) == NULL) { 252 device_printf(dev, "Unable to register interrupt handler\n"); 253 return (ENXIO); 254 } 255 256 if (intr_pic_claim_root(dev, xref, apple_aic_irq, sc, 257 INTR_ROOT_IRQ) != 0) { 258 device_printf(dev, 259 "Unable to set root interrupt controller\n"); 260 intr_pic_deregister(dev, xref); 261 return (ENXIO); 262 } 263 264 if (intr_pic_claim_root(dev, xref, apple_aic_fiq, sc, 265 INTR_ROOT_FIQ) != 0) { 266 device_printf(dev, 267 "Unable to set root fiq controller\n"); 268 intr_pic_deregister(dev, xref); 269 return (ENXIO); 270 } 271 272 #ifdef SMP 273 if (intr_ipi_pic_register(dev, 0) != 0) { 274 device_printf(dev, "could not register for IPIs\n"); 275 return (ENXIO); 276 } 277 #endif 278 279 OF_device_register_xref(xref, dev); 280 281 return (0); 282 } 283 284 static int 285 apple_aic_map_intr_fdt(struct apple_aic_softc *sc, 286 struct intr_map_data_fdt *data, u_int *irq, enum apple_aic_irq_type *typep, 287 enum intr_polarity *polp, enum intr_trigger *trigp, u_int *die) 288 { 289 if (data->ncells != 3) 290 return (EINVAL); 291 292 /* XXX AIC2 */ 293 *die = 0; 294 295 /* 296 * The first cell is the interrupt type: 297 * 0 = IRQ 298 * 1 = FIQ 299 * The second cell is the interrupt number 300 * The third cell is the flags 301 */ 302 switch(data->cells[0]) { 303 case 0: 304 if (typep != NULL) 305 *typep = AIC_TYPE_IRQ; 306 break; 307 case 1: 308 if (typep != NULL) 309 *typep = AIC_TYPE_FIQ; 310 break; 311 default: 312 return (EINVAL); 313 } 314 315 *irq = data->cells[1]; 316 if (*irq > sc->sc_nirqs) 317 return (EINVAL); 318 319 if (trigp != NULL) { 320 if ((data->cells[2] & FDT_INTR_EDGE_MASK) != 0) 321 *trigp = INTR_TRIGGER_EDGE; 322 else 323 *trigp = INTR_TRIGGER_LEVEL; 324 } 325 if (polp != NULL) { 326 if ((data->cells[2] & FDT_INTR_LEVEL_HIGH) != 0) 327 *polp = INTR_POLARITY_HIGH; 328 else 329 *polp = INTR_POLARITY_LOW; 330 } 331 332 return (0); 333 } 334 335 static int 336 apple_aic_map_intr(device_t dev, struct intr_map_data *data, 337 struct intr_irqsrc **isrcp) 338 { 339 struct apple_aic_softc *sc; 340 int error; 341 u_int irq; 342 u_int die; 343 344 sc = device_get_softc(dev); 345 346 error = 0; 347 switch(data->type) { 348 case INTR_MAP_DATA_FDT: 349 error = apple_aic_map_intr_fdt(sc, 350 (struct intr_map_data_fdt *)data, &irq, NULL, NULL, NULL, 351 &die); 352 if (error == 0) 353 *isrcp = &sc->sc_isrcs[0 /* XXX */][irq].ai_isrc; 354 break; 355 default: 356 return (ENOTSUP); 357 } 358 359 return (error); 360 } 361 362 static int 363 apple_aic_setup_intr(device_t dev, struct intr_irqsrc *isrc, 364 struct resource *res, struct intr_map_data *data) 365 { 366 struct apple_aic_softc *sc; 367 enum apple_aic_irq_type type; 368 struct apple_aic_irqsrc *ai; 369 enum intr_trigger trig; 370 enum intr_polarity pol; 371 int error; 372 u_int die, irq; 373 374 sc = device_get_softc(dev); 375 ai = (struct apple_aic_irqsrc *)isrc; 376 377 if (data != NULL) { 378 KASSERT(data->type == INTR_MAP_DATA_FDT, 379 ("%s: Only FDT data is supported (got %#x)", __func__, 380 data->type)); 381 error = apple_aic_map_intr_fdt(sc, 382 (struct intr_map_data_fdt *)data, &irq, &type, &pol, &trig, 383 &die); 384 if (error != 0) 385 return (error); 386 } else { 387 pol = INTR_POLARITY_CONFORM; 388 trig = INTR_TRIGGER_CONFORM; 389 } 390 391 if (isrc->isrc_handlers != 0) { 392 /* TODO */ 393 return (0); 394 } 395 396 if (pol == INTR_POLARITY_CONFORM) 397 pol = INTR_POLARITY_LOW; 398 if (trig == INTR_TRIGGER_CONFORM) 399 trig = INTR_TRIGGER_EDGE; 400 401 ai->ai_pol = pol; 402 ai->ai_trig = trig; 403 ai->ai_type = type; 404 405 /* 406 * Only the timer uses FIQs. These could be sent to any CPU. 407 */ 408 switch (type) { 409 case AIC_TYPE_IRQ: 410 /* XXX die sensitive? */ 411 aic_next_cpu = intr_irq_next_cpu(aic_next_cpu, &all_cpus); 412 bus_write_4(sc->sc_mem, AIC_TARGET_CPU(irq), 413 1 << sc->sc_cpuids[aic_next_cpu]); 414 break; 415 case AIC_TYPE_FIQ: 416 isrc->isrc_flags |= INTR_ISRCF_PPI; 417 break; 418 default: 419 return (EINVAL); 420 } 421 422 return (0); 423 } 424 425 static int 426 apple_aic_teardown_intr(device_t dev, struct intr_irqsrc *isrc, 427 struct resource *res, struct intr_map_data *data) 428 { 429 panic("%s\n", __func__); 430 } 431 432 static void 433 apple_aic_enable_intr(device_t dev, struct intr_irqsrc *isrc) 434 { 435 struct apple_aic_irqsrc *ai; 436 struct apple_aic_softc *sc; 437 u_int irq; 438 439 ai = (struct apple_aic_irqsrc *)isrc; 440 irq = ai->ai_irq; 441 switch(ai->ai_type) { 442 case AIC_TYPE_IRQ: 443 sc = device_get_softc(dev); 444 bus_write_4(sc->sc_mem, AIC_MASK_CLEAR(irq), AIC_IRQ_MASK(irq)); 445 break; 446 case AIC_TYPE_IPI: 447 /* Nothing needed here. */ 448 break; 449 case AIC_TYPE_FIQ: 450 /* TODO */ 451 break; 452 default: 453 panic("%s: %x\n", __func__, ai->ai_type); 454 } 455 } 456 457 static void 458 apple_aic_disable_intr(device_t dev, struct intr_irqsrc *isrc) 459 { 460 struct apple_aic_irqsrc *ai; 461 struct apple_aic_softc *sc; 462 u_int irq; 463 464 ai = (struct apple_aic_irqsrc *)isrc; 465 irq = ai->ai_irq; 466 switch(ai->ai_type) { 467 case AIC_TYPE_IRQ: 468 sc = device_get_softc(dev); 469 bus_write_4(sc->sc_mem, AIC_MASK_SET(irq), AIC_IRQ_MASK(irq)); 470 break; 471 case AIC_TYPE_IPI: 472 /* Nothing needed here. */ 473 break; 474 case AIC_TYPE_FIQ: 475 /* TODO */ 476 break; 477 default: 478 panic("%s: %x\n", __func__, ai->ai_type); 479 } 480 } 481 482 static void 483 apple_aic_post_filter(device_t dev, struct intr_irqsrc *isrc) 484 { 485 struct apple_aic_softc *sc; 486 struct apple_aic_irqsrc *ai; 487 int irq; 488 489 ai = (struct apple_aic_irqsrc *)isrc; 490 irq = ai->ai_irq; 491 switch(ai->ai_type) { 492 case AIC_TYPE_IRQ: 493 sc = device_get_softc(dev); 494 bus_write_4(sc->sc_mem, AIC_SW_CLEAR(irq), AIC_IRQ_MASK(irq)); 495 bus_write_4(sc->sc_mem, AIC_MASK_CLEAR(irq), AIC_IRQ_MASK(irq)); 496 break; 497 case AIC_TYPE_FIQ: 498 /* TODO */ 499 break; 500 default: 501 panic("%s: %x\n", __func__, ai->ai_type); 502 } 503 } 504 505 static void 506 apple_aic_pre_ithread(device_t dev, struct intr_irqsrc *isrc) 507 { 508 struct apple_aic_softc *sc; 509 struct apple_aic_irqsrc *ai; 510 int irq; 511 512 ai = (struct apple_aic_irqsrc *)isrc; 513 sc = device_get_softc(dev); 514 irq = ai->ai_irq; 515 bus_write_4(sc->sc_mem, AIC_SW_CLEAR(irq), AIC_IRQ_MASK(irq)); 516 apple_aic_disable_intr(dev, isrc); 517 /* ACK IT */ 518 } 519 520 static void 521 apple_aic_post_ithread(device_t dev, struct intr_irqsrc *isrc) 522 { 523 struct apple_aic_softc *sc; 524 struct apple_aic_irqsrc *ai; 525 int irq; 526 527 ai = (struct apple_aic_irqsrc *)isrc; 528 sc = device_get_softc(dev); 529 irq = ai->ai_irq; 530 531 bus_write_4(sc->sc_mem, AIC_MASK_CLEAR(irq), AIC_IRQ_MASK(irq)); 532 apple_aic_enable_intr(dev, isrc); 533 } 534 535 #ifdef SMP 536 static void 537 apple_aic_ipi_received(struct apple_aic_softc *sc, struct trapframe *tf) 538 { 539 uint32_t mask; 540 uint32_t ipi; 541 int cpu; 542 543 cpu = PCPU_GET(cpuid); 544 545 mask = atomic_readandclear_32(&sc->sc_ipimasks[cpu]); 546 547 while (mask != 0) { 548 ipi = ffs(mask) - 1; 549 mask &= ~(1 << ipi); 550 551 intr_ipi_dispatch(ipi); 552 } 553 } 554 #endif 555 556 static int 557 apple_aic_irq(void *arg) 558 { 559 struct apple_aic_softc *sc; 560 uint32_t die, event, irq, type; 561 struct apple_aic_irqsrc *aisrc; 562 struct trapframe *tf; 563 564 sc = arg; 565 tf = curthread->td_intr_frame; 566 567 event = bus_read_4(sc->sc_mem, AIC_EVENT); 568 type = AIC_EVENT_TYPE(event); 569 570 /* If we get an IPI here, we really goofed. */ 571 MPASS(type != AIC_EVENT_TYPE_IPI); 572 573 if (type != AIC_EVENT_TYPE_IRQ) { 574 if (type != AIC_EVENT_TYPE_NONE) 575 device_printf(sc->sc_dev, "unexpected event type %d\n", 576 type); 577 return (FILTER_STRAY); 578 } 579 580 die = AIC_EVENT_DIE(event); 581 irq = AIC_EVENT_IRQ(event); 582 583 if (die >= sc->sc_ndie) 584 panic("%s: unexpected die %d", __func__, die); 585 if (irq >= sc->sc_nirqs) 586 panic("%s: unexpected irq %d", __func__, irq); 587 588 aisrc = &sc->sc_isrcs[die][irq]; 589 if (intr_isrc_dispatch(&aisrc->ai_isrc, tf) != 0) { 590 device_printf(sc->sc_dev, "Stray irq %u:%u disabled\n", 591 die, irq); 592 return (FILTER_STRAY); 593 } 594 595 return (FILTER_HANDLED); 596 } 597 598 static int 599 apple_aic_fiq(void *arg) 600 { 601 struct apple_aic_softc *sc; 602 struct apple_aic_irqsrc *isrcs; 603 struct trapframe *tf; 604 605 sc = arg; 606 tf = curthread->td_intr_frame; 607 608 #ifdef SMP 609 /* Handle IPIs. */ 610 if ((READ_SPECIALREG(AIC_IPI_SR_EL1) & AIC_IPI_SR_EL1_PENDING) != 0) { 611 WRITE_SPECIALREG(AIC_IPI_SR_EL1, AIC_IPI_SR_EL1_PENDING); 612 apple_aic_ipi_received(sc, tf); 613 } 614 #endif 615 616 /* 617 * FIQs don't store any state in the interrupt controller at all outside 618 * of IPI handling, so we have to probe around outside of AIC to 619 * determine if we might have been fired off due to a timer. 620 */ 621 isrcs = sc->sc_isrcs[0]; 622 if ((READ_SPECIALREG(cntv_ctl_el0) & CNTV_CTL_BITS) == 623 (CNTV_CTL_ENABLE | CNTV_CTL_ISTATUS)) { 624 intr_isrc_dispatch(&isrcs[AIC_TMR_GUEST_VIRT].ai_isrc, tf); 625 } 626 627 if (has_hyp()) { 628 uint64_t reg; 629 630 if ((READ_SPECIALREG(cntp_ctl_el0) & CNTV_CTL_ISTATUS) != 0) { 631 intr_isrc_dispatch(&isrcs[AIC_TMR_GUEST_PHYS].ai_isrc, 632 tf); 633 } 634 635 reg = READ_SPECIALREG(AIC_FIQ_VM_TIMER); 636 if ((reg & AIC_FIQ_VM_TIMER_PEN) != 0) { 637 intr_isrc_dispatch(&isrcs[AIC_TMR_HV_PHYS].ai_isrc, tf); 638 } 639 640 if ((reg & AIC_FIQ_VM_TIMER_VEN) != 0) { 641 intr_isrc_dispatch(&isrcs[AIC_TMR_HV_VIRT].ai_isrc, tf); 642 } 643 } 644 645 return (FILTER_HANDLED); 646 } 647 648 #ifdef SMP 649 static int 650 apple_aic_bind_intr(device_t dev, struct intr_irqsrc *isrc) 651 { 652 struct apple_aic_softc *sc = device_get_softc(dev); 653 static int aic_next_cpu; 654 uint32_t targets = 0; 655 u_int irq, cpu; 656 657 MPASS(((struct apple_aic_irqsrc *)isrc)->ai_type == AIC_TYPE_IRQ); 658 irq = ((struct apple_aic_irqsrc *)isrc)->ai_irq; 659 if (CPU_EMPTY(&isrc->isrc_cpu)) { 660 aic_next_cpu = intr_irq_next_cpu(aic_next_cpu, &all_cpus); 661 CPU_SETOF(aic_next_cpu, &isrc->isrc_cpu); 662 bus_write_4(sc->sc_mem, AIC_TARGET_CPU(irq), 663 sc->sc_cpuids[aic_next_cpu] << 1); 664 } else { 665 CPU_FOREACH_ISSET(cpu, &isrc->isrc_cpu) { 666 targets |= sc->sc_cpuids[cpu] << 1; 667 } 668 bus_write_4(sc->sc_mem, AIC_TARGET_CPU(irq), targets); 669 } 670 return (0); 671 } 672 673 static void 674 apple_aic_ipi_send(device_t dev, struct intr_irqsrc *isrc, cpuset_t cpus, 675 u_int ipi) 676 { 677 struct apple_aic_softc *sc; 678 uint64_t aff, localgrp, sendmask; 679 u_int cpu; 680 681 sc = device_get_softc(dev); 682 sendmask = 0; 683 localgrp = CPU_AFF1(CPU_AFFINITY(PCPU_GET(cpuid))); 684 685 KASSERT(isrc == &sc->sc_ipi_srcs[ipi].ai_isrc, 686 ("%s: bad ISRC %p argument", __func__, isrc)); 687 for (cpu = 0; cpu <= mp_maxid; cpu++) { 688 if (CPU_ISSET(cpu, &cpus)) { 689 aff = CPU_AFFINITY(cpu); 690 sendmask = CPU_AFF0(aff); 691 atomic_set_32(&sc->sc_ipimasks[cpu], 1 << ipi); 692 693 /* 694 * The above write to sc_ipimasks needs to be visible 695 * before we write to the ipi register to avoid the 696 * targetted CPU missing the dispatch in 697 * apple_aic_ipi_received(). Note that WRITE_SPECIALREG 698 * isn't a memory operation, so we can't relax this to a 699 * a dmb. 700 */ 701 dsb(ishst); 702 703 if (CPU_AFF1(aff) == localgrp) { 704 WRITE_SPECIALREG(AIC_IPI_LOCAL_RR_EL1, 705 sendmask); 706 } else { 707 sendmask |= CPU_AFF1(aff) << 16; 708 WRITE_SPECIALREG(AIC_IPI_GLOBAL_RR_EL1, 709 sendmask); 710 } 711 712 isb(); 713 } 714 } 715 } 716 717 static int 718 apple_aic_ipi_setup(device_t dev, u_int ipi, struct intr_irqsrc **isrcp) 719 { 720 struct apple_aic_softc *sc = device_get_softc(dev); 721 struct apple_aic_irqsrc *ai; 722 723 KASSERT(ipi < AIC_NIPIS, ("%s: ipi %u too high", __func__, ipi)); 724 725 ai = &sc->sc_ipi_srcs[ipi]; 726 ai->ai_type = AIC_TYPE_IPI; 727 728 *isrcp = &ai->ai_isrc; 729 return (0); 730 } 731 732 static void 733 apple_aic_init_secondary(device_t dev, uint32_t rootnum) 734 { 735 struct apple_aic_softc *sc = device_get_softc(dev); 736 u_int cpu = PCPU_GET(cpuid); 737 738 /* We don't need to re-initialize for the FIQ root. */ 739 if (rootnum != INTR_ROOT_IRQ) 740 return; 741 742 sc->sc_cpuids[cpu] = bus_read_4(sc->sc_mem, AIC_WHOAMI); 743 if (bootverbose) 744 device_printf(dev, "CPU %d: whoami %x\n", cpu, 745 sc->sc_cpuids[cpu]); 746 747 bus_write_4(sc->sc_mem, AIC_IPI_MASK_SET, AIC_IPI_SELF | AIC_IPI_OTHER); 748 } 749 #endif 750 751 static device_method_t apple_aic_methods[] = { 752 /* Device interface */ 753 DEVMETHOD(device_probe, apple_aic_probe), 754 DEVMETHOD(device_attach, apple_aic_attach), 755 756 /* Interrupt controller interface */ 757 DEVMETHOD(pic_disable_intr, apple_aic_disable_intr), 758 DEVMETHOD(pic_enable_intr, apple_aic_enable_intr), 759 DEVMETHOD(pic_map_intr, apple_aic_map_intr), 760 DEVMETHOD(pic_setup_intr, apple_aic_setup_intr), 761 DEVMETHOD(pic_teardown_intr, apple_aic_teardown_intr), 762 DEVMETHOD(pic_post_filter, apple_aic_post_filter), 763 DEVMETHOD(pic_post_ithread, apple_aic_post_ithread), 764 DEVMETHOD(pic_pre_ithread, apple_aic_pre_ithread), 765 #ifdef SMP 766 DEVMETHOD(pic_bind_intr, apple_aic_bind_intr), 767 DEVMETHOD(pic_init_secondary, apple_aic_init_secondary), 768 DEVMETHOD(pic_ipi_send, apple_aic_ipi_send), 769 DEVMETHOD(pic_ipi_setup, apple_aic_ipi_setup), 770 #endif 771 772 /* End */ 773 DEVMETHOD_END 774 }; 775 776 static DEFINE_CLASS_0(aic, apple_aic_driver, apple_aic_methods, 777 sizeof(struct apple_aic_softc)); 778 779 EARLY_DRIVER_MODULE(aic, simplebus, apple_aic_driver, 0, 0, 780 BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); 781