1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2021 Andrew Turner 5 * Copyright (c) 2022 Michael J. Karels <karels@freebsd.org> 6 * Copyright (c) 2022 Kyle Evans <kevans@FreeBSD.org> 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include "opt_platform.h" 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/bus.h> 35 #include <sys/kernel.h> 36 #include <sys/module.h> 37 #include <sys/proc.h> 38 #include <sys/rman.h> 39 #include <sys/smp.h> 40 41 #include <machine/bus.h> 42 #include <machine/machdep.h> 43 #ifdef SMP 44 #include <machine/intr.h> 45 #include <machine/smp.h> 46 #endif 47 48 #include <dev/fdt/fdt_intr.h> 49 50 #include <dev/ofw/openfirm.h> 51 #include <dev/ofw/ofw_bus.h> 52 #include <dev/ofw/ofw_bus_subr.h> 53 54 #include <dt-bindings/interrupt-controller/apple-aic.h> 55 56 #include "pic_if.h" 57 58 #define AIC_INFO 0x0004 59 #define AIC_INFO_NDIE(val) (((val) >> 24) & 0xf) 60 #define AIC_INFO_NIRQS(val) ((val) & 0x0000ffff) 61 62 #define AIC_WHOAMI 0x2000 63 #define AIC_EVENT 0x2004 64 #define AIC_EVENT_DIE(val) (((val) >> 24) & 0xff) 65 #define AIC_EVENT_TYPE(val) (((val) >> 16) & 0xff) 66 #define AIC_EVENT_TYPE_NONE 0 67 #define AIC_EVENT_TYPE_IRQ 1 68 #define AIC_EVENT_TYPE_IPI 4 69 #define AIC_EVENT_IRQ(val) ((val) & 0xffff) 70 #define AIC_EVENT_IPI_OTHER 1 71 #define AIC_EVENT_IPI_SELF 2 72 #define AIC_IPI_SEND 0x2008 73 #define AIC_IPI_ACK 0x200c 74 #define AIC_IPI_MASK_SET 0x2024 75 #define AIC_IPI_MASK_CLR 0x2028 76 #define AIC_IPI_OTHER 0x00000001 77 #define AIC_IPI_SELF 0x80000000 78 #define AIC_TARGET_CPU(irq) (0x3000 + (irq) * 4) 79 #define AIC_SW_SET(irq) (0x4000 + (((irq) >> 5) * 4)) 80 #define AIC_SW_CLEAR(irq) (0x4080 + (((irq) >> 5) * 4)) 81 #define AIC_MASK_SET(irq) (0x4100 + (((irq) >> 5) * 4)) 82 #define AIC_MASK_CLEAR(irq) (0x4180 + (((irq) >> 5) * 4)) 83 #define AIC_IRQ_MASK(irq) (1u << ((irq) & 0x1f)) 84 85 #define AIC_IPI_LOCAL_RR_EL1 s3_5_c15_c0_0 86 #define AIC_IPI_GLOBAL_RR_EL1 s3_5_c15_c0_1 87 88 #define AIC_IPI_SR_EL1 s3_5_c15_c1_1 89 #define AIC_IPI_SR_EL1_PENDING (1 << 0) 90 91 #define AIC_FIQ_VM_TIMER s3_5_c15_c1_3 92 #define AIC_FIQ_VM_TIMER_VEN (1 << 0) 93 #define AIC_FIQ_VM_TIMER_PEN (1 << 1) 94 #define AIC_FIQ_VM_TIMER_BITS (AIC_FIQ_VM_TIMER_VEN | AIC_FIQ_VM_TIMER_PEN) 95 96 #define CNTV_CTL_ENABLE (1 << 0) 97 #define CNTV_CTL_IMASK (1 << 1) 98 #define CNTV_CTL_ISTATUS (1 << 2) 99 #define CNTV_CTL_BITS \ 100 (CNTV_CTL_ENABLE | CNTV_CTL_IMASK | CNTV_CTL_ISTATUS) 101 102 #define AIC_MAXCPUS 32 103 #define AIC_MAXDIES 4 104 105 static struct ofw_compat_data compat_data[] = { 106 { "apple,aic", 1 }, 107 { NULL, 0 } 108 }; 109 110 enum apple_aic_irq_type { 111 AIC_TYPE_INVAL, 112 AIC_TYPE_IRQ, 113 AIC_TYPE_FIQ, 114 AIC_TYPE_IPI, 115 }; 116 117 struct apple_aic_irqsrc { 118 struct intr_irqsrc ai_isrc; 119 enum apple_aic_irq_type ai_type; 120 struct { 121 /* AIC_TYPE_IRQ */ 122 enum intr_polarity ai_pol; 123 enum intr_trigger ai_trig; 124 u_int ai_irq; 125 }; 126 }; 127 128 #ifdef SMP 129 #define AIC_NIPIS INTR_IPI_COUNT 130 #endif 131 132 struct apple_aic_softc { 133 device_t sc_dev; 134 struct resource *sc_mem; 135 struct apple_aic_irqsrc *sc_isrcs[AIC_MAXDIES]; 136 u_int sc_nirqs; 137 u_int sc_ndie; 138 #ifdef SMP 139 struct apple_aic_irqsrc sc_ipi_srcs[AIC_NIPIS]; 140 u_int *sc_cpuids; /* cpu index to AIC CPU ID */ 141 uint32_t *sc_ipimasks; 142 #endif 143 }; 144 145 static u_int aic_next_cpu; 146 147 static device_probe_t apple_aic_probe; 148 static device_attach_t apple_aic_attach; 149 150 static pic_disable_intr_t apple_aic_disable_intr; 151 static pic_enable_intr_t apple_aic_enable_intr; 152 static pic_map_intr_t apple_aic_map_intr; 153 static pic_setup_intr_t apple_aic_setup_intr; 154 static pic_teardown_intr_t apple_aic_teardown_intr; 155 static pic_post_filter_t apple_aic_post_filter; 156 static pic_post_ithread_t apple_aic_post_ithread; 157 static pic_pre_ithread_t apple_aic_pre_ithread; 158 #ifdef SMP 159 static pic_bind_intr_t apple_aic_bind_intr; 160 static pic_init_secondary_t apple_aic_init_secondary; 161 static pic_ipi_send_t apple_aic_ipi_send; 162 static pic_ipi_setup_t apple_aic_ipi_setup; 163 #endif 164 165 static int apple_aic_irq(void *); 166 static int apple_aic_fiq(void *); 167 168 static int 169 apple_aic_probe(device_t dev) 170 { 171 172 if (!ofw_bus_status_okay(dev)) 173 return (ENXIO); 174 175 if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0) 176 return (ENXIO); 177 178 device_set_desc(dev, "Apple Interrupt Controller"); 179 return (BUS_PROBE_DEFAULT); 180 } 181 182 static int 183 apple_aic_attach(device_t dev) 184 { 185 struct apple_aic_softc *sc; 186 struct intr_irqsrc *isrc; 187 const char *name; 188 intptr_t xref; 189 int error, rid; 190 u_int i, cpu, j, info; 191 192 sc = device_get_softc(dev); 193 sc->sc_dev = dev; 194 195 rid = 0; 196 sc->sc_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 197 RF_ACTIVE); 198 if (sc->sc_mem == NULL) { 199 device_printf(dev, "Unable to allocate memory\n"); 200 return (ENXIO); 201 } 202 203 info = bus_read_4(sc->sc_mem, AIC_INFO); 204 sc->sc_nirqs = AIC_INFO_NIRQS(info); 205 sc->sc_ndie = AIC_INFO_NDIE(info) + 1; 206 if (bootverbose) 207 device_printf(dev, "Found %d interrupts, %d die\n", 208 sc->sc_nirqs, sc->sc_ndie); 209 210 for (i = 0; i < sc->sc_ndie; i++) { 211 sc->sc_isrcs[i] = mallocarray(sc->sc_nirqs, 212 sizeof(**sc->sc_isrcs), M_DEVBUF, M_WAITOK | M_ZERO); 213 } 214 215 #ifdef SMP 216 sc->sc_ipimasks = malloc(sizeof(*sc->sc_ipimasks) * mp_maxid + 1, 217 M_DEVBUF, M_WAITOK | M_ZERO); 218 sc->sc_cpuids = malloc(sizeof(*sc->sc_cpuids) * mp_maxid + 1, 219 M_DEVBUF, M_WAITOK | M_ZERO); 220 221 cpu = PCPU_GET(cpuid); 222 sc->sc_cpuids[cpu] = bus_read_4(sc->sc_mem, AIC_WHOAMI); 223 if (bootverbose) 224 device_printf(dev, "BSP CPU %d: whoami %x\n", cpu, 225 sc->sc_cpuids[cpu]); 226 #endif 227 228 229 name = device_get_nameunit(dev); 230 for (i = 0; i < sc->sc_ndie; i++) { 231 struct apple_aic_irqsrc *die_isrcs; 232 233 die_isrcs = sc->sc_isrcs[i]; 234 for (j = 0; j < sc->sc_nirqs; j++) { 235 isrc = &die_isrcs[j].ai_isrc; 236 die_isrcs[j].ai_pol = INTR_POLARITY_CONFORM; 237 die_isrcs[j].ai_trig = INTR_TRIGGER_CONFORM; 238 die_isrcs[j].ai_type = AIC_TYPE_INVAL; 239 die_isrcs[j].ai_irq = j; 240 241 error = intr_isrc_register(isrc, dev, 0, "%s,d%us%u", name, 242 i, j); 243 if (error != 0) { 244 device_printf(dev, "Unable to register irq %u:%u\n", 245 i, j); 246 return (error); 247 } 248 } 249 } 250 251 xref = OF_xref_from_node(ofw_bus_get_node(dev)); 252 if (intr_pic_register(dev, xref) == NULL) { 253 device_printf(dev, "Unable to register interrupt handler\n"); 254 return (ENXIO); 255 } 256 257 if (intr_pic_claim_root(dev, xref, apple_aic_irq, sc, 258 INTR_ROOT_IRQ) != 0) { 259 device_printf(dev, 260 "Unable to set root interrupt controller\n"); 261 intr_pic_deregister(dev, xref); 262 return (ENXIO); 263 } 264 265 if (intr_pic_claim_root(dev, xref, apple_aic_fiq, sc, 266 INTR_ROOT_FIQ) != 0) { 267 device_printf(dev, 268 "Unable to set root fiq controller\n"); 269 intr_pic_deregister(dev, xref); 270 return (ENXIO); 271 } 272 273 #ifdef SMP 274 if (intr_ipi_pic_register(dev, 0) != 0) { 275 device_printf(dev, "could not register for IPIs\n"); 276 return (ENXIO); 277 } 278 #endif 279 280 OF_device_register_xref(xref, dev); 281 282 return (0); 283 } 284 285 static int 286 apple_aic_map_intr_fdt(struct apple_aic_softc *sc, 287 struct intr_map_data_fdt *data, u_int *irq, enum apple_aic_irq_type *typep, 288 enum intr_polarity *polp, enum intr_trigger *trigp, u_int *die) 289 { 290 if (data->ncells != 3) 291 return (EINVAL); 292 293 /* XXX AIC2 */ 294 *die = 0; 295 296 /* 297 * The first cell is the interrupt type: 298 * 0 = IRQ 299 * 1 = FIQ 300 * The second cell is the interrupt number 301 * The third cell is the flags 302 */ 303 switch(data->cells[0]) { 304 case 0: 305 if (typep != NULL) 306 *typep = AIC_TYPE_IRQ; 307 break; 308 case 1: 309 if (typep != NULL) 310 *typep = AIC_TYPE_FIQ; 311 break; 312 default: 313 return (EINVAL); 314 } 315 316 *irq = data->cells[1]; 317 if (*irq > sc->sc_nirqs) 318 return (EINVAL); 319 320 if (trigp != NULL) { 321 if ((data->cells[2] & FDT_INTR_EDGE_MASK) != 0) 322 *trigp = INTR_TRIGGER_EDGE; 323 else 324 *trigp = INTR_TRIGGER_LEVEL; 325 } 326 if (polp != NULL) { 327 if ((data->cells[2] & FDT_INTR_LEVEL_HIGH) != 0) 328 *polp = INTR_POLARITY_HIGH; 329 else 330 *polp = INTR_POLARITY_LOW; 331 } 332 333 return (0); 334 } 335 336 static int 337 apple_aic_map_intr(device_t dev, struct intr_map_data *data, 338 struct intr_irqsrc **isrcp) 339 { 340 struct apple_aic_softc *sc; 341 int error; 342 u_int irq; 343 u_int die; 344 345 sc = device_get_softc(dev); 346 347 error = 0; 348 switch(data->type) { 349 case INTR_MAP_DATA_FDT: 350 error = apple_aic_map_intr_fdt(sc, 351 (struct intr_map_data_fdt *)data, &irq, NULL, NULL, NULL, 352 &die); 353 if (error == 0) 354 *isrcp = &sc->sc_isrcs[0 /* XXX */][irq].ai_isrc; 355 break; 356 default: 357 return (ENOTSUP); 358 } 359 360 return (error); 361 } 362 363 static int 364 apple_aic_setup_intr(device_t dev, struct intr_irqsrc *isrc, 365 struct resource *res, struct intr_map_data *data) 366 { 367 struct apple_aic_softc *sc; 368 enum apple_aic_irq_type type; 369 struct apple_aic_irqsrc *ai; 370 enum intr_trigger trig; 371 enum intr_polarity pol; 372 int error; 373 u_int die, irq; 374 375 sc = device_get_softc(dev); 376 ai = (struct apple_aic_irqsrc *)isrc; 377 378 if (data != NULL) { 379 KASSERT(data->type == INTR_MAP_DATA_FDT, 380 ("%s: Only FDT data is supported (got %#x)", __func__, 381 data->type)); 382 error = apple_aic_map_intr_fdt(sc, 383 (struct intr_map_data_fdt *)data, &irq, &type, &pol, &trig, 384 &die); 385 if (error != 0) 386 return (error); 387 } else { 388 pol = INTR_POLARITY_CONFORM; 389 trig = INTR_TRIGGER_CONFORM; 390 } 391 392 if (isrc->isrc_handlers != 0) { 393 /* TODO */ 394 return (0); 395 } 396 397 if (pol == INTR_POLARITY_CONFORM) 398 pol = INTR_POLARITY_LOW; 399 if (trig == INTR_TRIGGER_CONFORM) 400 trig = INTR_TRIGGER_EDGE; 401 402 ai->ai_pol = pol; 403 ai->ai_trig = trig; 404 ai->ai_type = type; 405 406 /* 407 * Only the timer uses FIQs. These could be sent to any CPU. 408 */ 409 switch (type) { 410 case AIC_TYPE_IRQ: 411 /* XXX die sensitive? */ 412 aic_next_cpu = intr_irq_next_cpu(aic_next_cpu, &all_cpus); 413 bus_write_4(sc->sc_mem, AIC_TARGET_CPU(irq), 414 1 << sc->sc_cpuids[aic_next_cpu]); 415 break; 416 case AIC_TYPE_FIQ: 417 isrc->isrc_flags |= INTR_ISRCF_PPI; 418 break; 419 default: 420 return (EINVAL); 421 } 422 423 return (0); 424 } 425 426 static int 427 apple_aic_teardown_intr(device_t dev, struct intr_irqsrc *isrc, 428 struct resource *res, struct intr_map_data *data) 429 { 430 panic("%s\n", __func__); 431 } 432 433 static void 434 apple_aic_enable_intr(device_t dev, struct intr_irqsrc *isrc) 435 { 436 struct apple_aic_irqsrc *ai; 437 struct apple_aic_softc *sc; 438 u_int irq; 439 440 ai = (struct apple_aic_irqsrc *)isrc; 441 irq = ai->ai_irq; 442 switch(ai->ai_type) { 443 case AIC_TYPE_IRQ: 444 sc = device_get_softc(dev); 445 bus_write_4(sc->sc_mem, AIC_MASK_CLEAR(irq), AIC_IRQ_MASK(irq)); 446 break; 447 case AIC_TYPE_IPI: 448 /* Nothing needed here. */ 449 break; 450 case AIC_TYPE_FIQ: 451 /* TODO */ 452 break; 453 default: 454 panic("%s: %x\n", __func__, ai->ai_type); 455 } 456 } 457 458 static void 459 apple_aic_disable_intr(device_t dev, struct intr_irqsrc *isrc) 460 { 461 struct apple_aic_irqsrc *ai; 462 struct apple_aic_softc *sc; 463 u_int irq; 464 465 ai = (struct apple_aic_irqsrc *)isrc; 466 irq = ai->ai_irq; 467 switch(ai->ai_type) { 468 case AIC_TYPE_IRQ: 469 sc = device_get_softc(dev); 470 bus_write_4(sc->sc_mem, AIC_MASK_SET(irq), AIC_IRQ_MASK(irq)); 471 break; 472 case AIC_TYPE_IPI: 473 /* Nothing needed here. */ 474 break; 475 case AIC_TYPE_FIQ: 476 /* TODO */ 477 break; 478 default: 479 panic("%s: %x\n", __func__, ai->ai_type); 480 } 481 } 482 483 static void 484 apple_aic_post_filter(device_t dev, struct intr_irqsrc *isrc) 485 { 486 struct apple_aic_softc *sc; 487 struct apple_aic_irqsrc *ai; 488 int irq; 489 490 ai = (struct apple_aic_irqsrc *)isrc; 491 irq = ai->ai_irq; 492 switch(ai->ai_type) { 493 case AIC_TYPE_IRQ: 494 sc = device_get_softc(dev); 495 bus_write_4(sc->sc_mem, AIC_SW_CLEAR(irq), AIC_IRQ_MASK(irq)); 496 bus_write_4(sc->sc_mem, AIC_MASK_CLEAR(irq), AIC_IRQ_MASK(irq)); 497 break; 498 case AIC_TYPE_FIQ: 499 /* TODO */ 500 break; 501 default: 502 panic("%s: %x\n", __func__, ai->ai_type); 503 } 504 } 505 506 static void 507 apple_aic_pre_ithread(device_t dev, struct intr_irqsrc *isrc) 508 { 509 struct apple_aic_softc *sc; 510 struct apple_aic_irqsrc *ai; 511 int irq; 512 513 ai = (struct apple_aic_irqsrc *)isrc; 514 sc = device_get_softc(dev); 515 irq = ai->ai_irq; 516 bus_write_4(sc->sc_mem, AIC_SW_CLEAR(irq), AIC_IRQ_MASK(irq)); 517 apple_aic_disable_intr(dev, isrc); 518 /* ACK IT */ 519 } 520 521 static void 522 apple_aic_post_ithread(device_t dev, struct intr_irqsrc *isrc) 523 { 524 struct apple_aic_softc *sc; 525 struct apple_aic_irqsrc *ai; 526 int irq; 527 528 ai = (struct apple_aic_irqsrc *)isrc; 529 sc = device_get_softc(dev); 530 irq = ai->ai_irq; 531 532 bus_write_4(sc->sc_mem, AIC_MASK_CLEAR(irq), AIC_IRQ_MASK(irq)); 533 apple_aic_enable_intr(dev, isrc); 534 } 535 536 #ifdef SMP 537 static void 538 apple_aic_ipi_received(struct apple_aic_softc *sc, struct trapframe *tf) 539 { 540 uint32_t mask; 541 uint32_t ipi; 542 int cpu; 543 544 cpu = PCPU_GET(cpuid); 545 546 mask = atomic_readandclear_32(&sc->sc_ipimasks[cpu]); 547 548 while (mask != 0) { 549 ipi = ffs(mask) - 1; 550 mask &= ~(1 << ipi); 551 552 intr_ipi_dispatch(ipi); 553 } 554 } 555 #endif 556 557 static int 558 apple_aic_irq(void *arg) 559 { 560 struct apple_aic_softc *sc; 561 uint32_t die, event, irq, type; 562 struct apple_aic_irqsrc *aisrc; 563 struct trapframe *tf; 564 565 sc = arg; 566 tf = curthread->td_intr_frame; 567 568 event = bus_read_4(sc->sc_mem, AIC_EVENT); 569 type = AIC_EVENT_TYPE(event); 570 571 /* If we get an IPI here, we really goofed. */ 572 MPASS(type != AIC_EVENT_TYPE_IPI); 573 574 if (type != AIC_EVENT_TYPE_IRQ) { 575 if (type != AIC_EVENT_TYPE_NONE) 576 device_printf(sc->sc_dev, "unexpected event type %d\n", 577 type); 578 return (FILTER_STRAY); 579 } 580 581 die = AIC_EVENT_DIE(event); 582 irq = AIC_EVENT_IRQ(event); 583 584 if (die >= sc->sc_ndie) 585 panic("%s: unexpected die %d", __func__, die); 586 if (irq >= sc->sc_nirqs) 587 panic("%s: unexpected irq %d", __func__, irq); 588 589 aisrc = &sc->sc_isrcs[die][irq]; 590 if (intr_isrc_dispatch(&aisrc->ai_isrc, tf) != 0) { 591 device_printf(sc->sc_dev, "Stray irq %u:%u disabled\n", 592 die, irq); 593 return (FILTER_STRAY); 594 } 595 596 return (FILTER_HANDLED); 597 } 598 599 static int 600 apple_aic_fiq(void *arg) 601 { 602 struct apple_aic_softc *sc; 603 struct apple_aic_irqsrc *isrcs; 604 struct trapframe *tf; 605 606 sc = arg; 607 tf = curthread->td_intr_frame; 608 609 #ifdef SMP 610 /* Handle IPIs. */ 611 if ((READ_SPECIALREG(AIC_IPI_SR_EL1) & AIC_IPI_SR_EL1_PENDING) != 0) { 612 WRITE_SPECIALREG(AIC_IPI_SR_EL1, AIC_IPI_SR_EL1_PENDING); 613 apple_aic_ipi_received(sc, tf); 614 } 615 #endif 616 617 /* 618 * FIQs don't store any state in the interrupt controller at all outside 619 * of IPI handling, so we have to probe around outside of AIC to 620 * determine if we might have been fired off due to a timer. 621 */ 622 isrcs = sc->sc_isrcs[0]; 623 if ((READ_SPECIALREG(cntv_ctl_el0) & CNTV_CTL_BITS) == 624 (CNTV_CTL_ENABLE | CNTV_CTL_ISTATUS)) { 625 intr_isrc_dispatch(&isrcs[AIC_TMR_GUEST_VIRT].ai_isrc, tf); 626 } 627 628 if (has_hyp()) { 629 uint64_t reg; 630 631 if ((READ_SPECIALREG(cntp_ctl_el0) & CNTV_CTL_ISTATUS) != 0) { 632 intr_isrc_dispatch(&isrcs[AIC_TMR_GUEST_PHYS].ai_isrc, 633 tf); 634 } 635 636 reg = READ_SPECIALREG(AIC_FIQ_VM_TIMER); 637 if ((reg & AIC_FIQ_VM_TIMER_PEN) != 0) { 638 intr_isrc_dispatch(&isrcs[AIC_TMR_HV_PHYS].ai_isrc, tf); 639 } 640 641 if ((reg & AIC_FIQ_VM_TIMER_VEN) != 0) { 642 intr_isrc_dispatch(&isrcs[AIC_TMR_HV_VIRT].ai_isrc, tf); 643 } 644 } 645 646 return (FILTER_HANDLED); 647 } 648 649 #ifdef SMP 650 static int 651 apple_aic_bind_intr(device_t dev, struct intr_irqsrc *isrc) 652 { 653 struct apple_aic_softc *sc = device_get_softc(dev); 654 static int aic_next_cpu; 655 uint32_t targets = 0; 656 u_int irq, cpu; 657 658 MPASS(((struct apple_aic_irqsrc *)isrc)->ai_type == AIC_TYPE_IRQ); 659 irq = ((struct apple_aic_irqsrc *)isrc)->ai_irq; 660 if (CPU_EMPTY(&isrc->isrc_cpu)) { 661 aic_next_cpu = intr_irq_next_cpu(aic_next_cpu, &all_cpus); 662 CPU_SETOF(aic_next_cpu, &isrc->isrc_cpu); 663 bus_write_4(sc->sc_mem, AIC_TARGET_CPU(irq), 664 sc->sc_cpuids[aic_next_cpu] << 1); 665 } else { 666 CPU_FOREACH_ISSET(cpu, &isrc->isrc_cpu) { 667 targets |= sc->sc_cpuids[cpu] << 1; 668 } 669 bus_write_4(sc->sc_mem, AIC_TARGET_CPU(irq), targets); 670 } 671 return (0); 672 } 673 674 static void 675 apple_aic_ipi_send(device_t dev, struct intr_irqsrc *isrc, cpuset_t cpus, 676 u_int ipi) 677 { 678 struct apple_aic_softc *sc; 679 uint64_t aff, localgrp, sendmask; 680 u_int cpu; 681 682 sc = device_get_softc(dev); 683 sendmask = 0; 684 localgrp = CPU_AFF1(CPU_AFFINITY(PCPU_GET(cpuid))); 685 686 KASSERT(isrc == &sc->sc_ipi_srcs[ipi].ai_isrc, 687 ("%s: bad ISRC %p argument", __func__, isrc)); 688 for (cpu = 0; cpu <= mp_maxid; cpu++) { 689 if (CPU_ISSET(cpu, &cpus)) { 690 aff = CPU_AFFINITY(cpu); 691 sendmask = CPU_AFF0(aff); 692 atomic_set_32(&sc->sc_ipimasks[cpu], 1 << ipi); 693 694 /* 695 * The above write to sc_ipimasks needs to be visible 696 * before we write to the ipi register to avoid the 697 * targetted CPU missing the dispatch in 698 * apple_aic_ipi_received(). Note that WRITE_SPECIALREG 699 * isn't a memory operation, so we can't relax this to a 700 * a dmb. 701 */ 702 dsb(ishst); 703 704 if (CPU_AFF1(aff) == localgrp) { 705 WRITE_SPECIALREG(AIC_IPI_LOCAL_RR_EL1, 706 sendmask); 707 } else { 708 sendmask |= CPU_AFF1(aff) << 16; 709 WRITE_SPECIALREG(AIC_IPI_GLOBAL_RR_EL1, 710 sendmask); 711 } 712 713 isb(); 714 } 715 } 716 } 717 718 static int 719 apple_aic_ipi_setup(device_t dev, u_int ipi, struct intr_irqsrc **isrcp) 720 { 721 struct apple_aic_softc *sc = device_get_softc(dev); 722 struct apple_aic_irqsrc *ai; 723 724 KASSERT(ipi < AIC_NIPIS, ("%s: ipi %u too high", __func__, ipi)); 725 726 ai = &sc->sc_ipi_srcs[ipi]; 727 ai->ai_type = AIC_TYPE_IPI; 728 729 *isrcp = &ai->ai_isrc; 730 return (0); 731 } 732 733 static void 734 apple_aic_init_secondary(device_t dev, uint32_t rootnum) 735 { 736 struct apple_aic_softc *sc = device_get_softc(dev); 737 u_int cpu = PCPU_GET(cpuid); 738 739 /* We don't need to re-initialize for the FIQ root. */ 740 if (rootnum != INTR_ROOT_IRQ) 741 return; 742 743 sc->sc_cpuids[cpu] = bus_read_4(sc->sc_mem, AIC_WHOAMI); 744 if (bootverbose) 745 device_printf(dev, "CPU %d: whoami %x\n", cpu, 746 sc->sc_cpuids[cpu]); 747 748 bus_write_4(sc->sc_mem, AIC_IPI_MASK_SET, AIC_IPI_SELF | AIC_IPI_OTHER); 749 } 750 #endif 751 752 static device_method_t apple_aic_methods[] = { 753 /* Device interface */ 754 DEVMETHOD(device_probe, apple_aic_probe), 755 DEVMETHOD(device_attach, apple_aic_attach), 756 757 /* Interrupt controller interface */ 758 DEVMETHOD(pic_disable_intr, apple_aic_disable_intr), 759 DEVMETHOD(pic_enable_intr, apple_aic_enable_intr), 760 DEVMETHOD(pic_map_intr, apple_aic_map_intr), 761 DEVMETHOD(pic_setup_intr, apple_aic_setup_intr), 762 DEVMETHOD(pic_teardown_intr, apple_aic_teardown_intr), 763 DEVMETHOD(pic_post_filter, apple_aic_post_filter), 764 DEVMETHOD(pic_post_ithread, apple_aic_post_ithread), 765 DEVMETHOD(pic_pre_ithread, apple_aic_pre_ithread), 766 #ifdef SMP 767 DEVMETHOD(pic_bind_intr, apple_aic_bind_intr), 768 DEVMETHOD(pic_init_secondary, apple_aic_init_secondary), 769 DEVMETHOD(pic_ipi_send, apple_aic_ipi_send), 770 DEVMETHOD(pic_ipi_setup, apple_aic_ipi_setup), 771 #endif 772 773 /* End */ 774 DEVMETHOD_END 775 }; 776 777 static DEFINE_CLASS_0(aic, apple_aic_driver, apple_aic_methods, 778 sizeof(struct apple_aic_softc)); 779 780 EARLY_DRIVER_MODULE(aic, simplebus, apple_aic_driver, 0, 0, 781 BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); 782