1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2021 Andrew Turner
5 * Copyright (c) 2022 Michael J. Karels <karels@freebsd.org>
6 * Copyright (c) 2022 Kyle Evans <kevans@FreeBSD.org>
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 #include "opt_platform.h"
31
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/bus.h>
35 #include <sys/kernel.h>
36 #include <sys/module.h>
37 #include <sys/proc.h>
38 #include <sys/rman.h>
39 #include <sys/smp.h>
40
41 #include <machine/bus.h>
42 #include <machine/machdep.h>
43 #ifdef SMP
44 #include <machine/intr.h>
45 #include <machine/smp.h>
46 #endif
47
48 #include <dev/fdt/fdt_intr.h>
49
50 #include <dev/ofw/openfirm.h>
51 #include <dev/ofw/ofw_bus.h>
52 #include <dev/ofw/ofw_bus_subr.h>
53
54 #include <dt-bindings/interrupt-controller/apple-aic.h>
55
56 #include "pic_if.h"
57
58 #define AIC_INFO 0x0004
59 #define AIC_INFO_NDIE(val) (((val) >> 24) & 0xf)
60 #define AIC_INFO_NIRQS(val) ((val) & 0x0000ffff)
61
62 #define AIC_WHOAMI 0x2000
63 #define AIC_EVENT 0x2004
64 #define AIC_EVENT_DIE(val) (((val) >> 24) & 0xff)
65 #define AIC_EVENT_TYPE(val) (((val) >> 16) & 0xff)
66 #define AIC_EVENT_TYPE_NONE 0
67 #define AIC_EVENT_TYPE_IRQ 1
68 #define AIC_EVENT_TYPE_IPI 4
69 #define AIC_EVENT_IRQ(val) ((val) & 0xffff)
70 #define AIC_EVENT_IPI_OTHER 1
71 #define AIC_EVENT_IPI_SELF 2
72 #define AIC_IPI_SEND 0x2008
73 #define AIC_IPI_ACK 0x200c
74 #define AIC_IPI_MASK_SET 0x2024
75 #define AIC_IPI_MASK_CLR 0x2028
76 #define AIC_IPI_OTHER 0x00000001
77 #define AIC_IPI_SELF 0x80000000
78 #define AIC_TARGET_CPU(irq) (0x3000 + (irq) * 4)
79 #define AIC_SW_SET(irq) (0x4000 + (((irq) >> 5) * 4))
80 #define AIC_SW_CLEAR(irq) (0x4080 + (((irq) >> 5) * 4))
81 #define AIC_MASK_SET(irq) (0x4100 + (((irq) >> 5) * 4))
82 #define AIC_MASK_CLEAR(irq) (0x4180 + (((irq) >> 5) * 4))
83 #define AIC_IRQ_MASK(irq) (1u << ((irq) & 0x1f))
84
85 #define AIC_IPI_LOCAL_RR_EL1 s3_5_c15_c0_0
86 #define AIC_IPI_GLOBAL_RR_EL1 s3_5_c15_c0_1
87
88 #define AIC_IPI_SR_EL1 s3_5_c15_c1_1
89 #define AIC_IPI_SR_EL1_PENDING (1 << 0)
90
91 #define AIC_FIQ_VM_TIMER s3_5_c15_c1_3
92 #define AIC_FIQ_VM_TIMER_VEN (1 << 0)
93 #define AIC_FIQ_VM_TIMER_PEN (1 << 1)
94 #define AIC_FIQ_VM_TIMER_BITS (AIC_FIQ_VM_TIMER_VEN | AIC_FIQ_VM_TIMER_PEN)
95
96 #define CNTV_CTL_ENABLE (1 << 0)
97 #define CNTV_CTL_IMASK (1 << 1)
98 #define CNTV_CTL_ISTATUS (1 << 2)
99 #define CNTV_CTL_BITS \
100 (CNTV_CTL_ENABLE | CNTV_CTL_IMASK | CNTV_CTL_ISTATUS)
101
102 #define AIC_MAXCPUS 32
103 #define AIC_MAXDIES 4
104
105 static struct ofw_compat_data compat_data[] = {
106 { "apple,aic", 1 },
107 { NULL, 0 }
108 };
109
110 enum apple_aic_irq_type {
111 AIC_TYPE_INVAL,
112 AIC_TYPE_IRQ,
113 AIC_TYPE_FIQ,
114 AIC_TYPE_IPI,
115 };
116
117 struct apple_aic_irqsrc {
118 struct intr_irqsrc ai_isrc;
119 enum apple_aic_irq_type ai_type;
120 struct {
121 /* AIC_TYPE_IRQ */
122 enum intr_polarity ai_pol;
123 enum intr_trigger ai_trig;
124 u_int ai_irq;
125 };
126 };
127
128 #ifdef SMP
129 #define AIC_NIPIS INTR_IPI_COUNT
130 #endif
131
132 struct apple_aic_softc {
133 device_t sc_dev;
134 struct resource *sc_mem;
135 struct apple_aic_irqsrc *sc_isrcs[AIC_MAXDIES];
136 u_int sc_nirqs;
137 u_int sc_ndie;
138 #ifdef SMP
139 struct apple_aic_irqsrc sc_ipi_srcs[AIC_NIPIS];
140 u_int *sc_cpuids; /* cpu index to AIC CPU ID */
141 uint32_t *sc_ipimasks;
142 #endif
143 };
144
145 static u_int aic_next_cpu;
146
147 static device_probe_t apple_aic_probe;
148 static device_attach_t apple_aic_attach;
149
150 static pic_disable_intr_t apple_aic_disable_intr;
151 static pic_enable_intr_t apple_aic_enable_intr;
152 static pic_map_intr_t apple_aic_map_intr;
153 static pic_setup_intr_t apple_aic_setup_intr;
154 static pic_teardown_intr_t apple_aic_teardown_intr;
155 static pic_post_filter_t apple_aic_post_filter;
156 static pic_post_ithread_t apple_aic_post_ithread;
157 static pic_pre_ithread_t apple_aic_pre_ithread;
158 #ifdef SMP
159 static pic_bind_intr_t apple_aic_bind_intr;
160 static pic_init_secondary_t apple_aic_init_secondary;
161 static pic_ipi_send_t apple_aic_ipi_send;
162 static pic_ipi_setup_t apple_aic_ipi_setup;
163 #endif
164
165 static int apple_aic_irq(void *);
166 static int apple_aic_fiq(void *);
167
168 static int
apple_aic_probe(device_t dev)169 apple_aic_probe(device_t dev)
170 {
171
172 if (!ofw_bus_status_okay(dev))
173 return (ENXIO);
174
175 if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0)
176 return (ENXIO);
177
178 device_set_desc(dev, "Apple Interrupt Controller");
179 return (BUS_PROBE_DEFAULT);
180 }
181
182 static int
apple_aic_attach(device_t dev)183 apple_aic_attach(device_t dev)
184 {
185 struct apple_aic_softc *sc;
186 struct intr_irqsrc *isrc;
187 const char *name;
188 intptr_t xref;
189 int error, rid;
190 u_int i, cpu, j, info;
191
192 sc = device_get_softc(dev);
193 sc->sc_dev = dev;
194
195 rid = 0;
196 sc->sc_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
197 RF_ACTIVE);
198 if (sc->sc_mem == NULL) {
199 device_printf(dev, "Unable to allocate memory\n");
200 return (ENXIO);
201 }
202
203 info = bus_read_4(sc->sc_mem, AIC_INFO);
204 sc->sc_nirqs = AIC_INFO_NIRQS(info);
205 sc->sc_ndie = AIC_INFO_NDIE(info) + 1;
206 if (bootverbose)
207 device_printf(dev, "Found %d interrupts, %d die\n",
208 sc->sc_nirqs, sc->sc_ndie);
209
210 for (i = 0; i < sc->sc_ndie; i++) {
211 sc->sc_isrcs[i] = mallocarray(sc->sc_nirqs,
212 sizeof(**sc->sc_isrcs), M_DEVBUF, M_WAITOK | M_ZERO);
213 }
214
215 #ifdef SMP
216 sc->sc_ipimasks = malloc(sizeof(*sc->sc_ipimasks) * mp_maxid + 1,
217 M_DEVBUF, M_WAITOK | M_ZERO);
218 sc->sc_cpuids = malloc(sizeof(*sc->sc_cpuids) * mp_maxid + 1,
219 M_DEVBUF, M_WAITOK | M_ZERO);
220
221 cpu = PCPU_GET(cpuid);
222 sc->sc_cpuids[cpu] = bus_read_4(sc->sc_mem, AIC_WHOAMI);
223 if (bootverbose)
224 device_printf(dev, "BSP CPU %d: whoami %x\n", cpu,
225 sc->sc_cpuids[cpu]);
226 #endif
227
228
229 name = device_get_nameunit(dev);
230 for (i = 0; i < sc->sc_ndie; i++) {
231 struct apple_aic_irqsrc *die_isrcs;
232
233 die_isrcs = sc->sc_isrcs[i];
234 for (j = 0; j < sc->sc_nirqs; j++) {
235 isrc = &die_isrcs[j].ai_isrc;
236 die_isrcs[j].ai_pol = INTR_POLARITY_CONFORM;
237 die_isrcs[j].ai_trig = INTR_TRIGGER_CONFORM;
238 die_isrcs[j].ai_type = AIC_TYPE_INVAL;
239 die_isrcs[j].ai_irq = j;
240
241 error = intr_isrc_register(isrc, dev, 0, "%s,d%us%u", name,
242 i, j);
243 if (error != 0) {
244 device_printf(dev, "Unable to register irq %u:%u\n",
245 i, j);
246 return (error);
247 }
248 }
249 }
250
251 xref = OF_xref_from_node(ofw_bus_get_node(dev));
252 if (intr_pic_register(dev, xref) == NULL) {
253 device_printf(dev, "Unable to register interrupt handler\n");
254 return (ENXIO);
255 }
256
257 if (intr_pic_claim_root(dev, xref, apple_aic_irq, sc,
258 INTR_ROOT_IRQ) != 0) {
259 device_printf(dev,
260 "Unable to set root interrupt controller\n");
261 intr_pic_deregister(dev, xref);
262 return (ENXIO);
263 }
264
265 if (intr_pic_claim_root(dev, xref, apple_aic_fiq, sc,
266 INTR_ROOT_FIQ) != 0) {
267 device_printf(dev,
268 "Unable to set root fiq controller\n");
269 intr_pic_deregister(dev, xref);
270 return (ENXIO);
271 }
272
273 #ifdef SMP
274 if (intr_ipi_pic_register(dev, 0) != 0) {
275 device_printf(dev, "could not register for IPIs\n");
276 return (ENXIO);
277 }
278 #endif
279
280 OF_device_register_xref(xref, dev);
281
282 return (0);
283 }
284
285 static int
apple_aic_map_intr_fdt(struct apple_aic_softc * sc,struct intr_map_data_fdt * data,u_int * irq,enum apple_aic_irq_type * typep,enum intr_polarity * polp,enum intr_trigger * trigp,u_int * die)286 apple_aic_map_intr_fdt(struct apple_aic_softc *sc,
287 struct intr_map_data_fdt *data, u_int *irq, enum apple_aic_irq_type *typep,
288 enum intr_polarity *polp, enum intr_trigger *trigp, u_int *die)
289 {
290 if (data->ncells != 3)
291 return (EINVAL);
292
293 /* XXX AIC2 */
294 *die = 0;
295
296 /*
297 * The first cell is the interrupt type:
298 * 0 = IRQ
299 * 1 = FIQ
300 * The second cell is the interrupt number
301 * The third cell is the flags
302 */
303 switch(data->cells[0]) {
304 case 0:
305 if (typep != NULL)
306 *typep = AIC_TYPE_IRQ;
307 break;
308 case 1:
309 if (typep != NULL)
310 *typep = AIC_TYPE_FIQ;
311 break;
312 default:
313 return (EINVAL);
314 }
315
316 *irq = data->cells[1];
317 if (*irq > sc->sc_nirqs)
318 return (EINVAL);
319
320 if (trigp != NULL) {
321 if ((data->cells[2] & FDT_INTR_EDGE_MASK) != 0)
322 *trigp = INTR_TRIGGER_EDGE;
323 else
324 *trigp = INTR_TRIGGER_LEVEL;
325 }
326 if (polp != NULL) {
327 if ((data->cells[2] & FDT_INTR_LEVEL_HIGH) != 0)
328 *polp = INTR_POLARITY_HIGH;
329 else
330 *polp = INTR_POLARITY_LOW;
331 }
332
333 return (0);
334 }
335
336 static int
apple_aic_map_intr(device_t dev,struct intr_map_data * data,struct intr_irqsrc ** isrcp)337 apple_aic_map_intr(device_t dev, struct intr_map_data *data,
338 struct intr_irqsrc **isrcp)
339 {
340 struct apple_aic_softc *sc;
341 int error;
342 u_int irq;
343 u_int die;
344
345 sc = device_get_softc(dev);
346
347 error = 0;
348 switch(data->type) {
349 case INTR_MAP_DATA_FDT:
350 error = apple_aic_map_intr_fdt(sc,
351 (struct intr_map_data_fdt *)data, &irq, NULL, NULL, NULL,
352 &die);
353 if (error == 0)
354 *isrcp = &sc->sc_isrcs[0 /* XXX */][irq].ai_isrc;
355 break;
356 default:
357 return (ENOTSUP);
358 }
359
360 return (error);
361 }
362
363 static int
apple_aic_setup_intr(device_t dev,struct intr_irqsrc * isrc,struct resource * res,struct intr_map_data * data)364 apple_aic_setup_intr(device_t dev, struct intr_irqsrc *isrc,
365 struct resource *res, struct intr_map_data *data)
366 {
367 struct apple_aic_softc *sc;
368 enum apple_aic_irq_type type;
369 struct apple_aic_irqsrc *ai;
370 enum intr_trigger trig;
371 enum intr_polarity pol;
372 int error;
373 u_int die, irq;
374
375 sc = device_get_softc(dev);
376 ai = (struct apple_aic_irqsrc *)isrc;
377
378 if (data != NULL) {
379 KASSERT(data->type == INTR_MAP_DATA_FDT,
380 ("%s: Only FDT data is supported (got %#x)", __func__,
381 data->type));
382 error = apple_aic_map_intr_fdt(sc,
383 (struct intr_map_data_fdt *)data, &irq, &type, &pol, &trig,
384 &die);
385 if (error != 0)
386 return (error);
387 } else {
388 pol = INTR_POLARITY_CONFORM;
389 trig = INTR_TRIGGER_CONFORM;
390 }
391
392 if (isrc->isrc_handlers != 0) {
393 /* TODO */
394 return (0);
395 }
396
397 if (pol == INTR_POLARITY_CONFORM)
398 pol = INTR_POLARITY_LOW;
399 if (trig == INTR_TRIGGER_CONFORM)
400 trig = INTR_TRIGGER_EDGE;
401
402 ai->ai_pol = pol;
403 ai->ai_trig = trig;
404 ai->ai_type = type;
405
406 /*
407 * Only the timer uses FIQs. These could be sent to any CPU.
408 */
409 switch (type) {
410 case AIC_TYPE_IRQ:
411 /* XXX die sensitive? */
412 aic_next_cpu = intr_irq_next_cpu(aic_next_cpu, &all_cpus);
413 bus_write_4(sc->sc_mem, AIC_TARGET_CPU(irq),
414 1 << sc->sc_cpuids[aic_next_cpu]);
415 break;
416 case AIC_TYPE_FIQ:
417 isrc->isrc_flags |= INTR_ISRCF_PPI;
418 break;
419 default:
420 return (EINVAL);
421 }
422
423 return (0);
424 }
425
426 static int
apple_aic_teardown_intr(device_t dev,struct intr_irqsrc * isrc,struct resource * res,struct intr_map_data * data)427 apple_aic_teardown_intr(device_t dev, struct intr_irqsrc *isrc,
428 struct resource *res, struct intr_map_data *data)
429 {
430 panic("%s\n", __func__);
431 }
432
433 static void
apple_aic_enable_intr(device_t dev,struct intr_irqsrc * isrc)434 apple_aic_enable_intr(device_t dev, struct intr_irqsrc *isrc)
435 {
436 struct apple_aic_irqsrc *ai;
437 struct apple_aic_softc *sc;
438 u_int irq;
439
440 ai = (struct apple_aic_irqsrc *)isrc;
441 irq = ai->ai_irq;
442 switch(ai->ai_type) {
443 case AIC_TYPE_IRQ:
444 sc = device_get_softc(dev);
445 bus_write_4(sc->sc_mem, AIC_MASK_CLEAR(irq), AIC_IRQ_MASK(irq));
446 break;
447 case AIC_TYPE_IPI:
448 /* Nothing needed here. */
449 break;
450 case AIC_TYPE_FIQ:
451 /* TODO */
452 break;
453 default:
454 panic("%s: %x\n", __func__, ai->ai_type);
455 }
456 }
457
458 static void
apple_aic_disable_intr(device_t dev,struct intr_irqsrc * isrc)459 apple_aic_disable_intr(device_t dev, struct intr_irqsrc *isrc)
460 {
461 struct apple_aic_irqsrc *ai;
462 struct apple_aic_softc *sc;
463 u_int irq;
464
465 ai = (struct apple_aic_irqsrc *)isrc;
466 irq = ai->ai_irq;
467 switch(ai->ai_type) {
468 case AIC_TYPE_IRQ:
469 sc = device_get_softc(dev);
470 bus_write_4(sc->sc_mem, AIC_MASK_SET(irq), AIC_IRQ_MASK(irq));
471 break;
472 case AIC_TYPE_IPI:
473 /* Nothing needed here. */
474 break;
475 case AIC_TYPE_FIQ:
476 /* TODO */
477 break;
478 default:
479 panic("%s: %x\n", __func__, ai->ai_type);
480 }
481 }
482
483 static void
apple_aic_post_filter(device_t dev,struct intr_irqsrc * isrc)484 apple_aic_post_filter(device_t dev, struct intr_irqsrc *isrc)
485 {
486 struct apple_aic_softc *sc;
487 struct apple_aic_irqsrc *ai;
488 int irq;
489
490 ai = (struct apple_aic_irqsrc *)isrc;
491 irq = ai->ai_irq;
492 switch(ai->ai_type) {
493 case AIC_TYPE_IRQ:
494 sc = device_get_softc(dev);
495 bus_write_4(sc->sc_mem, AIC_SW_CLEAR(irq), AIC_IRQ_MASK(irq));
496 bus_write_4(sc->sc_mem, AIC_MASK_CLEAR(irq), AIC_IRQ_MASK(irq));
497 break;
498 case AIC_TYPE_FIQ:
499 /* TODO */
500 break;
501 default:
502 panic("%s: %x\n", __func__, ai->ai_type);
503 }
504 }
505
506 static void
apple_aic_pre_ithread(device_t dev,struct intr_irqsrc * isrc)507 apple_aic_pre_ithread(device_t dev, struct intr_irqsrc *isrc)
508 {
509 struct apple_aic_softc *sc;
510 struct apple_aic_irqsrc *ai;
511 int irq;
512
513 ai = (struct apple_aic_irqsrc *)isrc;
514 sc = device_get_softc(dev);
515 irq = ai->ai_irq;
516 bus_write_4(sc->sc_mem, AIC_SW_CLEAR(irq), AIC_IRQ_MASK(irq));
517 apple_aic_disable_intr(dev, isrc);
518 /* ACK IT */
519 }
520
521 static void
apple_aic_post_ithread(device_t dev,struct intr_irqsrc * isrc)522 apple_aic_post_ithread(device_t dev, struct intr_irqsrc *isrc)
523 {
524 struct apple_aic_softc *sc;
525 struct apple_aic_irqsrc *ai;
526 int irq;
527
528 ai = (struct apple_aic_irqsrc *)isrc;
529 sc = device_get_softc(dev);
530 irq = ai->ai_irq;
531
532 bus_write_4(sc->sc_mem, AIC_MASK_CLEAR(irq), AIC_IRQ_MASK(irq));
533 apple_aic_enable_intr(dev, isrc);
534 }
535
536 #ifdef SMP
537 static void
apple_aic_ipi_received(struct apple_aic_softc * sc,struct trapframe * tf)538 apple_aic_ipi_received(struct apple_aic_softc *sc, struct trapframe *tf)
539 {
540 uint32_t mask;
541 uint32_t ipi;
542 int cpu;
543
544 cpu = PCPU_GET(cpuid);
545
546 mask = atomic_readandclear_32(&sc->sc_ipimasks[cpu]);
547
548 while (mask != 0) {
549 ipi = ffs(mask) - 1;
550 mask &= ~(1 << ipi);
551
552 intr_ipi_dispatch(ipi);
553 }
554 }
555 #endif
556
557 static int
apple_aic_irq(void * arg)558 apple_aic_irq(void *arg)
559 {
560 struct apple_aic_softc *sc;
561 uint32_t die, event, irq, type;
562 struct apple_aic_irqsrc *aisrc;
563 struct trapframe *tf;
564
565 sc = arg;
566 tf = curthread->td_intr_frame;
567
568 event = bus_read_4(sc->sc_mem, AIC_EVENT);
569 type = AIC_EVENT_TYPE(event);
570
571 /* If we get an IPI here, we really goofed. */
572 MPASS(type != AIC_EVENT_TYPE_IPI);
573
574 if (type != AIC_EVENT_TYPE_IRQ) {
575 if (type != AIC_EVENT_TYPE_NONE)
576 device_printf(sc->sc_dev, "unexpected event type %d\n",
577 type);
578 return (FILTER_STRAY);
579 }
580
581 die = AIC_EVENT_DIE(event);
582 irq = AIC_EVENT_IRQ(event);
583
584 if (die >= sc->sc_ndie)
585 panic("%s: unexpected die %d", __func__, die);
586 if (irq >= sc->sc_nirqs)
587 panic("%s: unexpected irq %d", __func__, irq);
588
589 aisrc = &sc->sc_isrcs[die][irq];
590 if (intr_isrc_dispatch(&aisrc->ai_isrc, tf) != 0) {
591 device_printf(sc->sc_dev, "Stray irq %u:%u disabled\n",
592 die, irq);
593 return (FILTER_STRAY);
594 }
595
596 return (FILTER_HANDLED);
597 }
598
599 static int
apple_aic_fiq(void * arg)600 apple_aic_fiq(void *arg)
601 {
602 struct apple_aic_softc *sc;
603 struct apple_aic_irqsrc *isrcs;
604 struct trapframe *tf;
605
606 sc = arg;
607 tf = curthread->td_intr_frame;
608
609 #ifdef SMP
610 /* Handle IPIs. */
611 if ((READ_SPECIALREG(AIC_IPI_SR_EL1) & AIC_IPI_SR_EL1_PENDING) != 0) {
612 WRITE_SPECIALREG(AIC_IPI_SR_EL1, AIC_IPI_SR_EL1_PENDING);
613 apple_aic_ipi_received(sc, tf);
614 }
615 #endif
616
617 /*
618 * FIQs don't store any state in the interrupt controller at all outside
619 * of IPI handling, so we have to probe around outside of AIC to
620 * determine if we might have been fired off due to a timer.
621 */
622 isrcs = sc->sc_isrcs[0];
623 if ((READ_SPECIALREG(cntv_ctl_el0) & CNTV_CTL_BITS) ==
624 (CNTV_CTL_ENABLE | CNTV_CTL_ISTATUS)) {
625 intr_isrc_dispatch(&isrcs[AIC_TMR_GUEST_VIRT].ai_isrc, tf);
626 }
627
628 if (has_hyp()) {
629 uint64_t reg;
630
631 if ((READ_SPECIALREG(cntp_ctl_el0) & CNTV_CTL_ISTATUS) != 0) {
632 intr_isrc_dispatch(&isrcs[AIC_TMR_GUEST_PHYS].ai_isrc,
633 tf);
634 }
635
636 reg = READ_SPECIALREG(AIC_FIQ_VM_TIMER);
637 if ((reg & AIC_FIQ_VM_TIMER_PEN) != 0) {
638 intr_isrc_dispatch(&isrcs[AIC_TMR_HV_PHYS].ai_isrc, tf);
639 }
640
641 if ((reg & AIC_FIQ_VM_TIMER_VEN) != 0) {
642 intr_isrc_dispatch(&isrcs[AIC_TMR_HV_VIRT].ai_isrc, tf);
643 }
644 }
645
646 return (FILTER_HANDLED);
647 }
648
649 #ifdef SMP
650 static int
apple_aic_bind_intr(device_t dev,struct intr_irqsrc * isrc)651 apple_aic_bind_intr(device_t dev, struct intr_irqsrc *isrc)
652 {
653 struct apple_aic_softc *sc = device_get_softc(dev);
654 static int aic_next_cpu;
655 uint32_t targets = 0;
656 u_int irq, cpu;
657
658 MPASS(((struct apple_aic_irqsrc *)isrc)->ai_type == AIC_TYPE_IRQ);
659 irq = ((struct apple_aic_irqsrc *)isrc)->ai_irq;
660 if (CPU_EMPTY(&isrc->isrc_cpu)) {
661 aic_next_cpu = intr_irq_next_cpu(aic_next_cpu, &all_cpus);
662 CPU_SETOF(aic_next_cpu, &isrc->isrc_cpu);
663 bus_write_4(sc->sc_mem, AIC_TARGET_CPU(irq),
664 sc->sc_cpuids[aic_next_cpu] << 1);
665 } else {
666 CPU_FOREACH_ISSET(cpu, &isrc->isrc_cpu) {
667 targets |= sc->sc_cpuids[cpu] << 1;
668 }
669 bus_write_4(sc->sc_mem, AIC_TARGET_CPU(irq), targets);
670 }
671 return (0);
672 }
673
674 static void
apple_aic_ipi_send(device_t dev,struct intr_irqsrc * isrc,cpuset_t cpus,u_int ipi)675 apple_aic_ipi_send(device_t dev, struct intr_irqsrc *isrc, cpuset_t cpus,
676 u_int ipi)
677 {
678 struct apple_aic_softc *sc;
679 uint64_t aff, localgrp, sendmask;
680 u_int cpu;
681
682 sc = device_get_softc(dev);
683 sendmask = 0;
684 localgrp = CPU_AFF1(CPU_AFFINITY(PCPU_GET(cpuid)));
685
686 KASSERT(isrc == &sc->sc_ipi_srcs[ipi].ai_isrc,
687 ("%s: bad ISRC %p argument", __func__, isrc));
688 for (cpu = 0; cpu <= mp_maxid; cpu++) {
689 if (CPU_ISSET(cpu, &cpus)) {
690 aff = CPU_AFFINITY(cpu);
691 sendmask = CPU_AFF0(aff);
692 atomic_set_32(&sc->sc_ipimasks[cpu], 1 << ipi);
693
694 /*
695 * The above write to sc_ipimasks needs to be visible
696 * before we write to the ipi register to avoid the
697 * targetted CPU missing the dispatch in
698 * apple_aic_ipi_received(). Note that WRITE_SPECIALREG
699 * isn't a memory operation, so we can't relax this to a
700 * a dmb.
701 */
702 dsb(ishst);
703
704 if (CPU_AFF1(aff) == localgrp) {
705 WRITE_SPECIALREG(AIC_IPI_LOCAL_RR_EL1,
706 sendmask);
707 } else {
708 sendmask |= CPU_AFF1(aff) << 16;
709 WRITE_SPECIALREG(AIC_IPI_GLOBAL_RR_EL1,
710 sendmask);
711 }
712
713 isb();
714 }
715 }
716 }
717
718 static int
apple_aic_ipi_setup(device_t dev,u_int ipi,struct intr_irqsrc ** isrcp)719 apple_aic_ipi_setup(device_t dev, u_int ipi, struct intr_irqsrc **isrcp)
720 {
721 struct apple_aic_softc *sc = device_get_softc(dev);
722 struct apple_aic_irqsrc *ai;
723
724 KASSERT(ipi < AIC_NIPIS, ("%s: ipi %u too high", __func__, ipi));
725
726 ai = &sc->sc_ipi_srcs[ipi];
727 ai->ai_type = AIC_TYPE_IPI;
728
729 *isrcp = &ai->ai_isrc;
730 return (0);
731 }
732
733 static void
apple_aic_init_secondary(device_t dev,uint32_t rootnum)734 apple_aic_init_secondary(device_t dev, uint32_t rootnum)
735 {
736 struct apple_aic_softc *sc = device_get_softc(dev);
737 u_int cpu = PCPU_GET(cpuid);
738
739 /* We don't need to re-initialize for the FIQ root. */
740 if (rootnum != INTR_ROOT_IRQ)
741 return;
742
743 sc->sc_cpuids[cpu] = bus_read_4(sc->sc_mem, AIC_WHOAMI);
744 if (bootverbose)
745 device_printf(dev, "CPU %d: whoami %x\n", cpu,
746 sc->sc_cpuids[cpu]);
747
748 bus_write_4(sc->sc_mem, AIC_IPI_MASK_SET, AIC_IPI_SELF | AIC_IPI_OTHER);
749 }
750 #endif
751
752 static device_method_t apple_aic_methods[] = {
753 /* Device interface */
754 DEVMETHOD(device_probe, apple_aic_probe),
755 DEVMETHOD(device_attach, apple_aic_attach),
756
757 /* Interrupt controller interface */
758 DEVMETHOD(pic_disable_intr, apple_aic_disable_intr),
759 DEVMETHOD(pic_enable_intr, apple_aic_enable_intr),
760 DEVMETHOD(pic_map_intr, apple_aic_map_intr),
761 DEVMETHOD(pic_setup_intr, apple_aic_setup_intr),
762 DEVMETHOD(pic_teardown_intr, apple_aic_teardown_intr),
763 DEVMETHOD(pic_post_filter, apple_aic_post_filter),
764 DEVMETHOD(pic_post_ithread, apple_aic_post_ithread),
765 DEVMETHOD(pic_pre_ithread, apple_aic_pre_ithread),
766 #ifdef SMP
767 DEVMETHOD(pic_bind_intr, apple_aic_bind_intr),
768 DEVMETHOD(pic_init_secondary, apple_aic_init_secondary),
769 DEVMETHOD(pic_ipi_send, apple_aic_ipi_send),
770 DEVMETHOD(pic_ipi_setup, apple_aic_ipi_setup),
771 #endif
772
773 /* End */
774 DEVMETHOD_END
775 };
776
777 static DEFINE_CLASS_0(aic, apple_aic_driver, apple_aic_methods,
778 sizeof(struct apple_aic_softc));
779
780 EARLY_DRIVER_MODULE(aic, simplebus, apple_aic_driver, 0, 0,
781 BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE);
782