1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2021 Andrew Turner
5 * Copyright (c) 2022 Michael J. Karels <karels@freebsd.org>
6 * Copyright (c) 2022 Kyle Evans <kevans@FreeBSD.org>
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 #include "opt_platform.h"
31
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/bus.h>
35 #include <sys/kernel.h>
36 #include <sys/module.h>
37 #include <sys/proc.h>
38 #include <sys/rman.h>
39 #include <sys/smp.h>
40
41 #include <machine/bus.h>
42 #include <machine/machdep.h>
43 #ifdef SMP
44 #include <machine/intr.h>
45 #include <machine/smp.h>
46 #endif
47
48 #include <dev/fdt/fdt_intr.h>
49
50 #include <dev/ofw/openfirm.h>
51 #include <dev/ofw/ofw_bus.h>
52 #include <dev/ofw/ofw_bus_subr.h>
53
54 #include <dt-bindings/interrupt-controller/apple-aic.h>
55
56 #include "pic_if.h"
57
58 #define AIC_INFO 0x0004
59 #define AIC_INFO_NDIE(val) (((val) >> 24) & 0xf)
60 #define AIC_INFO_NIRQS(val) ((val) & 0x0000ffff)
61
62 #define AIC_WHOAMI 0x2000
63 #define AIC_EVENT 0x2004
64 #define AIC_EVENT_DIE(val) (((val) >> 24) & 0xff)
65 #define AIC_EVENT_TYPE(val) (((val) >> 16) & 0xff)
66 #define AIC_EVENT_TYPE_NONE 0
67 #define AIC_EVENT_TYPE_IRQ 1
68 #define AIC_EVENT_TYPE_IPI 4
69 #define AIC_EVENT_IRQ(val) ((val) & 0xffff)
70 #define AIC_EVENT_IPI_OTHER 1
71 #define AIC_EVENT_IPI_SELF 2
72 #define AIC_IPI_SEND 0x2008
73 #define AIC_IPI_ACK 0x200c
74 #define AIC_IPI_MASK_SET 0x2024
75 #define AIC_IPI_MASK_CLR 0x2028
76 #define AIC_IPI_OTHER 0x00000001
77 #define AIC_IPI_SELF 0x80000000
78 #define AIC_TARGET_CPU(irq) (0x3000 + (irq) * 4)
79 #define AIC_SW_SET(irq) (0x4000 + (((irq) >> 5) * 4))
80 #define AIC_SW_CLEAR(irq) (0x4080 + (((irq) >> 5) * 4))
81 #define AIC_MASK_SET(irq) (0x4100 + (((irq) >> 5) * 4))
82 #define AIC_MASK_CLEAR(irq) (0x4180 + (((irq) >> 5) * 4))
83 #define AIC_IRQ_MASK(irq) (1u << ((irq) & 0x1f))
84
85 #define AIC_IPI_LOCAL_RR_EL1 s3_5_c15_c0_0
86 #define AIC_IPI_GLOBAL_RR_EL1 s3_5_c15_c0_1
87
88 #define AIC_IPI_SR_EL1 s3_5_c15_c1_1
89 #define AIC_IPI_SR_EL1_PENDING (1 << 0)
90
91 #define AIC_FIQ_VM_TIMER s3_5_c15_c1_3
92 #define AIC_FIQ_VM_TIMER_VEN (1 << 0)
93 #define AIC_FIQ_VM_TIMER_PEN (1 << 1)
94 #define AIC_FIQ_VM_TIMER_BITS (AIC_FIQ_VM_TIMER_VEN | AIC_FIQ_VM_TIMER_PEN)
95
96 #define CNTV_CTL_ENABLE (1 << 0)
97 #define CNTV_CTL_IMASK (1 << 1)
98 #define CNTV_CTL_ISTATUS (1 << 2)
99 #define CNTV_CTL_BITS \
100 (CNTV_CTL_ENABLE | CNTV_CTL_IMASK | CNTV_CTL_ISTATUS)
101
102 #define AIC_MAXCPUS 32
103 #define AIC_MAXDIES 4
104
105 static struct ofw_compat_data compat_data[] = {
106 { "apple,aic", 1 },
107 { NULL, 0 }
108 };
109
110 enum apple_aic_irq_type {
111 AIC_TYPE_INVAL,
112 AIC_TYPE_IRQ,
113 AIC_TYPE_FIQ,
114 AIC_TYPE_IPI,
115 };
116
117 struct apple_aic_irqsrc {
118 struct intr_irqsrc ai_isrc;
119 enum apple_aic_irq_type ai_type;
120 struct {
121 /* AIC_TYPE_IRQ */
122 enum intr_polarity ai_pol;
123 enum intr_trigger ai_trig;
124 u_int ai_irq;
125 };
126 };
127
128 #ifdef SMP
129 #define AIC_NIPIS INTR_IPI_COUNT
130 #endif
131
132 struct apple_aic_softc {
133 device_t sc_dev;
134 struct resource *sc_mem;
135 struct apple_aic_irqsrc *sc_isrcs[AIC_MAXDIES];
136 u_int sc_nirqs;
137 u_int sc_ndie;
138 #ifdef SMP
139 struct apple_aic_irqsrc sc_ipi_srcs[AIC_NIPIS];
140 uint32_t *sc_ipimasks;
141 #endif
142 u_int *sc_cpuids; /* cpu index to AIC CPU ID */
143 };
144
145 static u_int aic_next_cpu;
146
147 static device_probe_t apple_aic_probe;
148 static device_attach_t apple_aic_attach;
149
150 static pic_disable_intr_t apple_aic_disable_intr;
151 static pic_enable_intr_t apple_aic_enable_intr;
152 static pic_map_intr_t apple_aic_map_intr;
153 static pic_setup_intr_t apple_aic_setup_intr;
154 static pic_teardown_intr_t apple_aic_teardown_intr;
155 static pic_post_filter_t apple_aic_post_filter;
156 static pic_post_ithread_t apple_aic_post_ithread;
157 static pic_pre_ithread_t apple_aic_pre_ithread;
158 #ifdef SMP
159 static pic_bind_intr_t apple_aic_bind_intr;
160 static pic_init_secondary_t apple_aic_init_secondary;
161 static pic_ipi_send_t apple_aic_ipi_send;
162 static pic_ipi_setup_t apple_aic_ipi_setup;
163 #endif
164
165 static int apple_aic_irq(void *);
166 static int apple_aic_fiq(void *);
167
168 static int
apple_aic_probe(device_t dev)169 apple_aic_probe(device_t dev)
170 {
171
172 if (!ofw_bus_status_okay(dev))
173 return (ENXIO);
174
175 if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0)
176 return (ENXIO);
177
178 device_set_desc(dev, "Apple Interrupt Controller");
179 return (BUS_PROBE_DEFAULT);
180 }
181
182 static int
apple_aic_attach(device_t dev)183 apple_aic_attach(device_t dev)
184 {
185 struct apple_aic_softc *sc;
186 struct intr_irqsrc *isrc;
187 const char *name;
188 intptr_t xref;
189 int error, rid;
190 u_int i, cpu, j, info;
191
192 sc = device_get_softc(dev);
193 sc->sc_dev = dev;
194
195 rid = 0;
196 sc->sc_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
197 RF_ACTIVE);
198 if (sc->sc_mem == NULL) {
199 device_printf(dev, "Unable to allocate memory\n");
200 return (ENXIO);
201 }
202
203 info = bus_read_4(sc->sc_mem, AIC_INFO);
204 sc->sc_nirqs = AIC_INFO_NIRQS(info);
205 sc->sc_ndie = AIC_INFO_NDIE(info) + 1;
206 if (bootverbose)
207 device_printf(dev, "Found %d interrupts, %d die\n",
208 sc->sc_nirqs, sc->sc_ndie);
209
210 for (i = 0; i < sc->sc_ndie; i++) {
211 sc->sc_isrcs[i] = mallocarray(sc->sc_nirqs,
212 sizeof(**sc->sc_isrcs), M_DEVBUF, M_WAITOK | M_ZERO);
213 }
214
215 #ifdef SMP
216 sc->sc_ipimasks = malloc(sizeof(*sc->sc_ipimasks) * mp_maxid + 1,
217 M_DEVBUF, M_WAITOK | M_ZERO);
218 #endif
219 sc->sc_cpuids = malloc(sizeof(*sc->sc_cpuids) * mp_maxid + 1,
220 M_DEVBUF, M_WAITOK | M_ZERO);
221
222 cpu = PCPU_GET(cpuid);
223 sc->sc_cpuids[cpu] = bus_read_4(sc->sc_mem, AIC_WHOAMI);
224 if (bootverbose)
225 device_printf(dev, "BSP CPU %d: whoami %x\n", cpu,
226 sc->sc_cpuids[cpu]);
227
228 name = device_get_nameunit(dev);
229 for (i = 0; i < sc->sc_ndie; i++) {
230 struct apple_aic_irqsrc *die_isrcs;
231
232 die_isrcs = sc->sc_isrcs[i];
233 for (j = 0; j < sc->sc_nirqs; j++) {
234 isrc = &die_isrcs[j].ai_isrc;
235 die_isrcs[j].ai_pol = INTR_POLARITY_CONFORM;
236 die_isrcs[j].ai_trig = INTR_TRIGGER_CONFORM;
237 die_isrcs[j].ai_type = AIC_TYPE_INVAL;
238 die_isrcs[j].ai_irq = j;
239
240 error = intr_isrc_register(isrc, dev, 0, "%s,d%us%u", name,
241 i, j);
242 if (error != 0) {
243 device_printf(dev, "Unable to register irq %u:%u\n",
244 i, j);
245 return (error);
246 }
247 }
248 }
249
250 xref = OF_xref_from_node(ofw_bus_get_node(dev));
251 if (intr_pic_register(dev, xref) == NULL) {
252 device_printf(dev, "Unable to register interrupt handler\n");
253 return (ENXIO);
254 }
255
256 if (intr_pic_claim_root(dev, xref, apple_aic_irq, sc,
257 INTR_ROOT_IRQ) != 0) {
258 device_printf(dev,
259 "Unable to set root interrupt controller\n");
260 intr_pic_deregister(dev, xref);
261 return (ENXIO);
262 }
263
264 if (intr_pic_claim_root(dev, xref, apple_aic_fiq, sc,
265 INTR_ROOT_FIQ) != 0) {
266 device_printf(dev,
267 "Unable to set root fiq controller\n");
268 intr_pic_deregister(dev, xref);
269 return (ENXIO);
270 }
271
272 #ifdef SMP
273 if (intr_ipi_pic_register(dev, 0) != 0) {
274 device_printf(dev, "could not register for IPIs\n");
275 return (ENXIO);
276 }
277 #endif
278
279 OF_device_register_xref(xref, dev);
280
281 return (0);
282 }
283
284 static int
apple_aic_map_intr_fdt(struct apple_aic_softc * sc,struct intr_map_data_fdt * data,u_int * irq,enum apple_aic_irq_type * typep,enum intr_polarity * polp,enum intr_trigger * trigp,u_int * die)285 apple_aic_map_intr_fdt(struct apple_aic_softc *sc,
286 struct intr_map_data_fdt *data, u_int *irq, enum apple_aic_irq_type *typep,
287 enum intr_polarity *polp, enum intr_trigger *trigp, u_int *die)
288 {
289 if (data->ncells != 3)
290 return (EINVAL);
291
292 /* XXX AIC2 */
293 *die = 0;
294
295 /*
296 * The first cell is the interrupt type:
297 * 0 = IRQ
298 * 1 = FIQ
299 * The second cell is the interrupt number
300 * The third cell is the flags
301 */
302 switch(data->cells[0]) {
303 case 0:
304 if (typep != NULL)
305 *typep = AIC_TYPE_IRQ;
306 break;
307 case 1:
308 if (typep != NULL)
309 *typep = AIC_TYPE_FIQ;
310 break;
311 default:
312 return (EINVAL);
313 }
314
315 *irq = data->cells[1];
316 if (*irq > sc->sc_nirqs)
317 return (EINVAL);
318
319 if (trigp != NULL) {
320 if ((data->cells[2] & FDT_INTR_EDGE_MASK) != 0)
321 *trigp = INTR_TRIGGER_EDGE;
322 else
323 *trigp = INTR_TRIGGER_LEVEL;
324 }
325 if (polp != NULL) {
326 if ((data->cells[2] & FDT_INTR_LEVEL_HIGH) != 0)
327 *polp = INTR_POLARITY_HIGH;
328 else
329 *polp = INTR_POLARITY_LOW;
330 }
331
332 return (0);
333 }
334
335 static int
apple_aic_map_intr(device_t dev,struct intr_map_data * data,struct intr_irqsrc ** isrcp)336 apple_aic_map_intr(device_t dev, struct intr_map_data *data,
337 struct intr_irqsrc **isrcp)
338 {
339 struct apple_aic_softc *sc;
340 int error;
341 u_int irq;
342 u_int die;
343
344 sc = device_get_softc(dev);
345
346 error = 0;
347 switch(data->type) {
348 case INTR_MAP_DATA_FDT:
349 error = apple_aic_map_intr_fdt(sc,
350 (struct intr_map_data_fdt *)data, &irq, NULL, NULL, NULL,
351 &die);
352 if (error == 0)
353 *isrcp = &sc->sc_isrcs[0 /* XXX */][irq].ai_isrc;
354 break;
355 default:
356 return (ENOTSUP);
357 }
358
359 return (error);
360 }
361
362 static int
apple_aic_setup_intr(device_t dev,struct intr_irqsrc * isrc,struct resource * res,struct intr_map_data * data)363 apple_aic_setup_intr(device_t dev, struct intr_irqsrc *isrc,
364 struct resource *res, struct intr_map_data *data)
365 {
366 struct apple_aic_softc *sc;
367 enum apple_aic_irq_type type;
368 struct apple_aic_irqsrc *ai;
369 enum intr_trigger trig;
370 enum intr_polarity pol;
371 int error;
372 u_int die, irq;
373
374 sc = device_get_softc(dev);
375 ai = (struct apple_aic_irqsrc *)isrc;
376
377 if (data != NULL) {
378 KASSERT(data->type == INTR_MAP_DATA_FDT,
379 ("%s: Only FDT data is supported (got %#x)", __func__,
380 data->type));
381 error = apple_aic_map_intr_fdt(sc,
382 (struct intr_map_data_fdt *)data, &irq, &type, &pol, &trig,
383 &die);
384 if (error != 0)
385 return (error);
386 } else {
387 pol = INTR_POLARITY_CONFORM;
388 trig = INTR_TRIGGER_CONFORM;
389 }
390
391 if (isrc->isrc_handlers != 0) {
392 /* TODO */
393 return (0);
394 }
395
396 if (pol == INTR_POLARITY_CONFORM)
397 pol = INTR_POLARITY_LOW;
398 if (trig == INTR_TRIGGER_CONFORM)
399 trig = INTR_TRIGGER_EDGE;
400
401 ai->ai_pol = pol;
402 ai->ai_trig = trig;
403 ai->ai_type = type;
404
405 /*
406 * Only the timer uses FIQs. These could be sent to any CPU.
407 */
408 switch (type) {
409 case AIC_TYPE_IRQ:
410 /* XXX die sensitive? */
411 aic_next_cpu = intr_irq_next_cpu(aic_next_cpu, &all_cpus);
412 bus_write_4(sc->sc_mem, AIC_TARGET_CPU(irq),
413 1 << sc->sc_cpuids[aic_next_cpu]);
414 break;
415 case AIC_TYPE_FIQ:
416 isrc->isrc_flags |= INTR_ISRCF_PPI;
417 break;
418 default:
419 return (EINVAL);
420 }
421
422 return (0);
423 }
424
425 static int
apple_aic_teardown_intr(device_t dev,struct intr_irqsrc * isrc,struct resource * res,struct intr_map_data * data)426 apple_aic_teardown_intr(device_t dev, struct intr_irqsrc *isrc,
427 struct resource *res, struct intr_map_data *data)
428 {
429 panic("%s\n", __func__);
430 }
431
432 static void
apple_aic_enable_intr(device_t dev,struct intr_irqsrc * isrc)433 apple_aic_enable_intr(device_t dev, struct intr_irqsrc *isrc)
434 {
435 struct apple_aic_irqsrc *ai;
436 struct apple_aic_softc *sc;
437 u_int irq;
438
439 ai = (struct apple_aic_irqsrc *)isrc;
440 irq = ai->ai_irq;
441 switch(ai->ai_type) {
442 case AIC_TYPE_IRQ:
443 sc = device_get_softc(dev);
444 bus_write_4(sc->sc_mem, AIC_MASK_CLEAR(irq), AIC_IRQ_MASK(irq));
445 break;
446 case AIC_TYPE_IPI:
447 /* Nothing needed here. */
448 break;
449 case AIC_TYPE_FIQ:
450 /* TODO */
451 break;
452 default:
453 panic("%s: %x\n", __func__, ai->ai_type);
454 }
455 }
456
457 static void
apple_aic_disable_intr(device_t dev,struct intr_irqsrc * isrc)458 apple_aic_disable_intr(device_t dev, struct intr_irqsrc *isrc)
459 {
460 struct apple_aic_irqsrc *ai;
461 struct apple_aic_softc *sc;
462 u_int irq;
463
464 ai = (struct apple_aic_irqsrc *)isrc;
465 irq = ai->ai_irq;
466 switch(ai->ai_type) {
467 case AIC_TYPE_IRQ:
468 sc = device_get_softc(dev);
469 bus_write_4(sc->sc_mem, AIC_MASK_SET(irq), AIC_IRQ_MASK(irq));
470 break;
471 case AIC_TYPE_IPI:
472 /* Nothing needed here. */
473 break;
474 case AIC_TYPE_FIQ:
475 /* TODO */
476 break;
477 default:
478 panic("%s: %x\n", __func__, ai->ai_type);
479 }
480 }
481
482 static void
apple_aic_post_filter(device_t dev,struct intr_irqsrc * isrc)483 apple_aic_post_filter(device_t dev, struct intr_irqsrc *isrc)
484 {
485 struct apple_aic_softc *sc;
486 struct apple_aic_irqsrc *ai;
487 int irq;
488
489 ai = (struct apple_aic_irqsrc *)isrc;
490 irq = ai->ai_irq;
491 switch(ai->ai_type) {
492 case AIC_TYPE_IRQ:
493 sc = device_get_softc(dev);
494 bus_write_4(sc->sc_mem, AIC_SW_CLEAR(irq), AIC_IRQ_MASK(irq));
495 bus_write_4(sc->sc_mem, AIC_MASK_CLEAR(irq), AIC_IRQ_MASK(irq));
496 break;
497 case AIC_TYPE_FIQ:
498 /* TODO */
499 break;
500 default:
501 panic("%s: %x\n", __func__, ai->ai_type);
502 }
503 }
504
505 static void
apple_aic_pre_ithread(device_t dev,struct intr_irqsrc * isrc)506 apple_aic_pre_ithread(device_t dev, struct intr_irqsrc *isrc)
507 {
508 struct apple_aic_softc *sc;
509 struct apple_aic_irqsrc *ai;
510 int irq;
511
512 ai = (struct apple_aic_irqsrc *)isrc;
513 sc = device_get_softc(dev);
514 irq = ai->ai_irq;
515 bus_write_4(sc->sc_mem, AIC_SW_CLEAR(irq), AIC_IRQ_MASK(irq));
516 apple_aic_disable_intr(dev, isrc);
517 /* ACK IT */
518 }
519
520 static void
apple_aic_post_ithread(device_t dev,struct intr_irqsrc * isrc)521 apple_aic_post_ithread(device_t dev, struct intr_irqsrc *isrc)
522 {
523 struct apple_aic_softc *sc;
524 struct apple_aic_irqsrc *ai;
525 int irq;
526
527 ai = (struct apple_aic_irqsrc *)isrc;
528 sc = device_get_softc(dev);
529 irq = ai->ai_irq;
530
531 bus_write_4(sc->sc_mem, AIC_MASK_CLEAR(irq), AIC_IRQ_MASK(irq));
532 apple_aic_enable_intr(dev, isrc);
533 }
534
535 #ifdef SMP
536 static void
apple_aic_ipi_received(struct apple_aic_softc * sc,struct trapframe * tf)537 apple_aic_ipi_received(struct apple_aic_softc *sc, struct trapframe *tf)
538 {
539 uint32_t mask;
540 uint32_t ipi;
541 int cpu;
542
543 cpu = PCPU_GET(cpuid);
544
545 mask = atomic_readandclear_32(&sc->sc_ipimasks[cpu]);
546
547 while (mask != 0) {
548 ipi = ffs(mask) - 1;
549 mask &= ~(1 << ipi);
550
551 intr_ipi_dispatch(ipi);
552 }
553 }
554 #endif
555
556 static int
apple_aic_irq(void * arg)557 apple_aic_irq(void *arg)
558 {
559 struct apple_aic_softc *sc;
560 uint32_t die, event, irq, type;
561 struct apple_aic_irqsrc *aisrc;
562 struct trapframe *tf;
563
564 sc = arg;
565 tf = curthread->td_intr_frame;
566
567 event = bus_read_4(sc->sc_mem, AIC_EVENT);
568 type = AIC_EVENT_TYPE(event);
569
570 /* If we get an IPI here, we really goofed. */
571 MPASS(type != AIC_EVENT_TYPE_IPI);
572
573 if (type != AIC_EVENT_TYPE_IRQ) {
574 if (type != AIC_EVENT_TYPE_NONE)
575 device_printf(sc->sc_dev, "unexpected event type %d\n",
576 type);
577 return (FILTER_STRAY);
578 }
579
580 die = AIC_EVENT_DIE(event);
581 irq = AIC_EVENT_IRQ(event);
582
583 if (die >= sc->sc_ndie)
584 panic("%s: unexpected die %d", __func__, die);
585 if (irq >= sc->sc_nirqs)
586 panic("%s: unexpected irq %d", __func__, irq);
587
588 aisrc = &sc->sc_isrcs[die][irq];
589 if (intr_isrc_dispatch(&aisrc->ai_isrc, tf) != 0) {
590 device_printf(sc->sc_dev, "Stray irq %u:%u disabled\n",
591 die, irq);
592 return (FILTER_STRAY);
593 }
594
595 return (FILTER_HANDLED);
596 }
597
598 static int
apple_aic_fiq(void * arg)599 apple_aic_fiq(void *arg)
600 {
601 struct apple_aic_softc *sc;
602 struct apple_aic_irqsrc *isrcs;
603 struct trapframe *tf;
604
605 sc = arg;
606 tf = curthread->td_intr_frame;
607
608 #ifdef SMP
609 /* Handle IPIs. */
610 if ((READ_SPECIALREG(AIC_IPI_SR_EL1) & AIC_IPI_SR_EL1_PENDING) != 0) {
611 WRITE_SPECIALREG(AIC_IPI_SR_EL1, AIC_IPI_SR_EL1_PENDING);
612 apple_aic_ipi_received(sc, tf);
613 }
614 #endif
615
616 /*
617 * FIQs don't store any state in the interrupt controller at all outside
618 * of IPI handling, so we have to probe around outside of AIC to
619 * determine if we might have been fired off due to a timer.
620 */
621 isrcs = sc->sc_isrcs[0];
622 if ((READ_SPECIALREG(cntv_ctl_el0) & CNTV_CTL_BITS) ==
623 (CNTV_CTL_ENABLE | CNTV_CTL_ISTATUS)) {
624 intr_isrc_dispatch(&isrcs[AIC_TMR_GUEST_VIRT].ai_isrc, tf);
625 }
626
627 if (has_hyp()) {
628 uint64_t reg;
629
630 if ((READ_SPECIALREG(cntp_ctl_el0) & CNTV_CTL_ISTATUS) != 0) {
631 intr_isrc_dispatch(&isrcs[AIC_TMR_GUEST_PHYS].ai_isrc,
632 tf);
633 }
634
635 reg = READ_SPECIALREG(AIC_FIQ_VM_TIMER);
636 if ((reg & AIC_FIQ_VM_TIMER_PEN) != 0) {
637 intr_isrc_dispatch(&isrcs[AIC_TMR_HV_PHYS].ai_isrc, tf);
638 }
639
640 if ((reg & AIC_FIQ_VM_TIMER_VEN) != 0) {
641 intr_isrc_dispatch(&isrcs[AIC_TMR_HV_VIRT].ai_isrc, tf);
642 }
643 }
644
645 return (FILTER_HANDLED);
646 }
647
648 #ifdef SMP
649 static int
apple_aic_bind_intr(device_t dev,struct intr_irqsrc * isrc)650 apple_aic_bind_intr(device_t dev, struct intr_irqsrc *isrc)
651 {
652 struct apple_aic_softc *sc = device_get_softc(dev);
653 static int aic_next_cpu;
654 uint32_t targets = 0;
655 u_int irq, cpu;
656
657 MPASS(((struct apple_aic_irqsrc *)isrc)->ai_type == AIC_TYPE_IRQ);
658 irq = ((struct apple_aic_irqsrc *)isrc)->ai_irq;
659 if (CPU_EMPTY(&isrc->isrc_cpu)) {
660 aic_next_cpu = intr_irq_next_cpu(aic_next_cpu, &all_cpus);
661 CPU_SETOF(aic_next_cpu, &isrc->isrc_cpu);
662 bus_write_4(sc->sc_mem, AIC_TARGET_CPU(irq),
663 sc->sc_cpuids[aic_next_cpu] << 1);
664 } else {
665 CPU_FOREACH_ISSET(cpu, &isrc->isrc_cpu) {
666 targets |= sc->sc_cpuids[cpu] << 1;
667 }
668 bus_write_4(sc->sc_mem, AIC_TARGET_CPU(irq), targets);
669 }
670 return (0);
671 }
672
673 static void
apple_aic_ipi_send(device_t dev,struct intr_irqsrc * isrc,cpuset_t cpus,u_int ipi)674 apple_aic_ipi_send(device_t dev, struct intr_irqsrc *isrc, cpuset_t cpus,
675 u_int ipi)
676 {
677 struct apple_aic_softc *sc;
678 uint64_t aff, localgrp, sendmask;
679 u_int cpu;
680
681 sc = device_get_softc(dev);
682 sendmask = 0;
683 localgrp = CPU_AFF1(CPU_AFFINITY(PCPU_GET(cpuid)));
684
685 KASSERT(isrc == &sc->sc_ipi_srcs[ipi].ai_isrc,
686 ("%s: bad ISRC %p argument", __func__, isrc));
687 for (cpu = 0; cpu <= mp_maxid; cpu++) {
688 if (CPU_ISSET(cpu, &cpus)) {
689 aff = CPU_AFFINITY(cpu);
690 sendmask = CPU_AFF0(aff);
691 atomic_set_32(&sc->sc_ipimasks[cpu], 1 << ipi);
692
693 /*
694 * The above write to sc_ipimasks needs to be visible
695 * before we write to the ipi register to avoid the
696 * targetted CPU missing the dispatch in
697 * apple_aic_ipi_received(). Note that WRITE_SPECIALREG
698 * isn't a memory operation, so we can't relax this to a
699 * a dmb.
700 */
701 dsb(ishst);
702
703 if (CPU_AFF1(aff) == localgrp) {
704 WRITE_SPECIALREG(AIC_IPI_LOCAL_RR_EL1,
705 sendmask);
706 } else {
707 sendmask |= CPU_AFF1(aff) << 16;
708 WRITE_SPECIALREG(AIC_IPI_GLOBAL_RR_EL1,
709 sendmask);
710 }
711
712 isb();
713 }
714 }
715 }
716
717 static int
apple_aic_ipi_setup(device_t dev,u_int ipi,struct intr_irqsrc ** isrcp)718 apple_aic_ipi_setup(device_t dev, u_int ipi, struct intr_irqsrc **isrcp)
719 {
720 struct apple_aic_softc *sc = device_get_softc(dev);
721 struct apple_aic_irqsrc *ai;
722
723 KASSERT(ipi < AIC_NIPIS, ("%s: ipi %u too high", __func__, ipi));
724
725 ai = &sc->sc_ipi_srcs[ipi];
726 ai->ai_type = AIC_TYPE_IPI;
727
728 *isrcp = &ai->ai_isrc;
729 return (0);
730 }
731
732 static void
apple_aic_init_secondary(device_t dev,uint32_t rootnum)733 apple_aic_init_secondary(device_t dev, uint32_t rootnum)
734 {
735 struct apple_aic_softc *sc = device_get_softc(dev);
736 u_int cpu = PCPU_GET(cpuid);
737
738 /* We don't need to re-initialize for the FIQ root. */
739 if (rootnum != INTR_ROOT_IRQ)
740 return;
741
742 sc->sc_cpuids[cpu] = bus_read_4(sc->sc_mem, AIC_WHOAMI);
743 if (bootverbose)
744 device_printf(dev, "CPU %d: whoami %x\n", cpu,
745 sc->sc_cpuids[cpu]);
746
747 bus_write_4(sc->sc_mem, AIC_IPI_MASK_SET, AIC_IPI_SELF | AIC_IPI_OTHER);
748 }
749 #endif
750
751 static device_method_t apple_aic_methods[] = {
752 /* Device interface */
753 DEVMETHOD(device_probe, apple_aic_probe),
754 DEVMETHOD(device_attach, apple_aic_attach),
755
756 /* Interrupt controller interface */
757 DEVMETHOD(pic_disable_intr, apple_aic_disable_intr),
758 DEVMETHOD(pic_enable_intr, apple_aic_enable_intr),
759 DEVMETHOD(pic_map_intr, apple_aic_map_intr),
760 DEVMETHOD(pic_setup_intr, apple_aic_setup_intr),
761 DEVMETHOD(pic_teardown_intr, apple_aic_teardown_intr),
762 DEVMETHOD(pic_post_filter, apple_aic_post_filter),
763 DEVMETHOD(pic_post_ithread, apple_aic_post_ithread),
764 DEVMETHOD(pic_pre_ithread, apple_aic_pre_ithread),
765 #ifdef SMP
766 DEVMETHOD(pic_bind_intr, apple_aic_bind_intr),
767 DEVMETHOD(pic_init_secondary, apple_aic_init_secondary),
768 DEVMETHOD(pic_ipi_send, apple_aic_ipi_send),
769 DEVMETHOD(pic_ipi_setup, apple_aic_ipi_setup),
770 #endif
771
772 /* End */
773 DEVMETHOD_END
774 };
775
776 static DEFINE_CLASS_0(aic, apple_aic_driver, apple_aic_methods,
777 sizeof(struct apple_aic_softc));
778
779 EARLY_DRIVER_MODULE(aic, simplebus, apple_aic_driver, 0, 0,
780 BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE);
781