1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2021 Andrew Turner
5 * Copyright (c) 2022 Michael J. Karels <karels@freebsd.org>
6 * Copyright (c) 2022 Kyle Evans <kevans@FreeBSD.org>
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 #include "opt_platform.h"
31
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/bus.h>
35 #include <sys/kernel.h>
36 #include <sys/module.h>
37 #include <sys/proc.h>
38 #include <sys/rman.h>
39 #include <sys/smp.h>
40
41 #include <machine/bus.h>
42 #include <machine/machdep.h>
43 #ifdef SMP
44 #include <machine/intr.h>
45 #include <machine/smp.h>
46 #endif
47
48 #include <dev/fdt/fdt_intr.h>
49
50 #include <dev/ofw/openfirm.h>
51 #include <dev/ofw/ofw_bus.h>
52 #include <dev/ofw/ofw_bus_subr.h>
53
54 #include <dt-bindings/interrupt-controller/apple-aic.h>
55
56 #include "pic_if.h"
57
58 #define AIC_INFO 0x0004
59 #define AIC_INFO_NDIE(val) (((val) >> 24) & 0xf)
60 #define AIC_INFO_NIRQS(val) ((val) & 0x0000ffff)
61
62 #define AIC_WHOAMI 0x2000
63 #define AIC_EVENT 0x2004
64 #define AIC_EVENT_DIE(val) (((val) >> 24) & 0xff)
65 #define AIC_EVENT_TYPE(val) (((val) >> 16) & 0xff)
66 #define AIC_EVENT_TYPE_NONE 0
67 #define AIC_EVENT_TYPE_IRQ 1
68 #define AIC_EVENT_TYPE_IPI 4
69 #define AIC_EVENT_IRQ(val) ((val) & 0xffff)
70 #define AIC_EVENT_IPI_OTHER 1
71 #define AIC_EVENT_IPI_SELF 2
72 #define AIC_IPI_SEND 0x2008
73 #define AIC_IPI_ACK 0x200c
74 #define AIC_IPI_MASK_SET 0x2024
75 #define AIC_IPI_MASK_CLR 0x2028
76 #define AIC_IPI_OTHER 0x00000001
77 #define AIC_IPI_SELF 0x80000000
78 #define AIC_TARGET_CPU(irq) (0x3000 + (irq) * 4)
79 #define AIC_SW_SET(irq) (0x4000 + (((irq) >> 5) * 4))
80 #define AIC_SW_CLEAR(irq) (0x4080 + (((irq) >> 5) * 4))
81 #define AIC_MASK_SET(irq) (0x4100 + (((irq) >> 5) * 4))
82 #define AIC_MASK_CLEAR(irq) (0x4180 + (((irq) >> 5) * 4))
83 #define AIC_IRQ_MASK(irq) (1u << ((irq) & 0x1f))
84
85 #define AIC_IPI_LOCAL_RR_EL1 s3_5_c15_c0_0
86 #define AIC_IPI_GLOBAL_RR_EL1 s3_5_c15_c0_1
87
88 #define AIC_IPI_SR_EL1 s3_5_c15_c1_1
89 #define AIC_IPI_SR_EL1_PENDING (1 << 0)
90
91 #define AIC_FIQ_VM_TIMER s3_5_c15_c1_3
92 #define AIC_FIQ_VM_TIMER_VEN (1 << 0)
93 #define AIC_FIQ_VM_TIMER_PEN (1 << 1)
94 #define AIC_FIQ_VM_TIMER_BITS (AIC_FIQ_VM_TIMER_VEN | AIC_FIQ_VM_TIMER_PEN)
95
96 #define CNTV_CTL_BITS \
97 (CNTV_CTL_ENABLE | CNTV_CTL_IMASK | CNTV_CTL_ISTATUS)
98
99 #define AIC_MAXCPUS 32
100 #define AIC_MAXDIES 4
101
102 static struct ofw_compat_data compat_data[] = {
103 { "apple,aic", 1 },
104 { NULL, 0 }
105 };
106
107 enum apple_aic_irq_type {
108 AIC_TYPE_INVAL,
109 AIC_TYPE_IRQ,
110 AIC_TYPE_FIQ,
111 AIC_TYPE_IPI,
112 };
113
114 struct apple_aic_irqsrc {
115 struct intr_irqsrc ai_isrc;
116 enum apple_aic_irq_type ai_type;
117 struct {
118 /* AIC_TYPE_IRQ */
119 enum intr_polarity ai_pol;
120 enum intr_trigger ai_trig;
121 u_int ai_irq;
122 };
123 };
124
125 #ifdef SMP
126 #define AIC_NIPIS INTR_IPI_COUNT
127 #endif
128
129 struct apple_aic_softc {
130 device_t sc_dev;
131 struct resource *sc_mem;
132 struct apple_aic_irqsrc *sc_isrcs[AIC_MAXDIES];
133 u_int sc_nirqs;
134 u_int sc_ndie;
135 #ifdef SMP
136 struct apple_aic_irqsrc sc_ipi_srcs[AIC_NIPIS];
137 uint32_t *sc_ipimasks;
138 #endif
139 u_int *sc_cpuids; /* cpu index to AIC CPU ID */
140 };
141
142 static u_int aic_next_cpu;
143
144 static device_probe_t apple_aic_probe;
145 static device_attach_t apple_aic_attach;
146
147 static pic_disable_intr_t apple_aic_disable_intr;
148 static pic_enable_intr_t apple_aic_enable_intr;
149 static pic_map_intr_t apple_aic_map_intr;
150 static pic_setup_intr_t apple_aic_setup_intr;
151 static pic_teardown_intr_t apple_aic_teardown_intr;
152 static pic_post_filter_t apple_aic_post_filter;
153 static pic_post_ithread_t apple_aic_post_ithread;
154 static pic_pre_ithread_t apple_aic_pre_ithread;
155 #ifdef SMP
156 static pic_bind_intr_t apple_aic_bind_intr;
157 static pic_init_secondary_t apple_aic_init_secondary;
158 static pic_ipi_send_t apple_aic_ipi_send;
159 static pic_ipi_setup_t apple_aic_ipi_setup;
160 #endif
161
162 static int apple_aic_irq(void *);
163 static int apple_aic_fiq(void *);
164
165 static int
apple_aic_probe(device_t dev)166 apple_aic_probe(device_t dev)
167 {
168
169 if (!ofw_bus_status_okay(dev))
170 return (ENXIO);
171
172 if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0)
173 return (ENXIO);
174
175 device_set_desc(dev, "Apple Interrupt Controller");
176 return (BUS_PROBE_DEFAULT);
177 }
178
179 static int
apple_aic_attach(device_t dev)180 apple_aic_attach(device_t dev)
181 {
182 struct apple_aic_softc *sc;
183 struct intr_irqsrc *isrc;
184 const char *name;
185 intptr_t xref;
186 int error, rid;
187 u_int i, cpu, j, info;
188
189 sc = device_get_softc(dev);
190 sc->sc_dev = dev;
191
192 rid = 0;
193 sc->sc_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
194 RF_ACTIVE);
195 if (sc->sc_mem == NULL) {
196 device_printf(dev, "Unable to allocate memory\n");
197 return (ENXIO);
198 }
199
200 info = bus_read_4(sc->sc_mem, AIC_INFO);
201 sc->sc_nirqs = AIC_INFO_NIRQS(info);
202 sc->sc_ndie = AIC_INFO_NDIE(info) + 1;
203 if (bootverbose)
204 device_printf(dev, "Found %d interrupts, %d die\n",
205 sc->sc_nirqs, sc->sc_ndie);
206
207 for (i = 0; i < sc->sc_ndie; i++) {
208 sc->sc_isrcs[i] = mallocarray(sc->sc_nirqs,
209 sizeof(**sc->sc_isrcs), M_DEVBUF, M_WAITOK | M_ZERO);
210 }
211
212 #ifdef SMP
213 sc->sc_ipimasks = malloc(sizeof(*sc->sc_ipimasks) * (mp_maxid + 1),
214 M_DEVBUF, M_WAITOK | M_ZERO);
215 #endif
216 sc->sc_cpuids = malloc(sizeof(*sc->sc_cpuids) * (mp_maxid + 1),
217 M_DEVBUF, M_WAITOK | M_ZERO);
218
219 cpu = PCPU_GET(cpuid);
220 sc->sc_cpuids[cpu] = bus_read_4(sc->sc_mem, AIC_WHOAMI);
221 if (bootverbose)
222 device_printf(dev, "BSP CPU %d: whoami %x\n", cpu,
223 sc->sc_cpuids[cpu]);
224
225 name = device_get_nameunit(dev);
226 for (i = 0; i < sc->sc_ndie; i++) {
227 struct apple_aic_irqsrc *die_isrcs;
228
229 die_isrcs = sc->sc_isrcs[i];
230 for (j = 0; j < sc->sc_nirqs; j++) {
231 isrc = &die_isrcs[j].ai_isrc;
232 die_isrcs[j].ai_pol = INTR_POLARITY_CONFORM;
233 die_isrcs[j].ai_trig = INTR_TRIGGER_CONFORM;
234 die_isrcs[j].ai_type = AIC_TYPE_INVAL;
235 die_isrcs[j].ai_irq = j;
236
237 error = intr_isrc_register(isrc, dev, 0, "%s,d%us%u", name,
238 i, j);
239 if (error != 0) {
240 device_printf(dev, "Unable to register irq %u:%u\n",
241 i, j);
242 return (error);
243 }
244 }
245 }
246
247 xref = OF_xref_from_node(ofw_bus_get_node(dev));
248 if (intr_pic_register(dev, xref) == NULL) {
249 device_printf(dev, "Unable to register interrupt handler\n");
250 return (ENXIO);
251 }
252
253 if (intr_pic_claim_root(dev, xref, apple_aic_irq, sc,
254 INTR_ROOT_IRQ) != 0) {
255 device_printf(dev,
256 "Unable to set root interrupt controller\n");
257 intr_pic_deregister(dev, xref);
258 return (ENXIO);
259 }
260
261 if (intr_pic_claim_root(dev, xref, apple_aic_fiq, sc,
262 INTR_ROOT_FIQ) != 0) {
263 device_printf(dev,
264 "Unable to set root fiq controller\n");
265 intr_pic_deregister(dev, xref);
266 return (ENXIO);
267 }
268
269 #ifdef SMP
270 if (intr_ipi_pic_register(dev, 0) != 0) {
271 device_printf(dev, "could not register for IPIs\n");
272 return (ENXIO);
273 }
274 #endif
275
276 OF_device_register_xref(xref, dev);
277
278 return (0);
279 }
280
281 static int
apple_aic_map_intr_fdt(struct apple_aic_softc * sc,struct intr_map_data_fdt * data,u_int * irq,enum apple_aic_irq_type * typep,enum intr_polarity * polp,enum intr_trigger * trigp,u_int * die)282 apple_aic_map_intr_fdt(struct apple_aic_softc *sc,
283 struct intr_map_data_fdt *data, u_int *irq, enum apple_aic_irq_type *typep,
284 enum intr_polarity *polp, enum intr_trigger *trigp, u_int *die)
285 {
286 if (data->ncells != 3)
287 return (EINVAL);
288
289 /* XXX AIC2 */
290 *die = 0;
291
292 /*
293 * The first cell is the interrupt type:
294 * 0 = IRQ
295 * 1 = FIQ
296 * The second cell is the interrupt number
297 * The third cell is the flags
298 */
299 switch(data->cells[0]) {
300 case 0:
301 if (typep != NULL)
302 *typep = AIC_TYPE_IRQ;
303 break;
304 case 1:
305 if (typep != NULL)
306 *typep = AIC_TYPE_FIQ;
307 break;
308 default:
309 return (EINVAL);
310 }
311
312 *irq = data->cells[1];
313 if (*irq > sc->sc_nirqs)
314 return (EINVAL);
315
316 if (trigp != NULL) {
317 if ((data->cells[2] & FDT_INTR_EDGE_MASK) != 0)
318 *trigp = INTR_TRIGGER_EDGE;
319 else
320 *trigp = INTR_TRIGGER_LEVEL;
321 }
322 if (polp != NULL) {
323 if ((data->cells[2] & FDT_INTR_LEVEL_HIGH) != 0)
324 *polp = INTR_POLARITY_HIGH;
325 else
326 *polp = INTR_POLARITY_LOW;
327 }
328
329 return (0);
330 }
331
332 static int
apple_aic_map_intr(device_t dev,struct intr_map_data * data,struct intr_irqsrc ** isrcp)333 apple_aic_map_intr(device_t dev, struct intr_map_data *data,
334 struct intr_irqsrc **isrcp)
335 {
336 struct apple_aic_softc *sc;
337 int error;
338 u_int irq;
339 u_int die;
340
341 sc = device_get_softc(dev);
342
343 error = 0;
344 switch(data->type) {
345 case INTR_MAP_DATA_FDT:
346 error = apple_aic_map_intr_fdt(sc,
347 (struct intr_map_data_fdt *)data, &irq, NULL, NULL, NULL,
348 &die);
349 if (error == 0)
350 *isrcp = &sc->sc_isrcs[0 /* XXX */][irq].ai_isrc;
351 break;
352 default:
353 return (ENOTSUP);
354 }
355
356 return (error);
357 }
358
359 static int
apple_aic_setup_intr(device_t dev,struct intr_irqsrc * isrc,struct resource * res,struct intr_map_data * data)360 apple_aic_setup_intr(device_t dev, struct intr_irqsrc *isrc,
361 struct resource *res, struct intr_map_data *data)
362 {
363 struct apple_aic_softc *sc;
364 enum apple_aic_irq_type type;
365 struct apple_aic_irqsrc *ai;
366 enum intr_trigger trig;
367 enum intr_polarity pol;
368 int error;
369 u_int die, irq;
370
371 sc = device_get_softc(dev);
372 ai = (struct apple_aic_irqsrc *)isrc;
373
374 if (data != NULL) {
375 KASSERT(data->type == INTR_MAP_DATA_FDT,
376 ("%s: Only FDT data is supported (got %#x)", __func__,
377 data->type));
378 error = apple_aic_map_intr_fdt(sc,
379 (struct intr_map_data_fdt *)data, &irq, &type, &pol, &trig,
380 &die);
381 if (error != 0)
382 return (error);
383 } else {
384 pol = INTR_POLARITY_CONFORM;
385 trig = INTR_TRIGGER_CONFORM;
386 }
387
388 if (isrc->isrc_handlers != 0) {
389 /* TODO */
390 return (0);
391 }
392
393 if (pol == INTR_POLARITY_CONFORM)
394 pol = INTR_POLARITY_LOW;
395 if (trig == INTR_TRIGGER_CONFORM)
396 trig = INTR_TRIGGER_EDGE;
397
398 ai->ai_pol = pol;
399 ai->ai_trig = trig;
400 ai->ai_type = type;
401
402 /*
403 * Only the timer uses FIQs. These could be sent to any CPU.
404 */
405 switch (type) {
406 case AIC_TYPE_IRQ:
407 /* XXX die sensitive? */
408 aic_next_cpu = intr_irq_next_cpu(aic_next_cpu, &all_cpus);
409 bus_write_4(sc->sc_mem, AIC_TARGET_CPU(irq),
410 1 << sc->sc_cpuids[aic_next_cpu]);
411 break;
412 case AIC_TYPE_FIQ:
413 isrc->isrc_flags |= INTR_ISRCF_PPI;
414 break;
415 default:
416 return (EINVAL);
417 }
418
419 return (0);
420 }
421
422 static int
apple_aic_teardown_intr(device_t dev,struct intr_irqsrc * isrc,struct resource * res,struct intr_map_data * data)423 apple_aic_teardown_intr(device_t dev, struct intr_irqsrc *isrc,
424 struct resource *res, struct intr_map_data *data)
425 {
426 panic("%s\n", __func__);
427 }
428
429 static void
apple_aic_enable_intr(device_t dev,struct intr_irqsrc * isrc)430 apple_aic_enable_intr(device_t dev, struct intr_irqsrc *isrc)
431 {
432 struct apple_aic_irqsrc *ai;
433 struct apple_aic_softc *sc;
434 u_int irq;
435
436 ai = (struct apple_aic_irqsrc *)isrc;
437 irq = ai->ai_irq;
438 switch(ai->ai_type) {
439 case AIC_TYPE_IRQ:
440 sc = device_get_softc(dev);
441 bus_write_4(sc->sc_mem, AIC_MASK_CLEAR(irq), AIC_IRQ_MASK(irq));
442 break;
443 case AIC_TYPE_IPI:
444 /* Nothing needed here. */
445 break;
446 case AIC_TYPE_FIQ:
447 /* TODO */
448 break;
449 default:
450 panic("%s: %x\n", __func__, ai->ai_type);
451 }
452 }
453
454 static void
apple_aic_disable_intr(device_t dev,struct intr_irqsrc * isrc)455 apple_aic_disable_intr(device_t dev, struct intr_irqsrc *isrc)
456 {
457 struct apple_aic_irqsrc *ai;
458 struct apple_aic_softc *sc;
459 u_int irq;
460
461 ai = (struct apple_aic_irqsrc *)isrc;
462 irq = ai->ai_irq;
463 switch(ai->ai_type) {
464 case AIC_TYPE_IRQ:
465 sc = device_get_softc(dev);
466 bus_write_4(sc->sc_mem, AIC_MASK_SET(irq), AIC_IRQ_MASK(irq));
467 break;
468 case AIC_TYPE_IPI:
469 /* Nothing needed here. */
470 break;
471 case AIC_TYPE_FIQ:
472 /* TODO */
473 break;
474 default:
475 panic("%s: %x\n", __func__, ai->ai_type);
476 }
477 }
478
479 static void
apple_aic_post_filter(device_t dev,struct intr_irqsrc * isrc)480 apple_aic_post_filter(device_t dev, struct intr_irqsrc *isrc)
481 {
482 struct apple_aic_softc *sc;
483 struct apple_aic_irqsrc *ai;
484 int irq;
485
486 ai = (struct apple_aic_irqsrc *)isrc;
487 irq = ai->ai_irq;
488 switch(ai->ai_type) {
489 case AIC_TYPE_IRQ:
490 sc = device_get_softc(dev);
491 bus_write_4(sc->sc_mem, AIC_SW_CLEAR(irq), AIC_IRQ_MASK(irq));
492 bus_write_4(sc->sc_mem, AIC_MASK_CLEAR(irq), AIC_IRQ_MASK(irq));
493 break;
494 case AIC_TYPE_FIQ:
495 /* TODO */
496 break;
497 default:
498 panic("%s: %x\n", __func__, ai->ai_type);
499 }
500 }
501
502 static void
apple_aic_pre_ithread(device_t dev,struct intr_irqsrc * isrc)503 apple_aic_pre_ithread(device_t dev, struct intr_irqsrc *isrc)
504 {
505 struct apple_aic_softc *sc;
506 struct apple_aic_irqsrc *ai;
507 int irq;
508
509 ai = (struct apple_aic_irqsrc *)isrc;
510 sc = device_get_softc(dev);
511 irq = ai->ai_irq;
512 bus_write_4(sc->sc_mem, AIC_SW_CLEAR(irq), AIC_IRQ_MASK(irq));
513 apple_aic_disable_intr(dev, isrc);
514 /* ACK IT */
515 }
516
517 static void
apple_aic_post_ithread(device_t dev,struct intr_irqsrc * isrc)518 apple_aic_post_ithread(device_t dev, struct intr_irqsrc *isrc)
519 {
520 struct apple_aic_softc *sc;
521 struct apple_aic_irqsrc *ai;
522 int irq;
523
524 ai = (struct apple_aic_irqsrc *)isrc;
525 sc = device_get_softc(dev);
526 irq = ai->ai_irq;
527
528 bus_write_4(sc->sc_mem, AIC_MASK_CLEAR(irq), AIC_IRQ_MASK(irq));
529 apple_aic_enable_intr(dev, isrc);
530 }
531
532 #ifdef SMP
533 static void
apple_aic_ipi_received(struct apple_aic_softc * sc,struct trapframe * tf)534 apple_aic_ipi_received(struct apple_aic_softc *sc, struct trapframe *tf)
535 {
536 uint32_t mask;
537 uint32_t ipi;
538 int cpu;
539
540 cpu = PCPU_GET(cpuid);
541
542 mask = atomic_readandclear_32(&sc->sc_ipimasks[cpu]);
543
544 while (mask != 0) {
545 ipi = ffs(mask) - 1;
546 mask &= ~(1 << ipi);
547
548 intr_ipi_dispatch(ipi);
549 }
550 }
551 #endif
552
553 static int
apple_aic_irq(void * arg)554 apple_aic_irq(void *arg)
555 {
556 struct apple_aic_softc *sc;
557 uint32_t die, event, irq, type;
558 struct apple_aic_irqsrc *aisrc;
559 struct trapframe *tf;
560
561 sc = arg;
562 tf = curthread->td_intr_frame;
563
564 event = bus_read_4(sc->sc_mem, AIC_EVENT);
565 type = AIC_EVENT_TYPE(event);
566
567 /* If we get an IPI here, we really goofed. */
568 MPASS(type != AIC_EVENT_TYPE_IPI);
569
570 if (type != AIC_EVENT_TYPE_IRQ) {
571 if (type != AIC_EVENT_TYPE_NONE)
572 device_printf(sc->sc_dev, "unexpected event type %d\n",
573 type);
574 return (FILTER_STRAY);
575 }
576
577 die = AIC_EVENT_DIE(event);
578 irq = AIC_EVENT_IRQ(event);
579
580 if (die >= sc->sc_ndie)
581 panic("%s: unexpected die %d", __func__, die);
582 if (irq >= sc->sc_nirqs)
583 panic("%s: unexpected irq %d", __func__, irq);
584
585 aisrc = &sc->sc_isrcs[die][irq];
586 if (intr_isrc_dispatch(&aisrc->ai_isrc, tf) != 0) {
587 device_printf(sc->sc_dev, "Stray irq %u:%u disabled\n",
588 die, irq);
589 return (FILTER_STRAY);
590 }
591
592 return (FILTER_HANDLED);
593 }
594
595 static int
apple_aic_fiq(void * arg)596 apple_aic_fiq(void *arg)
597 {
598 struct apple_aic_softc *sc;
599 struct apple_aic_irqsrc *isrcs;
600 struct trapframe *tf;
601
602 sc = arg;
603 tf = curthread->td_intr_frame;
604
605 #ifdef SMP
606 /* Handle IPIs. */
607 if ((READ_SPECIALREG(AIC_IPI_SR_EL1) & AIC_IPI_SR_EL1_PENDING) != 0) {
608 WRITE_SPECIALREG(AIC_IPI_SR_EL1, AIC_IPI_SR_EL1_PENDING);
609 apple_aic_ipi_received(sc, tf);
610 }
611 #endif
612
613 /*
614 * FIQs don't store any state in the interrupt controller at all outside
615 * of IPI handling, so we have to probe around outside of AIC to
616 * determine if we might have been fired off due to a timer.
617 */
618 isrcs = sc->sc_isrcs[0];
619 if ((READ_SPECIALREG(cntv_ctl_el0) & CNTV_CTL_BITS) ==
620 (CNTV_CTL_ENABLE | CNTV_CTL_ISTATUS)) {
621 intr_isrc_dispatch(&isrcs[AIC_TMR_GUEST_VIRT].ai_isrc, tf);
622 }
623
624 if (has_hyp()) {
625 uint64_t reg;
626
627 if ((READ_SPECIALREG(cntp_ctl_el0) & CNTV_CTL_ISTATUS) != 0) {
628 intr_isrc_dispatch(&isrcs[AIC_TMR_GUEST_PHYS].ai_isrc,
629 tf);
630 }
631
632 reg = READ_SPECIALREG(AIC_FIQ_VM_TIMER);
633 if ((reg & AIC_FIQ_VM_TIMER_PEN) != 0) {
634 intr_isrc_dispatch(&isrcs[AIC_TMR_HV_PHYS].ai_isrc, tf);
635 }
636
637 if ((reg & AIC_FIQ_VM_TIMER_VEN) != 0) {
638 intr_isrc_dispatch(&isrcs[AIC_TMR_HV_VIRT].ai_isrc, tf);
639 }
640 }
641
642 return (FILTER_HANDLED);
643 }
644
645 #ifdef SMP
646 static int
apple_aic_bind_intr(device_t dev,struct intr_irqsrc * isrc)647 apple_aic_bind_intr(device_t dev, struct intr_irqsrc *isrc)
648 {
649 struct apple_aic_softc *sc = device_get_softc(dev);
650 static int aic_next_cpu;
651 uint32_t targets = 0;
652 u_int irq, cpu;
653
654 MPASS(((struct apple_aic_irqsrc *)isrc)->ai_type == AIC_TYPE_IRQ);
655 irq = ((struct apple_aic_irqsrc *)isrc)->ai_irq;
656 if (CPU_EMPTY(&isrc->isrc_cpu)) {
657 aic_next_cpu = intr_irq_next_cpu(aic_next_cpu, &all_cpus);
658 CPU_SETOF(aic_next_cpu, &isrc->isrc_cpu);
659 bus_write_4(sc->sc_mem, AIC_TARGET_CPU(irq),
660 sc->sc_cpuids[aic_next_cpu] << 1);
661 } else {
662 CPU_FOREACH_ISSET(cpu, &isrc->isrc_cpu) {
663 targets |= sc->sc_cpuids[cpu] << 1;
664 }
665 bus_write_4(sc->sc_mem, AIC_TARGET_CPU(irq), targets);
666 }
667 return (0);
668 }
669
670 static void
apple_aic_ipi_send(device_t dev,struct intr_irqsrc * isrc,cpuset_t cpus,u_int ipi)671 apple_aic_ipi_send(device_t dev, struct intr_irqsrc *isrc, cpuset_t cpus,
672 u_int ipi)
673 {
674 struct apple_aic_softc *sc;
675 uint64_t aff, localgrp, sendmask;
676 u_int cpu;
677
678 sc = device_get_softc(dev);
679 sendmask = 0;
680 localgrp = CPU_AFF1(CPU_AFFINITY(PCPU_GET(cpuid)));
681
682 KASSERT(isrc == &sc->sc_ipi_srcs[ipi].ai_isrc,
683 ("%s: bad ISRC %p argument", __func__, isrc));
684 for (cpu = 0; cpu <= mp_maxid; cpu++) {
685 if (CPU_ISSET(cpu, &cpus)) {
686 aff = CPU_AFFINITY(cpu);
687 sendmask = CPU_AFF0(aff);
688 atomic_set_32(&sc->sc_ipimasks[cpu], 1 << ipi);
689
690 /*
691 * The above write to sc_ipimasks needs to be visible
692 * before we write to the ipi register to avoid the
693 * targetted CPU missing the dispatch in
694 * apple_aic_ipi_received(). Note that WRITE_SPECIALREG
695 * isn't a memory operation, so we can't relax this to a
696 * a dmb.
697 */
698 dsb(ishst);
699
700 if (CPU_AFF1(aff) == localgrp) {
701 WRITE_SPECIALREG(AIC_IPI_LOCAL_RR_EL1,
702 sendmask);
703 } else {
704 sendmask |= CPU_AFF1(aff) << 16;
705 WRITE_SPECIALREG(AIC_IPI_GLOBAL_RR_EL1,
706 sendmask);
707 }
708
709 isb();
710 }
711 }
712 }
713
714 static int
apple_aic_ipi_setup(device_t dev,u_int ipi,struct intr_irqsrc ** isrcp)715 apple_aic_ipi_setup(device_t dev, u_int ipi, struct intr_irqsrc **isrcp)
716 {
717 struct apple_aic_softc *sc = device_get_softc(dev);
718 struct apple_aic_irqsrc *ai;
719
720 KASSERT(ipi < AIC_NIPIS, ("%s: ipi %u too high", __func__, ipi));
721
722 ai = &sc->sc_ipi_srcs[ipi];
723 ai->ai_type = AIC_TYPE_IPI;
724
725 *isrcp = &ai->ai_isrc;
726 return (0);
727 }
728
729 static void
apple_aic_init_secondary(device_t dev,uint32_t rootnum)730 apple_aic_init_secondary(device_t dev, uint32_t rootnum)
731 {
732 struct apple_aic_softc *sc = device_get_softc(dev);
733 u_int cpu = PCPU_GET(cpuid);
734
735 /* We don't need to re-initialize for the FIQ root. */
736 if (rootnum != INTR_ROOT_IRQ)
737 return;
738
739 sc->sc_cpuids[cpu] = bus_read_4(sc->sc_mem, AIC_WHOAMI);
740 if (bootverbose)
741 device_printf(dev, "CPU %d: whoami %x\n", cpu,
742 sc->sc_cpuids[cpu]);
743
744 bus_write_4(sc->sc_mem, AIC_IPI_MASK_SET, AIC_IPI_SELF | AIC_IPI_OTHER);
745 }
746 #endif
747
748 static device_method_t apple_aic_methods[] = {
749 /* Device interface */
750 DEVMETHOD(device_probe, apple_aic_probe),
751 DEVMETHOD(device_attach, apple_aic_attach),
752
753 /* Interrupt controller interface */
754 DEVMETHOD(pic_disable_intr, apple_aic_disable_intr),
755 DEVMETHOD(pic_enable_intr, apple_aic_enable_intr),
756 DEVMETHOD(pic_map_intr, apple_aic_map_intr),
757 DEVMETHOD(pic_setup_intr, apple_aic_setup_intr),
758 DEVMETHOD(pic_teardown_intr, apple_aic_teardown_intr),
759 DEVMETHOD(pic_post_filter, apple_aic_post_filter),
760 DEVMETHOD(pic_post_ithread, apple_aic_post_ithread),
761 DEVMETHOD(pic_pre_ithread, apple_aic_pre_ithread),
762 #ifdef SMP
763 DEVMETHOD(pic_bind_intr, apple_aic_bind_intr),
764 DEVMETHOD(pic_init_secondary, apple_aic_init_secondary),
765 DEVMETHOD(pic_ipi_send, apple_aic_ipi_send),
766 DEVMETHOD(pic_ipi_setup, apple_aic_ipi_setup),
767 #endif
768
769 /* End */
770 DEVMETHOD_END
771 };
772
773 static DEFINE_CLASS_0(aic, apple_aic_driver, apple_aic_methods,
774 sizeof(struct apple_aic_softc));
775
776 EARLY_DRIVER_MODULE(aic, simplebus, apple_aic_driver, 0, 0,
777 BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE);
778