1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 2011 The FreeBSD Foundation
5 * All rights reserved.
6 *
7 * Developed by Damjan Marion <damjan.marion@gmail.com>
8 *
9 * Based on OMAP4 GIC code by Ben Gray
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 * 3. The name of the company nor the name of the author may be used to
20 * endorse or promote products derived from this software without specific
21 * prior written permission.
22 *
23 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
24 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
27 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
28 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
29 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
31 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
32 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
33 * SUCH DAMAGE.
34 */
35
36 #include <sys/cdefs.h>
37 #include "opt_acpi.h"
38 #include "opt_ddb.h"
39 #include "opt_platform.h"
40
41 #include <sys/param.h>
42 #include <sys/systm.h>
43 #include <sys/bus.h>
44 #include <sys/kernel.h>
45 #include <sys/ktr.h>
46 #include <sys/module.h>
47 #include <sys/malloc.h>
48 #include <sys/rman.h>
49 #include <sys/pcpu.h>
50 #include <sys/proc.h>
51 #include <sys/cpuset.h>
52 #include <sys/lock.h>
53 #include <sys/mutex.h>
54 #include <sys/smp.h>
55 #include <sys/sched.h>
56
57 #include <vm/vm.h>
58 #include <vm/pmap.h>
59
60 #include <machine/bus.h>
61 #include <machine/intr.h>
62 #include <machine/smp.h>
63
64 #ifdef FDT
65 #include <dev/fdt/fdt_intr.h>
66 #include <dev/ofw/ofw_bus_subr.h>
67 #endif
68
69 #ifdef DEV_ACPI
70 #include <contrib/dev/acpica/include/acpi.h>
71 #include <dev/acpica/acpivar.h>
72 #endif
73
74 #ifdef DDB
75 #include <ddb/ddb.h>
76 #include <ddb/db_lex.h>
77 #endif
78
79 #include <arm/arm/gic.h>
80 #include <arm/arm/gic_common.h>
81
82 #include "gic_if.h"
83 #include "pic_if.h"
84 #include "msi_if.h"
85
86 /* We are using GICv2 register naming */
87
88 /* Distributor Registers */
89
90 /* CPU Registers */
91 #define GICC_CTLR 0x0000 /* v1 ICCICR */
92 #define GICC_PMR 0x0004 /* v1 ICCPMR */
93 #define GICC_BPR 0x0008 /* v1 ICCBPR */
94 #define GICC_IAR 0x000C /* v1 ICCIAR */
95 #define GICC_EOIR 0x0010 /* v1 ICCEOIR */
96 #define GICC_RPR 0x0014 /* v1 ICCRPR */
97 #define GICC_HPPIR 0x0018 /* v1 ICCHPIR */
98 #define GICC_ABPR 0x001C /* v1 ICCABPR */
99 #define GICC_IIDR 0x00FC /* v1 ICCIIDR*/
100
101 /* TYPER Registers */
102 #define GICD_TYPER_SECURITYEXT 0x400
103 #define GIC_SUPPORT_SECEXT(_sc) \
104 ((_sc->typer & GICD_TYPER_SECURITYEXT) == GICD_TYPER_SECURITYEXT)
105
106 #ifndef GIC_DEFAULT_ICFGR_INIT
107 #define GIC_DEFAULT_ICFGR_INIT 0x00000000
108 #endif
109
110 struct gic_irqsrc {
111 struct intr_irqsrc gi_isrc;
112 uint32_t gi_irq;
113 enum intr_polarity gi_pol;
114 enum intr_trigger gi_trig;
115 #define GI_FLAG_EARLY_EOI (1 << 0)
116 #define GI_FLAG_MSI (1 << 1) /* This interrupt source should only */
117 /* be used for MSI/MSI-X interrupts */
118 #define GI_FLAG_MSI_USED (1 << 2) /* This irq is already allocated */
119 /* for a MSI/MSI-X interrupt */
120 u_int gi_flags;
121 };
122
123 static u_int gic_irq_cpu;
124 static int arm_gic_bind_intr(device_t dev, struct intr_irqsrc *isrc);
125
126 #ifdef SMP
127 static u_int sgi_to_ipi[GIC_LAST_SGI - GIC_FIRST_SGI + 1];
128 static u_int sgi_first_unused = GIC_FIRST_SGI;
129 #endif
130
131 #define GIC_INTR_ISRC(sc, irq) (&sc->gic_irqs[irq].gi_isrc)
132
133 static struct resource_spec arm_gic_spec[] = {
134 { SYS_RES_MEMORY, 0, RF_ACTIVE }, /* Distributor registers */
135 { SYS_RES_MEMORY, 1, RF_ACTIVE }, /* CPU Interrupt Intf. registers */
136 { SYS_RES_IRQ, 0, RF_ACTIVE | RF_OPTIONAL }, /* Parent interrupt */
137 { -1, 0 }
138 };
139
140 #if defined(__arm__) && defined(INVARIANTS)
141 static int gic_debug_spurious = 1;
142 #else
143 static int gic_debug_spurious = 0;
144 #endif
145 TUNABLE_INT("hw.gic.debug_spurious", &gic_debug_spurious);
146
147 static u_int arm_gic_map[GIC_MAXCPU];
148
149 static struct arm_gic_softc *gic_sc = NULL;
150
151 /* CPU Interface */
152 #define gic_c_read_4(_sc, _reg) \
153 bus_read_4((_sc)->gic_res[GIC_RES_CPU], (_reg))
154 #define gic_c_peek_4(_sc, _reg, _val) \
155 bus_peek_4((_sc)->gic_res[GIC_RES_CPU], (_reg), (_val))
156 #define gic_c_write_4(_sc, _reg, _val) \
157 bus_write_4((_sc)->gic_res[GIC_RES_CPU], (_reg), (_val))
158 /* Distributor Interface */
159 #define gic_d_read_4(_sc, _reg) \
160 bus_read_4((_sc)->gic_res[GIC_RES_DIST], (_reg))
161 #define gic_d_write_1(_sc, _reg, _val) \
162 bus_write_1((_sc)->gic_res[GIC_RES_DIST], (_reg), (_val))
163 #define gic_d_write_4(_sc, _reg, _val) \
164 bus_write_4((_sc)->gic_res[GIC_RES_DIST], (_reg), (_val))
165
166 static inline void
gic_irq_unmask(struct arm_gic_softc * sc,u_int irq)167 gic_irq_unmask(struct arm_gic_softc *sc, u_int irq)
168 {
169
170 gic_d_write_4(sc, GICD_ISENABLER(irq), GICD_I_MASK(irq));
171 }
172
173 static inline void
gic_irq_mask(struct arm_gic_softc * sc,u_int irq)174 gic_irq_mask(struct arm_gic_softc *sc, u_int irq)
175 {
176
177 gic_d_write_4(sc, GICD_ICENABLER(irq), GICD_I_MASK(irq));
178 }
179
180 static uint8_t
gic_cpu_mask(struct arm_gic_softc * sc)181 gic_cpu_mask(struct arm_gic_softc *sc)
182 {
183 uint32_t mask;
184 int i;
185
186 /* Read the current cpuid mask by reading ITARGETSR{0..7} */
187 for (i = 0; i < 8; i++) {
188 mask = gic_d_read_4(sc, GICD_ITARGETSR(4 * i));
189 if (mask != 0)
190 break;
191 }
192 /* No mask found, assume we are on CPU interface 0 */
193 if (mask == 0)
194 return (1);
195
196 /* Collect the mask in the lower byte */
197 mask |= mask >> 16;
198 mask |= mask >> 8;
199
200 return (mask);
201 }
202
203 #ifdef SMP
204 static void
arm_gic_init_secondary(device_t dev,uint32_t rootnum)205 arm_gic_init_secondary(device_t dev, uint32_t rootnum)
206 {
207 struct arm_gic_softc *sc = device_get_softc(dev);
208 u_int irq, cpu;
209
210 /* Set the mask so we can find this CPU to send it IPIs */
211 cpu = PCPU_GET(cpuid);
212 MPASS(cpu < GIC_MAXCPU);
213 arm_gic_map[cpu] = gic_cpu_mask(sc);
214
215 for (irq = 0; irq < sc->nirqs; irq += 4)
216 gic_d_write_4(sc, GICD_IPRIORITYR(irq), 0);
217
218 /* Set all the interrupts to be in Group 0 (secure) */
219 for (irq = 0; GIC_SUPPORT_SECEXT(sc) && irq < sc->nirqs; irq += 32) {
220 gic_d_write_4(sc, GICD_IGROUPR(irq), 0);
221 }
222
223 /* Enable CPU interface */
224 gic_c_write_4(sc, GICC_CTLR, 1);
225
226 /* Set priority mask register. */
227 gic_c_write_4(sc, GICC_PMR, 0xff);
228
229 /* Enable interrupt distribution */
230 gic_d_write_4(sc, GICD_CTLR, 0x01);
231
232 /* Unmask attached SGI interrupts. */
233 for (irq = GIC_FIRST_SGI; irq <= GIC_LAST_SGI; irq++)
234 if (intr_isrc_init_on_cpu(GIC_INTR_ISRC(sc, irq), cpu))
235 gic_irq_unmask(sc, irq);
236
237 /* Unmask attached PPI interrupts. */
238 for (irq = GIC_FIRST_PPI; irq <= GIC_LAST_PPI; irq++)
239 if (intr_isrc_init_on_cpu(GIC_INTR_ISRC(sc, irq), cpu))
240 gic_irq_unmask(sc, irq);
241 }
242 #endif /* SMP */
243
244 static int
arm_gic_register_isrcs(struct arm_gic_softc * sc,uint32_t num)245 arm_gic_register_isrcs(struct arm_gic_softc *sc, uint32_t num)
246 {
247 int error;
248 uint32_t irq;
249 struct gic_irqsrc *irqs;
250 struct intr_irqsrc *isrc;
251 const char *name;
252
253 irqs = malloc(num * sizeof(struct gic_irqsrc), M_DEVBUF,
254 M_WAITOK | M_ZERO);
255
256 name = device_get_nameunit(sc->gic_dev);
257 for (irq = 0; irq < num; irq++) {
258 irqs[irq].gi_irq = irq;
259 irqs[irq].gi_pol = INTR_POLARITY_CONFORM;
260 irqs[irq].gi_trig = INTR_TRIGGER_CONFORM;
261
262 isrc = &irqs[irq].gi_isrc;
263 if (irq <= GIC_LAST_SGI) {
264 error = intr_isrc_register(isrc, sc->gic_dev,
265 INTR_ISRCF_IPI, "%s,i%u", name, irq - GIC_FIRST_SGI);
266 } else if (irq <= GIC_LAST_PPI) {
267 error = intr_isrc_register(isrc, sc->gic_dev,
268 INTR_ISRCF_PPI, "%s,p%u", name, irq - GIC_FIRST_PPI);
269 } else {
270 error = intr_isrc_register(isrc, sc->gic_dev, 0,
271 "%s,s%u", name, irq - GIC_FIRST_SPI);
272 }
273 if (error != 0) {
274 /* XXX call intr_isrc_deregister() */
275 free(irqs, M_DEVBUF);
276 return (error);
277 }
278 }
279 sc->gic_irqs = irqs;
280 sc->nirqs = num;
281 return (0);
282 }
283
284 static void
arm_gic_reserve_msi_range(device_t dev,u_int start,u_int count)285 arm_gic_reserve_msi_range(device_t dev, u_int start, u_int count)
286 {
287 struct arm_gic_softc *sc;
288 int i;
289
290 sc = device_get_softc(dev);
291
292 KASSERT((start + count) <= sc->nirqs,
293 ("%s: Trying to allocate too many MSI IRQs: %d + %d > %d", __func__,
294 start, count, sc->nirqs));
295 for (i = 0; i < count; i++) {
296 KASSERT(sc->gic_irqs[start + i].gi_isrc.isrc_handlers == 0,
297 ("%s: MSI interrupt %d already has a handler", __func__,
298 count + i));
299 KASSERT(sc->gic_irqs[start + i].gi_pol == INTR_POLARITY_CONFORM,
300 ("%s: MSI interrupt %d already has a polarity", __func__,
301 count + i));
302 KASSERT(sc->gic_irqs[start + i].gi_trig == INTR_TRIGGER_CONFORM,
303 ("%s: MSI interrupt %d already has a trigger", __func__,
304 count + i));
305 sc->gic_irqs[start + i].gi_pol = INTR_POLARITY_HIGH;
306 sc->gic_irqs[start + i].gi_trig = INTR_TRIGGER_EDGE;
307 sc->gic_irqs[start + i].gi_flags |= GI_FLAG_MSI;
308 }
309 }
310
311 int
arm_gic_attach(device_t dev)312 arm_gic_attach(device_t dev)
313 {
314 struct arm_gic_softc *sc;
315 int i;
316 uint32_t icciidr, mask, nirqs;
317
318 if (gic_sc)
319 return (ENXIO);
320
321 if (mp_ncpus > GIC_MAXCPU) {
322 device_printf(dev, "Too many CPUs for IPIs to work (%d > %d)\n",
323 mp_ncpus, GIC_MAXCPU);
324 return (ENXIO);
325 }
326
327 sc = device_get_softc(dev);
328
329 if (bus_alloc_resources(dev, arm_gic_spec, sc->gic_res)) {
330 device_printf(dev, "could not allocate resources\n");
331 return (ENXIO);
332 }
333
334 sc->gic_dev = dev;
335 gic_sc = sc;
336
337 /* Initialize mutex */
338 mtx_init(&sc->mutex, "GIC lock", NULL, MTX_SPIN);
339
340 /* Disable interrupt forwarding to the CPU interface */
341 gic_d_write_4(sc, GICD_CTLR, 0x00);
342
343 /* Get the number of interrupts */
344 sc->typer = gic_d_read_4(sc, GICD_TYPER);
345 nirqs = GICD_TYPER_I_NUM(sc->typer);
346
347 if (arm_gic_register_isrcs(sc, nirqs)) {
348 device_printf(dev, "could not register irqs\n");
349 goto cleanup;
350 }
351
352 /*
353 * Try accessing a CPU interface register. On some broken
354 * virtualization environments this will raise an external
355 * data abort. When this happens we can detect it using
356 * by peeking at the register & checking for the fault.
357 * As there is no way to continue with a normal boot we
358 * panic.
359 */
360 if (gic_c_peek_4(sc, GICC_IIDR, &icciidr) != 0)
361 panic("Unable to access %s CPU registers, "
362 "broken hardware or hypervisor configuration",
363 device_get_nameunit(dev));
364 device_printf(dev,
365 "pn 0x%x, arch 0x%x, rev 0x%x, implementer 0x%x irqs %u\n",
366 GICD_IIDR_PROD(icciidr), GICD_IIDR_VAR(icciidr),
367 GICD_IIDR_REV(icciidr), GICD_IIDR_IMPL(icciidr), sc->nirqs);
368 sc->gic_iidr = icciidr;
369
370 /* Set all global interrupts to be level triggered, active low. */
371 for (i = 32; i < sc->nirqs; i += 16) {
372 gic_d_write_4(sc, GICD_ICFGR(i), GIC_DEFAULT_ICFGR_INIT);
373 }
374
375 /* Disable all interrupts. */
376 for (i = 32; i < sc->nirqs; i += 32) {
377 gic_d_write_4(sc, GICD_ICENABLER(i), 0xFFFFFFFF);
378 }
379
380 /* Find the current cpu mask */
381 mask = gic_cpu_mask(sc);
382 /* Set the mask so we can find this CPU to send it IPIs */
383 MPASS(PCPU_GET(cpuid) < GIC_MAXCPU);
384 arm_gic_map[PCPU_GET(cpuid)] = mask;
385 /* Set all four targets to this cpu */
386 mask |= mask << 8;
387 mask |= mask << 16;
388
389 for (i = 0; i < sc->nirqs; i += 4) {
390 gic_d_write_4(sc, GICD_IPRIORITYR(i), 0);
391 if (i > 32) {
392 gic_d_write_4(sc, GICD_ITARGETSR(i), mask);
393 }
394 }
395
396 /* Set all the interrupts to be in Group 0 (secure) */
397 for (i = 0; GIC_SUPPORT_SECEXT(sc) && i < sc->nirqs; i += 32) {
398 gic_d_write_4(sc, GICD_IGROUPR(i), 0);
399 }
400
401 /* Enable CPU interface */
402 gic_c_write_4(sc, GICC_CTLR, 1);
403
404 /* Set priority mask register. */
405 gic_c_write_4(sc, GICC_PMR, 0xff);
406
407 /* Enable interrupt distribution */
408 gic_d_write_4(sc, GICD_CTLR, 0x01);
409 return (0);
410
411 cleanup:
412 arm_gic_detach(dev);
413 return(ENXIO);
414 }
415
416 int
arm_gic_detach(device_t dev)417 arm_gic_detach(device_t dev)
418 {
419 struct arm_gic_softc *sc;
420
421 sc = device_get_softc(dev);
422
423 if (sc->gic_irqs != NULL)
424 free(sc->gic_irqs, M_DEVBUF);
425
426 bus_release_resources(dev, arm_gic_spec, sc->gic_res);
427
428 return (0);
429 }
430
431 static int
arm_gic_print_child(device_t bus,device_t child)432 arm_gic_print_child(device_t bus, device_t child)
433 {
434 struct resource_list *rl;
435 int rv;
436
437 rv = bus_print_child_header(bus, child);
438
439 rl = BUS_GET_RESOURCE_LIST(bus, child);
440 if (rl != NULL) {
441 rv += resource_list_print_type(rl, "mem", SYS_RES_MEMORY,
442 "%#jx");
443 rv += resource_list_print_type(rl, "irq", SYS_RES_IRQ, "%jd");
444 }
445
446 rv += bus_print_child_footer(bus, child);
447
448 return (rv);
449 }
450
451 static struct resource *
arm_gic_alloc_resource(device_t bus,device_t child,int type,int rid,rman_res_t start,rman_res_t end,rman_res_t count,u_int flags)452 arm_gic_alloc_resource(device_t bus, device_t child, int type, int rid,
453 rman_res_t start, rman_res_t end, rman_res_t count, u_int flags)
454 {
455 struct arm_gic_softc *sc;
456 struct resource_list_entry *rle;
457 struct resource_list *rl;
458 int j;
459
460 KASSERT(type == SYS_RES_MEMORY, ("Invalid resource type %x", type));
461
462 sc = device_get_softc(bus);
463
464 /*
465 * Request for the default allocation with a given rid: use resource
466 * list stored in the local device info.
467 */
468 if (RMAN_IS_DEFAULT_RANGE(start, end)) {
469 rl = BUS_GET_RESOURCE_LIST(bus, child);
470
471 if (type == SYS_RES_IOPORT)
472 type = SYS_RES_MEMORY;
473
474 rle = resource_list_find(rl, type, rid);
475 if (rle == NULL) {
476 if (bootverbose)
477 device_printf(bus, "no default resources for "
478 "rid = %d, type = %d\n", rid, type);
479 return (NULL);
480 }
481 start = rle->start;
482 end = rle->end;
483 count = rle->count;
484 }
485
486 /* Remap through ranges property */
487 for (j = 0; j < sc->nranges; j++) {
488 if (start >= sc->ranges[j].bus && end <
489 sc->ranges[j].bus + sc->ranges[j].size) {
490 start -= sc->ranges[j].bus;
491 start += sc->ranges[j].host;
492 end -= sc->ranges[j].bus;
493 end += sc->ranges[j].host;
494 break;
495 }
496 }
497 if (j == sc->nranges && sc->nranges != 0) {
498 if (bootverbose)
499 device_printf(bus, "Could not map resource "
500 "%#jx-%#jx\n", (uintmax_t)start, (uintmax_t)end);
501
502 return (NULL);
503 }
504
505 return (bus_generic_alloc_resource(bus, child, type, rid, start, end,
506 count, flags));
507 }
508
509 static int
arm_gic_read_ivar(device_t dev,device_t child,int which,uintptr_t * result)510 arm_gic_read_ivar(device_t dev, device_t child, int which, uintptr_t *result)
511 {
512 struct arm_gic_softc *sc;
513
514 sc = device_get_softc(dev);
515
516 switch(which) {
517 case GIC_IVAR_HW_REV:
518 KASSERT(GICD_IIDR_VAR(sc->gic_iidr) < 3,
519 ("arm_gic_read_ivar: Unknown IIDR revision %u (%.08x)",
520 GICD_IIDR_VAR(sc->gic_iidr), sc->gic_iidr));
521 *result = GICD_IIDR_VAR(sc->gic_iidr);
522 return (0);
523 case GIC_IVAR_BUS:
524 KASSERT(sc->gic_bus != GIC_BUS_UNKNOWN,
525 ("arm_gic_read_ivar: Unknown bus type"));
526 KASSERT(sc->gic_bus <= GIC_BUS_MAX,
527 ("arm_gic_read_ivar: Invalid bus type %u", sc->gic_bus));
528 *result = sc->gic_bus;
529 return (0);
530 case GIC_IVAR_VGIC:
531 *result = 0;
532 return (0);
533 case GIC_IVAR_SUPPORT_LPIS:
534 *result = false;
535 return (0);
536 }
537
538 return (ENOENT);
539 }
540
541 static int
arm_gic_write_ivar(device_t dev,device_t child,int which,uintptr_t value)542 arm_gic_write_ivar(device_t dev, device_t child, int which, uintptr_t value)
543 {
544 switch(which) {
545 case GIC_IVAR_HW_REV:
546 case GIC_IVAR_BUS:
547 return (EINVAL);
548 }
549
550 return (ENOENT);
551 }
552
553 int
arm_gic_intr(void * arg)554 arm_gic_intr(void *arg)
555 {
556 struct arm_gic_softc *sc = arg;
557 struct gic_irqsrc *gi;
558 uint32_t irq_active_reg, irq;
559 struct trapframe *tf;
560
561 irq_active_reg = gic_c_read_4(sc, GICC_IAR);
562 irq = irq_active_reg & 0x3FF;
563
564 /*
565 * 1. We do EOI here because recent read value from active interrupt
566 * register must be used for it. Another approach is to save this
567 * value into associated interrupt source.
568 * 2. EOI must be done on same CPU where interrupt has fired. Thus
569 * we must ensure that interrupted thread does not migrate to
570 * another CPU.
571 * 3. EOI cannot be delayed by any preemption which could happen on
572 * critical_exit() used in MI intr code, when interrupt thread is
573 * scheduled. See next point.
574 * 4. IPI_RENDEZVOUS assumes that no preemption is permitted during
575 * an action and any use of critical_exit() could break this
576 * assumption. See comments within smp_rendezvous_action().
577 * 5. We always return FILTER_HANDLED as this is an interrupt
578 * controller dispatch function. Otherwise, in cascaded interrupt
579 * case, the whole interrupt subtree would be masked.
580 */
581
582 if (irq >= sc->nirqs) {
583 if (gic_debug_spurious)
584 device_printf(sc->gic_dev,
585 "Spurious interrupt detected: last irq: %d on CPU%d\n",
586 sc->last_irq[PCPU_GET(cpuid)], PCPU_GET(cpuid));
587 return (FILTER_HANDLED);
588 }
589
590 tf = curthread->td_intr_frame;
591 dispatch_irq:
592 gi = sc->gic_irqs + irq;
593 /*
594 * Note that GIC_FIRST_SGI is zero and is not used in 'if' statement
595 * as compiler complains that comparing u_int >= 0 is always true.
596 */
597 if (irq <= GIC_LAST_SGI) {
598 #ifdef SMP
599 /* Call EOI for all IPI before dispatch. */
600 gic_c_write_4(sc, GICC_EOIR, irq_active_reg);
601 intr_ipi_dispatch(sgi_to_ipi[gi->gi_irq]);
602 goto next_irq;
603 #else
604 device_printf(sc->gic_dev, "SGI %u on UP system detected\n",
605 irq - GIC_FIRST_SGI);
606 gic_c_write_4(sc, GICC_EOIR, irq_active_reg);
607 goto next_irq;
608 #endif
609 }
610
611 if (gic_debug_spurious)
612 sc->last_irq[PCPU_GET(cpuid)] = irq;
613 if ((gi->gi_flags & GI_FLAG_EARLY_EOI) == GI_FLAG_EARLY_EOI)
614 gic_c_write_4(sc, GICC_EOIR, irq_active_reg);
615
616 if (intr_isrc_dispatch(&gi->gi_isrc, tf) != 0) {
617 gic_irq_mask(sc, irq);
618 if ((gi->gi_flags & GI_FLAG_EARLY_EOI) != GI_FLAG_EARLY_EOI)
619 gic_c_write_4(sc, GICC_EOIR, irq_active_reg);
620 device_printf(sc->gic_dev, "Stray irq %u disabled\n", irq);
621 }
622
623 next_irq:
624 arm_irq_memory_barrier(irq);
625 irq_active_reg = gic_c_read_4(sc, GICC_IAR);
626 irq = irq_active_reg & 0x3FF;
627 if (irq < sc->nirqs)
628 goto dispatch_irq;
629
630 return (FILTER_HANDLED);
631 }
632
633 static void
gic_config(struct arm_gic_softc * sc,u_int irq,enum intr_trigger trig,enum intr_polarity pol)634 gic_config(struct arm_gic_softc *sc, u_int irq, enum intr_trigger trig,
635 enum intr_polarity pol)
636 {
637 uint32_t reg;
638 uint32_t mask;
639
640 if (irq < GIC_FIRST_SPI)
641 return;
642
643 mtx_lock_spin(&sc->mutex);
644
645 reg = gic_d_read_4(sc, GICD_ICFGR(irq));
646 mask = (reg >> 2*(irq % 16)) & 0x3;
647
648 if (pol == INTR_POLARITY_LOW) {
649 mask &= ~GICD_ICFGR_POL_MASK;
650 mask |= GICD_ICFGR_POL_LOW;
651 } else if (pol == INTR_POLARITY_HIGH) {
652 mask &= ~GICD_ICFGR_POL_MASK;
653 mask |= GICD_ICFGR_POL_HIGH;
654 }
655
656 if (trig == INTR_TRIGGER_LEVEL) {
657 mask &= ~GICD_ICFGR_TRIG_MASK;
658 mask |= GICD_ICFGR_TRIG_LVL;
659 } else if (trig == INTR_TRIGGER_EDGE) {
660 mask &= ~GICD_ICFGR_TRIG_MASK;
661 mask |= GICD_ICFGR_TRIG_EDGE;
662 }
663
664 /* Set mask */
665 reg = reg & ~(0x3 << 2*(irq % 16));
666 reg = reg | (mask << 2*(irq % 16));
667 gic_d_write_4(sc, GICD_ICFGR(irq), reg);
668
669 mtx_unlock_spin(&sc->mutex);
670 }
671
672 static int
gic_bind(struct arm_gic_softc * sc,u_int irq,cpuset_t * cpus)673 gic_bind(struct arm_gic_softc *sc, u_int irq, cpuset_t *cpus)
674 {
675 uint32_t cpu, end, mask;
676
677 end = min(mp_ncpus, GIC_MAXCPU);
678 for (cpu = end; cpu < MAXCPU; cpu++)
679 if (CPU_ISSET(cpu, cpus))
680 return (EINVAL);
681
682 for (mask = 0, cpu = 0; cpu < end; cpu++)
683 if (CPU_ISSET(cpu, cpus))
684 mask |= arm_gic_map[cpu];
685
686 gic_d_write_1(sc, GICD_ITARGETSR(0) + irq, mask);
687 return (0);
688 }
689
690 #ifdef FDT
691 static int
gic_map_fdt(device_t dev,u_int ncells,pcell_t * cells,u_int * irqp,enum intr_polarity * polp,enum intr_trigger * trigp)692 gic_map_fdt(device_t dev, u_int ncells, pcell_t *cells, u_int *irqp,
693 enum intr_polarity *polp, enum intr_trigger *trigp)
694 {
695
696 if (ncells == 1) {
697 *irqp = cells[0];
698 *polp = INTR_POLARITY_CONFORM;
699 *trigp = INTR_TRIGGER_CONFORM;
700 return (0);
701 }
702 if (ncells == 3) {
703 u_int irq, tripol;
704
705 /*
706 * The 1st cell is the interrupt type:
707 * 0 = SPI
708 * 1 = PPI
709 * The 2nd cell contains the interrupt number:
710 * [0 - 987] for SPI
711 * [0 - 15] for PPI
712 * The 3rd cell is the flags, encoded as follows:
713 * bits[3:0] trigger type and level flags
714 * 1 = low-to-high edge triggered
715 * 2 = high-to-low edge triggered
716 * 4 = active high level-sensitive
717 * 8 = active low level-sensitive
718 * bits[15:8] PPI interrupt cpu mask
719 * Each bit corresponds to each of the 8 possible cpus
720 * attached to the GIC. A bit set to '1' indicated
721 * the interrupt is wired to that CPU.
722 */
723 switch (cells[0]) {
724 case 0:
725 irq = GIC_FIRST_SPI + cells[1];
726 /* SPI irq is checked later. */
727 break;
728 case 1:
729 irq = GIC_FIRST_PPI + cells[1];
730 if (irq > GIC_LAST_PPI) {
731 device_printf(dev, "unsupported PPI interrupt "
732 "number %u\n", cells[1]);
733 return (EINVAL);
734 }
735 break;
736 default:
737 device_printf(dev, "unsupported interrupt type "
738 "configuration %u\n", cells[0]);
739 return (EINVAL);
740 }
741
742 tripol = cells[2] & 0xff;
743 if (tripol & 0xf0 || (tripol & FDT_INTR_LOW_MASK &&
744 cells[0] == 0))
745 device_printf(dev, "unsupported trigger/polarity "
746 "configuration 0x%02x\n", tripol);
747
748 *irqp = irq;
749 *polp = INTR_POLARITY_CONFORM;
750 *trigp = tripol & FDT_INTR_EDGE_MASK ?
751 INTR_TRIGGER_EDGE : INTR_TRIGGER_LEVEL;
752 return (0);
753 }
754 return (EINVAL);
755 }
756 #endif
757
758 static int
gic_map_msi(device_t dev,struct intr_map_data_msi * msi_data,u_int * irqp,enum intr_polarity * polp,enum intr_trigger * trigp)759 gic_map_msi(device_t dev, struct intr_map_data_msi *msi_data, u_int *irqp,
760 enum intr_polarity *polp, enum intr_trigger *trigp)
761 {
762 struct gic_irqsrc *gi;
763
764 /* Map a non-GICv2m MSI */
765 gi = (struct gic_irqsrc *)msi_data->isrc;
766 if (gi == NULL)
767 return (ENXIO);
768
769 *irqp = gi->gi_irq;
770
771 /* MSI/MSI-X interrupts are always edge triggered with high polarity */
772 *polp = INTR_POLARITY_HIGH;
773 *trigp = INTR_TRIGGER_EDGE;
774
775 return (0);
776 }
777
778 static int
gic_map_intr(device_t dev,struct intr_map_data * data,u_int * irqp,enum intr_polarity * polp,enum intr_trigger * trigp)779 gic_map_intr(device_t dev, struct intr_map_data *data, u_int *irqp,
780 enum intr_polarity *polp, enum intr_trigger *trigp)
781 {
782 u_int irq;
783 enum intr_polarity pol;
784 enum intr_trigger trig;
785 struct arm_gic_softc *sc;
786 struct intr_map_data_msi *dam;
787 #ifdef FDT
788 struct intr_map_data_fdt *daf;
789 #endif
790 #ifdef DEV_ACPI
791 struct intr_map_data_acpi *daa;
792 #endif
793
794 sc = device_get_softc(dev);
795 switch (data->type) {
796 #ifdef FDT
797 case INTR_MAP_DATA_FDT:
798 daf = (struct intr_map_data_fdt *)data;
799 if (gic_map_fdt(dev, daf->ncells, daf->cells, &irq, &pol,
800 &trig) != 0)
801 return (EINVAL);
802 KASSERT(irq >= sc->nirqs ||
803 (sc->gic_irqs[irq].gi_flags & GI_FLAG_MSI) == 0,
804 ("%s: Attempting to map a MSI interrupt from FDT",
805 __func__));
806 break;
807 #endif
808 #ifdef DEV_ACPI
809 case INTR_MAP_DATA_ACPI:
810 daa = (struct intr_map_data_acpi *)data;
811 irq = daa->irq;
812 pol = daa->pol;
813 trig = daa->trig;
814 break;
815 #endif
816 case INTR_MAP_DATA_MSI:
817 /* Non-GICv2m MSI */
818 dam = (struct intr_map_data_msi *)data;
819 if (gic_map_msi(dev, dam, &irq, &pol, &trig) != 0)
820 return (EINVAL);
821 break;
822 default:
823 return (ENOTSUP);
824 }
825
826 if (irq >= sc->nirqs)
827 return (EINVAL);
828 if (pol != INTR_POLARITY_CONFORM && pol != INTR_POLARITY_LOW &&
829 pol != INTR_POLARITY_HIGH)
830 return (EINVAL);
831 if (trig != INTR_TRIGGER_CONFORM && trig != INTR_TRIGGER_EDGE &&
832 trig != INTR_TRIGGER_LEVEL)
833 return (EINVAL);
834
835 *irqp = irq;
836 if (polp != NULL)
837 *polp = pol;
838 if (trigp != NULL)
839 *trigp = trig;
840 return (0);
841 }
842
843 static int
arm_gic_map_intr(device_t dev,struct intr_map_data * data,struct intr_irqsrc ** isrcp)844 arm_gic_map_intr(device_t dev, struct intr_map_data *data,
845 struct intr_irqsrc **isrcp)
846 {
847 int error;
848 u_int irq;
849 struct arm_gic_softc *sc;
850
851 error = gic_map_intr(dev, data, &irq, NULL, NULL);
852 if (error == 0) {
853 sc = device_get_softc(dev);
854 *isrcp = GIC_INTR_ISRC(sc, irq);
855 }
856 return (error);
857 }
858
859 static int
arm_gic_setup_intr(device_t dev,struct intr_irqsrc * isrc,struct resource * res,struct intr_map_data * data)860 arm_gic_setup_intr(device_t dev, struct intr_irqsrc *isrc,
861 struct resource *res, struct intr_map_data *data)
862 {
863 struct arm_gic_softc *sc = device_get_softc(dev);
864 struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc;
865 enum intr_trigger trig;
866 enum intr_polarity pol;
867
868 if ((gi->gi_flags & GI_FLAG_MSI) == GI_FLAG_MSI) {
869 /* GICv2m MSI */
870 pol = gi->gi_pol;
871 trig = gi->gi_trig;
872 KASSERT(pol == INTR_POLARITY_HIGH,
873 ("%s: MSI interrupts must be active-high", __func__));
874 KASSERT(trig == INTR_TRIGGER_EDGE,
875 ("%s: MSI interrupts must be edge triggered", __func__));
876 } else if (data != NULL) {
877 u_int irq;
878
879 /* Get config for resource. */
880 if (gic_map_intr(dev, data, &irq, &pol, &trig) ||
881 gi->gi_irq != irq)
882 return (EINVAL);
883 } else {
884 pol = INTR_POLARITY_CONFORM;
885 trig = INTR_TRIGGER_CONFORM;
886 }
887
888 /* Compare config if this is not first setup. */
889 if (isrc->isrc_handlers != 0) {
890 if ((pol != INTR_POLARITY_CONFORM && pol != gi->gi_pol) ||
891 (trig != INTR_TRIGGER_CONFORM && trig != gi->gi_trig))
892 return (EINVAL);
893 else
894 return (0);
895 }
896
897 /* For MSI/MSI-X we should have already configured these */
898 if ((gi->gi_flags & GI_FLAG_MSI) == 0) {
899 if (pol == INTR_POLARITY_CONFORM)
900 pol = INTR_POLARITY_LOW; /* just pick some */
901 if (trig == INTR_TRIGGER_CONFORM)
902 trig = INTR_TRIGGER_EDGE; /* just pick some */
903
904 gi->gi_pol = pol;
905 gi->gi_trig = trig;
906
907 /* Edge triggered interrupts need an early EOI sent */
908 if (gi->gi_trig == INTR_TRIGGER_EDGE)
909 gi->gi_flags |= GI_FLAG_EARLY_EOI;
910 }
911
912 /*
913 * XXX - In case that per CPU interrupt is going to be enabled in time
914 * when SMP is already started, we need some IPI call which
915 * enables it on others CPUs. Further, it's more complicated as
916 * pic_enable_source() and pic_disable_source() should act on
917 * per CPU basis only. Thus, it should be solved here somehow.
918 */
919 if (isrc->isrc_flags & INTR_ISRCF_PPI)
920 CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu);
921
922 gic_config(sc, gi->gi_irq, gi->gi_trig, gi->gi_pol);
923 arm_gic_bind_intr(dev, isrc);
924 return (0);
925 }
926
927 static int
arm_gic_teardown_intr(device_t dev,struct intr_irqsrc * isrc,struct resource * res,struct intr_map_data * data)928 arm_gic_teardown_intr(device_t dev, struct intr_irqsrc *isrc,
929 struct resource *res, struct intr_map_data *data)
930 {
931 struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc;
932
933 if (isrc->isrc_handlers == 0 && (gi->gi_flags & GI_FLAG_MSI) == 0) {
934 gi->gi_pol = INTR_POLARITY_CONFORM;
935 gi->gi_trig = INTR_TRIGGER_CONFORM;
936 }
937 return (0);
938 }
939
940 static void
arm_gic_enable_intr(device_t dev,struct intr_irqsrc * isrc)941 arm_gic_enable_intr(device_t dev, struct intr_irqsrc *isrc)
942 {
943 struct arm_gic_softc *sc = device_get_softc(dev);
944 struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc;
945
946 arm_irq_memory_barrier(gi->gi_irq);
947 gic_irq_unmask(sc, gi->gi_irq);
948 }
949
950 static void
arm_gic_disable_intr(device_t dev,struct intr_irqsrc * isrc)951 arm_gic_disable_intr(device_t dev, struct intr_irqsrc *isrc)
952 {
953 struct arm_gic_softc *sc = device_get_softc(dev);
954 struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc;
955
956 gic_irq_mask(sc, gi->gi_irq);
957 }
958
959 static void
arm_gic_pre_ithread(device_t dev,struct intr_irqsrc * isrc)960 arm_gic_pre_ithread(device_t dev, struct intr_irqsrc *isrc)
961 {
962 struct arm_gic_softc *sc = device_get_softc(dev);
963 struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc;
964
965 arm_gic_disable_intr(dev, isrc);
966 gic_c_write_4(sc, GICC_EOIR, gi->gi_irq);
967 }
968
969 static void
arm_gic_post_ithread(device_t dev,struct intr_irqsrc * isrc)970 arm_gic_post_ithread(device_t dev, struct intr_irqsrc *isrc)
971 {
972
973 arm_irq_memory_barrier(0);
974 arm_gic_enable_intr(dev, isrc);
975 }
976
977 static void
arm_gic_post_filter(device_t dev,struct intr_irqsrc * isrc)978 arm_gic_post_filter(device_t dev, struct intr_irqsrc *isrc)
979 {
980 struct arm_gic_softc *sc = device_get_softc(dev);
981 struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc;
982
983 /* EOI for edge-triggered done earlier. */
984 if ((gi->gi_flags & GI_FLAG_EARLY_EOI) == GI_FLAG_EARLY_EOI)
985 return;
986
987 arm_irq_memory_barrier(0);
988 gic_c_write_4(sc, GICC_EOIR, gi->gi_irq);
989 }
990
991 static int
arm_gic_bind_intr(device_t dev,struct intr_irqsrc * isrc)992 arm_gic_bind_intr(device_t dev, struct intr_irqsrc *isrc)
993 {
994 struct arm_gic_softc *sc = device_get_softc(dev);
995 struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc;
996
997 if (gi->gi_irq < GIC_FIRST_SPI)
998 return (EINVAL);
999
1000 if (CPU_EMPTY(&isrc->isrc_cpu)) {
1001 gic_irq_cpu = intr_irq_next_cpu(gic_irq_cpu, &all_cpus);
1002 CPU_SETOF(gic_irq_cpu, &isrc->isrc_cpu);
1003 }
1004 return (gic_bind(sc, gi->gi_irq, &isrc->isrc_cpu));
1005 }
1006
1007 #ifdef SMP
1008 static void
arm_gic_ipi_send(device_t dev,struct intr_irqsrc * isrc,cpuset_t cpus,u_int ipi)1009 arm_gic_ipi_send(device_t dev, struct intr_irqsrc *isrc, cpuset_t cpus,
1010 u_int ipi)
1011 {
1012 struct arm_gic_softc *sc = device_get_softc(dev);
1013 struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc;
1014 uint32_t val = 0, i;
1015
1016 for (i = 0; i < MAXCPU; i++) {
1017 if (CPU_ISSET(i, &cpus)) {
1018 MPASS(i < GIC_MAXCPU);
1019 val |= arm_gic_map[i] << GICD_SGI_TARGET_SHIFT;
1020 }
1021 }
1022
1023 gic_d_write_4(sc, GICD_SGIR, val | gi->gi_irq);
1024 }
1025
1026 static int
arm_gic_ipi_setup(device_t dev,u_int ipi,struct intr_irqsrc ** isrcp)1027 arm_gic_ipi_setup(device_t dev, u_int ipi, struct intr_irqsrc **isrcp)
1028 {
1029 struct intr_irqsrc *isrc;
1030 struct arm_gic_softc *sc = device_get_softc(dev);
1031
1032 if (sgi_first_unused > GIC_LAST_SGI)
1033 return (ENOSPC);
1034
1035 isrc = GIC_INTR_ISRC(sc, sgi_first_unused);
1036 sgi_to_ipi[sgi_first_unused++] = ipi;
1037
1038 CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu);
1039
1040 *isrcp = isrc;
1041 return (0);
1042 }
1043 #endif
1044
1045 static int
arm_gic_alloc_msi(device_t dev,u_int mbi_start,u_int mbi_count,int count,int maxcount,struct intr_irqsrc ** isrc)1046 arm_gic_alloc_msi(device_t dev, u_int mbi_start, u_int mbi_count, int count,
1047 int maxcount, struct intr_irqsrc **isrc)
1048 {
1049 struct arm_gic_softc *sc;
1050 int i, irq, end_irq;
1051 bool found;
1052
1053 KASSERT(powerof2(count), ("%s: bad count", __func__));
1054 KASSERT(powerof2(maxcount), ("%s: bad maxcount", __func__));
1055
1056 sc = device_get_softc(dev);
1057
1058 mtx_lock_spin(&sc->mutex);
1059
1060 found = false;
1061 for (irq = mbi_start; irq < mbi_start + mbi_count; irq++) {
1062 /* Start on an aligned interrupt */
1063 if ((irq & (maxcount - 1)) != 0)
1064 continue;
1065
1066 /* Assume we found a valid range until shown otherwise */
1067 found = true;
1068
1069 /* Check this range is valid */
1070 for (end_irq = irq; end_irq != irq + count; end_irq++) {
1071 /* No free interrupts */
1072 if (end_irq == mbi_start + mbi_count) {
1073 found = false;
1074 break;
1075 }
1076
1077 KASSERT((sc->gic_irqs[end_irq].gi_flags & GI_FLAG_MSI)!= 0,
1078 ("%s: Non-MSI interrupt found", __func__));
1079
1080 /* This is already used */
1081 if ((sc->gic_irqs[end_irq].gi_flags & GI_FLAG_MSI_USED) ==
1082 GI_FLAG_MSI_USED) {
1083 found = false;
1084 break;
1085 }
1086 }
1087 if (found)
1088 break;
1089 }
1090
1091 /* Not enough interrupts were found */
1092 if (!found || irq == mbi_start + mbi_count) {
1093 mtx_unlock_spin(&sc->mutex);
1094 return (ENXIO);
1095 }
1096
1097 for (i = 0; i < count; i++) {
1098 /* Mark the interrupt as used */
1099 sc->gic_irqs[irq + i].gi_flags |= GI_FLAG_MSI_USED;
1100 }
1101 mtx_unlock_spin(&sc->mutex);
1102
1103 for (i = 0; i < count; i++)
1104 isrc[i] = (struct intr_irqsrc *)&sc->gic_irqs[irq + i];
1105
1106 return (0);
1107 }
1108
1109 static int
arm_gic_release_msi(device_t dev,int count,struct intr_irqsrc ** isrc)1110 arm_gic_release_msi(device_t dev, int count, struct intr_irqsrc **isrc)
1111 {
1112 struct arm_gic_softc *sc;
1113 struct gic_irqsrc *gi;
1114 int i;
1115
1116 sc = device_get_softc(dev);
1117
1118 mtx_lock_spin(&sc->mutex);
1119 for (i = 0; i < count; i++) {
1120 gi = (struct gic_irqsrc *)isrc[i];
1121
1122 KASSERT((gi->gi_flags & GI_FLAG_MSI_USED) == GI_FLAG_MSI_USED,
1123 ("%s: Trying to release an unused MSI-X interrupt",
1124 __func__));
1125
1126 gi->gi_flags &= ~GI_FLAG_MSI_USED;
1127 }
1128 mtx_unlock_spin(&sc->mutex);
1129
1130 return (0);
1131 }
1132
1133 static int
arm_gic_alloc_msix(device_t dev,u_int mbi_start,u_int mbi_count,struct intr_irqsrc ** isrc)1134 arm_gic_alloc_msix(device_t dev, u_int mbi_start, u_int mbi_count,
1135 struct intr_irqsrc **isrc)
1136 {
1137 struct arm_gic_softc *sc;
1138 int irq;
1139
1140 sc = device_get_softc(dev);
1141
1142 mtx_lock_spin(&sc->mutex);
1143 /* Find an unused interrupt */
1144 for (irq = mbi_start; irq < mbi_start + mbi_count; irq++) {
1145 KASSERT((sc->gic_irqs[irq].gi_flags & GI_FLAG_MSI) != 0,
1146 ("%s: Non-MSI interrupt found", __func__));
1147 if ((sc->gic_irqs[irq].gi_flags & GI_FLAG_MSI_USED) == 0)
1148 break;
1149 }
1150 /* No free interrupt was found */
1151 if (irq == mbi_start + mbi_count) {
1152 mtx_unlock_spin(&sc->mutex);
1153 return (ENXIO);
1154 }
1155
1156 /* Mark the interrupt as used */
1157 sc->gic_irqs[irq].gi_flags |= GI_FLAG_MSI_USED;
1158 mtx_unlock_spin(&sc->mutex);
1159
1160 *isrc = (struct intr_irqsrc *)&sc->gic_irqs[irq];
1161
1162 return (0);
1163 }
1164
1165 static int
arm_gic_release_msix(device_t dev,struct intr_irqsrc * isrc)1166 arm_gic_release_msix(device_t dev, struct intr_irqsrc *isrc)
1167 {
1168 struct arm_gic_softc *sc;
1169 struct gic_irqsrc *gi;
1170
1171 sc = device_get_softc(dev);
1172 gi = (struct gic_irqsrc *)isrc;
1173
1174 KASSERT((gi->gi_flags & GI_FLAG_MSI_USED) == GI_FLAG_MSI_USED,
1175 ("%s: Trying to release an unused MSI-X interrupt", __func__));
1176
1177 mtx_lock_spin(&sc->mutex);
1178 gi->gi_flags &= ~GI_FLAG_MSI_USED;
1179 mtx_unlock_spin(&sc->mutex);
1180
1181 return (0);
1182 }
1183
1184 #ifdef DDB
1185 static void
arm_gic_db_show(device_t dev)1186 arm_gic_db_show(device_t dev)
1187 {
1188 struct arm_gic_softc *sc = device_get_softc(dev);
1189 uint32_t val;
1190 u_int i;
1191
1192 db_printf("%s CPU registers:\n", device_get_nameunit(dev));
1193 db_printf(" CTLR: %08x PMR: %08x BPR: %08x RPR: %08x\n",
1194 gic_c_read_4(sc, GICC_CTLR), gic_c_read_4(sc, GICC_PMR),
1195 gic_c_read_4(sc, GICC_BPR), gic_c_read_4(sc, GICC_RPR));
1196 db_printf("HPPIR: %08x IIDR: %08x\n", gic_c_read_4(sc, GICC_HPPIR),
1197 gic_c_read_4(sc, GICC_IIDR));
1198
1199 db_printf("%s Distributor registers:\n", device_get_nameunit(dev));
1200 db_printf(" CTLR: %08x TYPER: %08x IIDR: %08x\n",
1201 gic_d_read_4(sc, GICD_CTLR), gic_d_read_4(sc, GICD_TYPER),
1202 gic_d_read_4(sc, GICD_IIDR));
1203 for (i = 0; i < sc->nirqs; i++) {
1204 if (i <= GIC_LAST_SGI)
1205 db_printf("SGI %2u ", i);
1206 else if (i <= GIC_LAST_PPI)
1207 db_printf("PPI %2u ", i - GIC_FIRST_PPI);
1208 else
1209 db_printf("SPI %2u ", i - GIC_FIRST_SPI);
1210 db_printf(" grp:%u",
1211 !!(gic_d_read_4(sc, GICD_IGROUPR(i)) & GICD_I_MASK(i)));
1212 db_printf(" enable:%u pend:%u active:%u",
1213 !!(gic_d_read_4(sc, GICD_ISENABLER(i)) & GICD_I_MASK(i)),
1214 !!(gic_d_read_4(sc, GICD_ISPENDR(i)) & GICD_I_MASK(i)),
1215 !!(gic_d_read_4(sc, GICD_ISACTIVER(i)) & GICD_I_MASK(i)));
1216 db_printf(" pri:%u",
1217 (gic_d_read_4(sc, GICD_IPRIORITYR(i)) >> 8 * (i & 0x3)) &
1218 0xff);
1219 db_printf(" trg:%u",
1220 (gic_d_read_4(sc, GICD_ITARGETSR(i)) >> 8 * (i & 0x3)) &
1221 0xff);
1222 val = gic_d_read_4(sc, GICD_ICFGR(i)) >> 2 * (i & 0xf);
1223 if ((val & GICD_ICFGR_POL_MASK) == GICD_ICFGR_POL_LOW)
1224 db_printf(" LO");
1225 else
1226 db_printf(" HI");
1227 if ((val & GICD_ICFGR_TRIG_MASK) == GICD_ICFGR_TRIG_LVL)
1228 db_printf(" LV");
1229 else
1230 db_printf(" ED");
1231 db_printf("\n");
1232 }
1233 }
1234 #endif
1235
1236 static device_method_t arm_gic_methods[] = {
1237 /* Bus interface */
1238 DEVMETHOD(bus_print_child, arm_gic_print_child),
1239 DEVMETHOD(bus_add_child, bus_generic_add_child),
1240 DEVMETHOD(bus_alloc_resource, arm_gic_alloc_resource),
1241 DEVMETHOD(bus_release_resource, bus_generic_release_resource),
1242 DEVMETHOD(bus_activate_resource,bus_generic_activate_resource),
1243 DEVMETHOD(bus_read_ivar, arm_gic_read_ivar),
1244 DEVMETHOD(bus_write_ivar, arm_gic_write_ivar),
1245
1246 /* Interrupt controller interface */
1247 DEVMETHOD(pic_disable_intr, arm_gic_disable_intr),
1248 DEVMETHOD(pic_enable_intr, arm_gic_enable_intr),
1249 DEVMETHOD(pic_map_intr, arm_gic_map_intr),
1250 DEVMETHOD(pic_setup_intr, arm_gic_setup_intr),
1251 DEVMETHOD(pic_teardown_intr, arm_gic_teardown_intr),
1252 DEVMETHOD(pic_post_filter, arm_gic_post_filter),
1253 DEVMETHOD(pic_post_ithread, arm_gic_post_ithread),
1254 DEVMETHOD(pic_pre_ithread, arm_gic_pre_ithread),
1255 #ifdef SMP
1256 DEVMETHOD(pic_bind_intr, arm_gic_bind_intr),
1257 DEVMETHOD(pic_init_secondary, arm_gic_init_secondary),
1258 DEVMETHOD(pic_ipi_send, arm_gic_ipi_send),
1259 DEVMETHOD(pic_ipi_setup, arm_gic_ipi_setup),
1260 #endif
1261
1262 /* GIC */
1263 DEVMETHOD(gic_reserve_msi_range, arm_gic_reserve_msi_range),
1264 DEVMETHOD(gic_alloc_msi, arm_gic_alloc_msi),
1265 DEVMETHOD(gic_release_msi, arm_gic_release_msi),
1266 DEVMETHOD(gic_alloc_msix, arm_gic_alloc_msix),
1267 DEVMETHOD(gic_release_msix, arm_gic_release_msix),
1268 #ifdef DDB
1269 DEVMETHOD(gic_db_show, arm_gic_db_show),
1270 #endif
1271
1272 { 0, 0 }
1273 };
1274
1275 DEFINE_CLASS_0(gic, arm_gic_driver, arm_gic_methods,
1276 sizeof(struct arm_gic_softc));
1277
1278 #ifdef DDB
DB_SHOW_COMMAND_FLAGS(gic,db_show_gic,CS_OWN)1279 DB_SHOW_COMMAND_FLAGS(gic, db_show_gic, CS_OWN)
1280 {
1281 device_t dev;
1282 int t;
1283 bool valid;
1284
1285 valid = false;
1286 t = db_read_token();
1287 if (t == tIDENT) {
1288 dev = device_lookup_by_name(db_tok_string);
1289 valid = true;
1290 }
1291 db_skip_to_eol();
1292 if (!valid) {
1293 db_printf("usage: show gic <name>\n");
1294 return;
1295 }
1296
1297 if (dev == NULL) {
1298 db_printf("device not found\n");
1299 return;
1300 }
1301
1302 GIC_DB_SHOW(dev);
1303 }
1304
DB_SHOW_ALL_COMMAND(gics,db_show_all_gics)1305 DB_SHOW_ALL_COMMAND(gics, db_show_all_gics)
1306 {
1307 devclass_t dc;
1308 device_t dev;
1309 int i;
1310
1311 dc = devclass_find("gic");
1312 if (dc == NULL)
1313 return;
1314
1315 for (i = 0; i < devclass_get_maxunit(dc); i++) {
1316 dev = devclass_get_device(dc, i);
1317 if (dev != NULL)
1318 GIC_DB_SHOW(dev);
1319 if (db_pager_quit)
1320 break;
1321 }
1322 }
1323
1324 #endif
1325
1326 /*
1327 * GICv2m support -- the GICv2 MSI/MSI-X controller.
1328 */
1329
1330 #define GICV2M_MSI_TYPER 0x008
1331 #define MSI_TYPER_SPI_BASE(x) (((x) >> 16) & 0x3ff)
1332 #define MSI_TYPER_SPI_COUNT(x) (((x) >> 0) & 0x3ff)
1333 #define GICv2M_MSI_SETSPI_NS 0x040
1334 #define GICV2M_MSI_IIDR 0xFCC
1335
1336 int
arm_gicv2m_attach(device_t dev)1337 arm_gicv2m_attach(device_t dev)
1338 {
1339 struct arm_gicv2m_softc *sc;
1340 uint32_t typer;
1341 int rid;
1342
1343 sc = device_get_softc(dev);
1344
1345 rid = 0;
1346 sc->sc_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
1347 RF_ACTIVE);
1348 if (sc->sc_mem == NULL) {
1349 device_printf(dev, "Unable to allocate resources\n");
1350 return (ENXIO);
1351 }
1352
1353 typer = bus_read_4(sc->sc_mem, GICV2M_MSI_TYPER);
1354 sc->sc_spi_start = MSI_TYPER_SPI_BASE(typer);
1355 sc->sc_spi_count = MSI_TYPER_SPI_COUNT(typer);
1356
1357 /* Reserve these interrupts for MSI/MSI-X use */
1358 GIC_RESERVE_MSI_RANGE(device_get_parent(dev), sc->sc_spi_start,
1359 sc->sc_spi_count);
1360
1361 intr_msi_register(dev, sc->sc_xref);
1362
1363 if (bootverbose)
1364 device_printf(dev, "using spi %u to %u\n", sc->sc_spi_start,
1365 sc->sc_spi_start + sc->sc_spi_count - 1);
1366
1367 return (0);
1368 }
1369
1370 static int
arm_gicv2m_alloc_msi(device_t dev,device_t child,int count,int maxcount,device_t * pic,struct intr_irqsrc ** srcs)1371 arm_gicv2m_alloc_msi(device_t dev, device_t child, int count, int maxcount,
1372 device_t *pic, struct intr_irqsrc **srcs)
1373 {
1374 struct arm_gicv2m_softc *sc;
1375 int error;
1376
1377 sc = device_get_softc(dev);
1378 error = GIC_ALLOC_MSI(device_get_parent(dev), sc->sc_spi_start,
1379 sc->sc_spi_count, count, maxcount, srcs);
1380 if (error != 0)
1381 return (error);
1382
1383 *pic = dev;
1384 return (0);
1385 }
1386
1387 static int
arm_gicv2m_release_msi(device_t dev,device_t child,int count,struct intr_irqsrc ** isrc)1388 arm_gicv2m_release_msi(device_t dev, device_t child, int count,
1389 struct intr_irqsrc **isrc)
1390 {
1391 return (GIC_RELEASE_MSI(device_get_parent(dev), count, isrc));
1392 }
1393
1394 static int
arm_gicv2m_alloc_msix(device_t dev,device_t child,device_t * pic,struct intr_irqsrc ** isrcp)1395 arm_gicv2m_alloc_msix(device_t dev, device_t child, device_t *pic,
1396 struct intr_irqsrc **isrcp)
1397 {
1398 struct arm_gicv2m_softc *sc;
1399 int error;
1400
1401 sc = device_get_softc(dev);
1402 error = GIC_ALLOC_MSIX(device_get_parent(dev), sc->sc_spi_start,
1403 sc->sc_spi_count, isrcp);
1404 if (error != 0)
1405 return (error);
1406
1407 *pic = dev;
1408 return (0);
1409 }
1410
1411 static int
arm_gicv2m_release_msix(device_t dev,device_t child,struct intr_irqsrc * isrc)1412 arm_gicv2m_release_msix(device_t dev, device_t child, struct intr_irqsrc *isrc)
1413 {
1414 return (GIC_RELEASE_MSIX(device_get_parent(dev), isrc));
1415 }
1416
1417 static int
arm_gicv2m_map_msi(device_t dev,device_t child,struct intr_irqsrc * isrc,uint64_t * addr,uint32_t * data)1418 arm_gicv2m_map_msi(device_t dev, device_t child, struct intr_irqsrc *isrc,
1419 uint64_t *addr, uint32_t *data)
1420 {
1421 struct arm_gicv2m_softc *sc = device_get_softc(dev);
1422 struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc;
1423
1424 *addr = vtophys(rman_get_virtual(sc->sc_mem)) + GICv2M_MSI_SETSPI_NS;
1425 *data = gi->gi_irq;
1426
1427 return (0);
1428 }
1429
1430 static device_method_t arm_gicv2m_methods[] = {
1431 /* Device interface */
1432 DEVMETHOD(device_attach, arm_gicv2m_attach),
1433
1434 /* MSI/MSI-X */
1435 DEVMETHOD(msi_alloc_msi, arm_gicv2m_alloc_msi),
1436 DEVMETHOD(msi_release_msi, arm_gicv2m_release_msi),
1437 DEVMETHOD(msi_alloc_msix, arm_gicv2m_alloc_msix),
1438 DEVMETHOD(msi_release_msix, arm_gicv2m_release_msix),
1439 DEVMETHOD(msi_map_msi, arm_gicv2m_map_msi),
1440
1441 /* End */
1442 DEVMETHOD_END
1443 };
1444
1445 DEFINE_CLASS_0(gicv2m, arm_gicv2m_driver, arm_gicv2m_methods,
1446 sizeof(struct arm_gicv2m_softc));
1447