1 /*-
2 * Copyright (c) 2015-2016 The FreeBSD Foundation
3 * Copyright (c) 2025 Arm Ltd
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27 #include "opt_acpi.h"
28 #include "opt_platform.h"
29
30 #include <sys/param.h>
31 #include <sys/systm.h>
32 #include <sys/bus.h>
33 #include <sys/cpuset.h>
34 #include <sys/intr.h>
35 #include <sys/interrupt.h>
36 #include <sys/kernel.h>
37 #include <sys/lock.h>
38 #include <sys/malloc.h>
39 #include <sys/module.h>
40 #include <sys/mutex.h>
41 #include <sys/proc.h>
42 #include <sys/rman.h>
43 #include <sys/smp.h>
44
45 #include <vm/vm.h>
46 #include <vm/pmap.h>
47
48 #include <machine/atomic.h>
49 #include <machine/bus.h>
50 #include <machine/cpu_feat.h>
51 #include <machine/smp.h>
52
53 #ifdef FDT
54 #include <dev/fdt/fdt_intr.h>
55 #include <dev/ofw/ofw_bus_subr.h>
56 #endif
57
58 #include "pic_if.h"
59
60 #include <arm/arm/gic_common.h>
61 #include "gicv5reg.h"
62 #include "gicv5var.h"
63 #include "gic_v3_var.h" /* For GICV3_IVAR_NIRQS */
64
65 #define GICV5_PPIS_PER_REG 64
66 #define GICV5_PPI_COUNT 128
67
68 #define LPI_IPI_BASE 0
69 #define LPI_IPI_LIMIT (LPI_IPI_BASE + (mp_maxid + 1) * INTR_IPI_COUNT)
70 #define LPI_IS_IPI(lpi) ((lpi) < LPI_ITS_BASE)
71 #define LPI_IPI_IDX(lpi) ((lpi) - LPI_IPI_BASE)
72 #define LPI_TO_IPI(lpi) (LPI_IPI_IDX(lpi) % INTR_IPI_COUNT)
73 #define IPI_TO_LPI(ipi, cpu) ((cpu) * INTR_IPI_COUNT + (ipi))
74
75 #define LPI_ITS_BASE (LPI_IPI_BASE + LPI_IPI_LIMIT)
76
77 /* 2^12 LPIs should be enough for a linear table */
78 #define GICV5_LPI_ID_BITS_MAX 12
79
80 #define IRS_CFG_READ_4(_irs, _reg) \
81 bus_read_4((_irs)->irs_cfg, (_reg))
82 #define IRS_CFG_WRITE_4(_irs, _reg, _val) \
83 bus_write_4((_irs)->irs_cfg, (_reg), (_val))
84 #define IRS_CFG_READ_8(_irs, _reg) \
85 bus_read_8((_irs)->irs_cfg, (_reg))
86 #define IRS_CFG_WRITE_8(_irs, _reg, _val) \
87 bus_write_8((_irs)->irs_cfg, (_reg), (_val))
88
89 struct gicv5_irs {
90 cpuset_t irs_cpus;
91 struct resource *irs_cfg;
92 uint64_t *ist_base;
93
94 u_int irs_next_irq_cpu;
95
96 u_int irs_parange;
97
98 int irs_cfg_rid;
99 u_int irs_spi_start;
100 u_int irs_spi_count;
101
102 struct mtx irs_lock;
103 size_t irs_lpi_l2size;
104 u_int irs_lpi_l2bits;
105 bool irs_lpi_2l;
106
107 #ifdef INVARIANTS
108 bool irs_ready;
109 #endif
110 };
111
112 struct gicv5_irqsrc {
113 struct gicv5_base_irqsrc gi_isrc;
114 struct gicv5_irs *gi_irs;
115 enum intr_polarity gi_pol;
116 enum intr_trigger gi_trig;
117 };
118
119 static __read_mostly int *gicv5_iaffids;
120
121 static bus_print_child_t gicv5_print_child;
122 static bus_read_ivar_t gicv5_read_ivar;
123 static bus_get_cpus_t gicv5_get_cpus;
124 static bus_get_resource_list_t gicv5_get_resource_list;
125 static pic_disable_intr_t gicv5_disable_intr;
126 static pic_enable_intr_t gicv5_enable_intr;
127 static pic_map_intr_t gicv5_map_intr;
128 static pic_setup_intr_t gicv5_setup_intr;
129 static pic_teardown_intr_t gicv5_teardown_intr;
130 static pic_post_filter_t gicv5_post_filter;
131 static pic_post_ithread_t gicv5_post_ithread;
132 static pic_pre_ithread_t gicv5_pre_ithread;
133 static pic_bind_intr_t gicv5_bind_intr;
134 #ifdef SMP
135 static pic_init_secondary_t gicv5_init_secondary;
136 static pic_ipi_send_t gicv5_ipi_send;
137 static pic_ipi_setup_t gicv5_ipi_setup;
138 #endif
139
140 static device_method_t gicv5_methods[] = {
141 /* Bus interface */
142 DEVMETHOD(bus_print_child, gicv5_print_child),
143 DEVMETHOD(bus_read_ivar, gicv5_read_ivar),
144 DEVMETHOD(bus_get_cpus, gicv5_get_cpus),
145 DEVMETHOD(bus_setup_intr, bus_generic_setup_intr),
146 DEVMETHOD(bus_teardown_intr, bus_generic_teardown_intr),
147 DEVMETHOD(bus_get_resource_list, gicv5_get_resource_list),
148 DEVMETHOD(bus_get_resource, bus_generic_rl_get_resource),
149 DEVMETHOD(bus_set_resource, bus_generic_rl_set_resource),
150 DEVMETHOD(bus_alloc_resource, bus_generic_rl_alloc_resource),
151 DEVMETHOD(bus_release_resource, bus_generic_rl_release_resource),
152 DEVMETHOD(bus_activate_resource, bus_generic_activate_resource),
153 DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource),
154
155 /* Interrupt controller interface */
156 DEVMETHOD(pic_disable_intr, gicv5_disable_intr),
157 DEVMETHOD(pic_enable_intr, gicv5_enable_intr),
158 DEVMETHOD(pic_map_intr, gicv5_map_intr),
159 DEVMETHOD(pic_setup_intr, gicv5_setup_intr),
160 DEVMETHOD(pic_teardown_intr, gicv5_teardown_intr),
161 DEVMETHOD(pic_post_filter, gicv5_post_filter),
162 DEVMETHOD(pic_post_ithread, gicv5_post_ithread),
163 DEVMETHOD(pic_pre_ithread, gicv5_pre_ithread),
164 DEVMETHOD(pic_bind_intr, gicv5_bind_intr),
165 #ifdef SMP
166 DEVMETHOD(pic_init_secondary, gicv5_init_secondary),
167 DEVMETHOD(pic_ipi_send, gicv5_ipi_send),
168 DEVMETHOD(pic_ipi_setup, gicv5_ipi_setup),
169 #endif
170
171 /* End */
172 DEVMETHOD_END
173 };
174
175 DEFINE_CLASS_0(gic, gicv5_driver, gicv5_methods, sizeof(struct gicv5_softc));
176
177 static int
gicv5_wait_for_op(struct gicv5_irs * irs,bus_size_t reg,uint32_t mask,uint32_t * valp)178 gicv5_wait_for_op(struct gicv5_irs *irs, bus_size_t reg, uint32_t mask,
179 uint32_t *valp)
180 {
181 uint32_t val;
182 int timeout;
183
184 #ifdef INVARIANTS
185 if (irs->irs_ready)
186 mtx_assert(&irs->irs_lock, MA_OWNED);
187 #endif
188
189 /* Timeout of ~10ms */
190 timeout = 10000;
191 do {
192 val = IRS_CFG_READ_4(irs, reg);
193 if ((val & mask) != 0) {
194 if (valp != NULL)
195 *valp = val;
196 return (0);
197 }
198 DELAY(1);
199 } while (--timeout > 0);
200
201 return (ETIMEDOUT);
202 }
203
204 static int
gicv5_wait_irs_cr0_idle(struct gicv5_irs * irs)205 gicv5_wait_irs_cr0_idle(struct gicv5_irs *irs)
206 {
207 return (gicv5_wait_for_op(irs, IRS_CR0, IRS_CR0_IDLE, NULL));
208 }
209
210 static int
gicv5_wait_irs_spi_status_idle(struct gicv5_irs * irs)211 gicv5_wait_irs_spi_status_idle(struct gicv5_irs *irs)
212 {
213 uint32_t val;
214 int error;
215
216 error = gicv5_wait_for_op(irs, IRS_SPI_STATUSR, IRS_SPI_STATUSR_IDLE,
217 &val);
218 if (error != 0)
219 return (error);
220
221 if ((val & IRS_SPI_STATUSR_V) == 0)
222 return (EIO);
223
224 return (0);
225 }
226
227 static void
gicv5_irs_init_ist(struct gicv5_softc * sc,struct gicv5_irs * irs,uint64_t cfgr)228 gicv5_irs_init_ist(struct gicv5_softc *sc, struct gicv5_irs *irs,
229 uint64_t cfgr)
230 {
231 IRS_CFG_WRITE_4(irs, IRS_IST_CFGR, cfgr);
232
233 KASSERT((vtophys(irs->ist_base) & ~IRS_IST_BASER_ADDR_MASK) == 0,
234 ("%s: Invalid IST base address %lx", __func__,
235 vtophys(irs->ist_base)));
236 IRS_CFG_WRITE_8(irs, IRS_IST_BASER, vtophys(irs->ist_base) |
237 IRS_IST_BASER_VALID);
238
239 gicv5_wait_for_op(irs, IRS_IST_STATUSR, IRS_IST_STATUSR_IDLE, NULL);
240 }
241
242 static void
gicv5_irs_alloc_ist(struct gicv5_softc * sc,struct gicv5_irs * irs,size_t size)243 gicv5_irs_alloc_ist(struct gicv5_softc *sc, struct gicv5_irs *irs,
244 size_t size)
245 {
246 irs->ist_base = contigmalloc(size, M_DEVBUF, M_WAITOK | M_ZERO, 0,
247 (1ul << irs->irs_parange) - 1, size, 0);
248 if (sc->gic_coherent)
249 /* Ensure the IRS observed zeroed memory */
250 dsb(ishst);
251 else
252 cpu_dcache_wbinv_range(irs->ist_base, size);
253
254 }
255
256 static void
gicv5_irs_alloc_linear(struct gicv5_softc * sc,struct gicv5_irs * irs,uint32_t * cfgrp,u_int lpi_id_bits,u_int istsz)257 gicv5_irs_alloc_linear(struct gicv5_softc *sc, struct gicv5_irs *irs,
258 uint32_t *cfgrp, u_int lpi_id_bits, u_int istsz)
259 {
260 size_t size;
261 uint32_t cfgr;
262 u_int n;
263
264 MPASS(istsz <= IRS_IST_CFGR_ISTSZ_16_VAL);
265
266 /*
267 * This is the alignment calculation from the IRS_IST_BASER
268 * definition. If the size is > 64 bytes then size == align.
269 * For sizes < 64 bytes we can just round up the size.
270 */
271 n = MAX(5, istsz + 1 + lpi_id_bits);
272 size = 1ul << (n + 1);
273
274 gicv5_irs_alloc_ist(sc, irs, size);
275
276 irs->irs_lpi_2l = false;
277
278 cfgr = IRS_IST_CFGR_STRUCTURE_LINEAR;
279 cfgr |= istsz << IRS_IST_CFGR_ISTSZ_SHIFT;
280 cfgr |= lpi_id_bits << IRS_IST_CFGR_LPI_ID_BITS_SHIFT;
281 *cfgrp = cfgr;
282 }
283
284 static void
gicv5_irs_alloc_2level(struct gicv5_softc * sc,struct gicv5_irs * irs,uint32_t * cfgrp,u_int lpi_id_bits,u_int istsz,u_int l2sz)285 gicv5_irs_alloc_2level(struct gicv5_softc *sc, struct gicv5_irs *irs,
286 uint32_t *cfgrp, u_int lpi_id_bits, u_int istsz, u_int l2sz)
287 {
288 size_t size;
289 uint32_t cfgr;
290 u_int n;
291
292 MPASS(istsz <= IRS_IST_CFGR_ISTSZ_16_VAL);
293
294 /*
295 * This is the alignment calculation from the IRS_IST_BASER
296 * definition. If the size is > 64 bytes then size == align.
297 * for sizes < 64 bytes we can just round up the size.
298 */
299 n = MAX(5, lpi_id_bits - L2_ISTE_LOG2_ENTRIES(istsz, l2sz) + 2);
300 size = 1ul << (n + 1);
301
302 gicv5_irs_alloc_ist(sc, irs, size);
303
304 irs->irs_lpi_l2size = 1ul << (L2_ISTE_LOG2_SIZE(l2sz));
305 irs->irs_lpi_l2bits = L2_ISTE_LOG2_ENTRIES(istsz, l2sz);
306 irs->irs_lpi_2l = true;
307
308 cfgr = IRS_IST_CFGR_STRUCTURE_2LVL;
309 cfgr |= istsz << IRS_IST_CFGR_ISTSZ_SHIFT;
310 cfgr |= l2sz << IRS_IST_CFGR_L2SZ_SHIFT;
311 cfgr |= lpi_id_bits << IRS_IST_CFGR_LPI_ID_BITS_SHIFT;
312 *cfgrp = cfgr;
313 }
314
315 void
gicv5_irs_extend_ist(device_t dev,device_t child,u_int lpi)316 gicv5_irs_extend_ist(device_t dev, device_t child, u_int lpi)
317 {
318 struct gicv5_softc *sc;
319 struct gicv5_devinfo *di;
320 struct gicv5_irs *irs;
321 void *l2_ist;
322 size_t size;
323 u_int index;
324
325 di = device_get_ivars(child);
326 irs = di->di_irs;
327 MPASS(irs != NULL);
328
329 /*
330 * If we have a linear table then we don't need to extend it, it is
331 * already large enough for all LPIs we could allocate.
332 */
333 if (!irs->irs_lpi_2l)
334 return;
335
336 sc = device_get_softc(dev);
337 index = lpi >> irs->irs_lpi_l2bits;
338 size = irs->irs_lpi_l2size;
339
340 /* Check if there the l2 pointer is valid */
341 if ((irs->ist_base[index] & L1_ISTE_VALID) != 0) {
342 return;
343 }
344
345 /* Try allocating the level 2 IST */
346 l2_ist = contigmalloc(size, M_DEVBUF, M_WAITOK | M_ZERO, 0,
347 (1ul << irs->irs_parange) - 1, size, 0);
348
349 mtx_lock_spin(&irs->irs_lock);
350
351 /* Check if we won the race */
352 if ((irs->ist_base[index] & L1_ISTE_VALID) != 0) {
353 mtx_unlock_spin(&irs->irs_lock);
354 free(l2_ist, M_DEVBUF);
355 return;
356 }
357
358 irs->ist_base[index] = vtophys(l2_ist) | L1_ISTE_VALID;
359 if (sc->gic_coherent) {
360 dsb(ishst);
361 } else {
362 cpu_dcache_wbinv_range(l2_ist, size);
363 cpu_dcache_wb_range(&irs->ist_base[index],
364 sizeof(irs->ist_base[index]));
365 }
366
367 IRS_CFG_WRITE_4(irs, IRS_MAP_L2_ISTR, lpi);
368
369 gicv5_wait_for_op(irs, IRS_IST_STATUSR, IRS_IST_STATUSR_IDLE, NULL);
370
371 if (!sc->gic_coherent)
372 cpu_dcache_inv_range(&irs->ist_base[index],
373 sizeof(irs->ist_base[index]));
374 mtx_unlock_spin(&irs->irs_lock);
375 }
376
377 void
gicv5_irs_init(device_t dev,u_int idx,cpuset_t * cpuset)378 gicv5_irs_init(device_t dev, u_int idx, cpuset_t *cpuset)
379 {
380 struct gicv5_softc *sc;
381
382 sc = device_get_softc(dev);
383
384 MPASS(idx < sc->gic_nirs);
385 MPASS(sc->gic_irs != NULL);
386 MPASS(sc->gic_irs[idx] == NULL);
387
388 sc->gic_irs[idx] = malloc(sizeof(*sc->gic_irs[0]), M_DEVBUF,
389 M_ZERO | M_WAITOK);
390 CPU_COPY(cpuset, &sc->gic_irs[idx]->irs_cpus);
391 sc->gic_irs[idx]->irs_cfg_rid = idx;
392 }
393
394 static void
gicv5_irs_attach(device_t dev,struct gicv5_irs * irs,u_int idx)395 gicv5_irs_attach(device_t dev, struct gicv5_irs *irs, u_int idx)
396 {
397 struct gicv5_softc *sc;
398 const char *name;
399 uint64_t icc_idr0;
400 uint32_t cfgr, cr1, idr2;
401 u_int istsz, lpi_id_bits, l2sz, spi_count, spi_end;
402 int error, iaffid;
403 bool two_levels;
404
405 sc = device_get_softc(dev);
406 name = device_get_nameunit(dev);
407
408 /* The attachment needs to set this */
409 MPASS(!CPU_EMPTY(&irs->irs_cpus));
410
411 #ifdef INVARIANTS
412 irs->irs_ready = false;
413 #endif
414 mtx_init(&irs->irs_lock, "GICv5 IRS lock", NULL, MTX_SPIN);
415
416 irs->irs_cfg = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
417 &irs->irs_cfg_rid, RF_ACTIVE);
418 if (irs->irs_cfg == NULL)
419 panic("%s: Unable to allocate memory resource",
420 device_get_nameunit(dev));
421
422 switch (bus_read_4(irs->irs_cfg, IRS_IDR0) & IRS_IDR0_PA_RANGE_MASK) {
423 default:
424 case IRS_IDR0_PA_RANGE_4G:
425 irs->irs_parange = 32;
426 break;
427 case IRS_IDR0_PA_RANGE_64G:
428 irs->irs_parange = 36;
429 break;
430 case IRS_IDR0_PA_RANGE_1T:
431 irs->irs_parange = 40;
432 break;
433 case IRS_IDR0_PA_RANGE_4T:
434 irs->irs_parange = 42;
435 break;
436 case IRS_IDR0_PA_RANGE_16T:
437 irs->irs_parange = 44;
438 break;
439 case IRS_IDR0_PA_RANGE_256T:
440 irs->irs_parange = 48;
441 break;
442 case IRS_IDR0_PA_RANGE_4P:
443 irs->irs_parange = 52;
444 break;
445 case IRS_IDR0_PA_RANGE_64P:
446 irs->irs_parange = 56;
447 break;
448 }
449
450 /* Set the control registers */
451 if (sc->gic_coherent) {
452 cr1 = IRS_CR1_VPED_WA |
453 IRS_CR1_VPED_RA |
454 IRS_CR1_VMD_WA |
455 IRS_CR1_VMD_RA |
456 IRS_CR1_VPET_WA |
457 IRS_CR1_VPET_RA |
458 IRS_CR1_VMT_WA |
459 IRS_CR1_VMT_RA |
460 IRS_CR1_IST_WA |
461 IRS_CR1_IST_RA |
462 IRS_CR1_IC_WB |
463 IRS_CR1_OC_WB |
464 IRS_CR1_SH_IS;
465 } else {
466 cr1 = IRS_CR1_VPED_NO_WA |
467 IRS_CR1_VPED_NO_RA |
468 IRS_CR1_VMD_NO_WA |
469 IRS_CR1_VMD_NO_RA |
470 IRS_CR1_VPET_NO_WA |
471 IRS_CR1_VPET_NO_RA |
472 IRS_CR1_VMT_NO_WA |
473 IRS_CR1_VMT_NO_RA |
474 IRS_CR1_IST_NO_WA |
475 IRS_CR1_IST_NO_RA |
476 IRS_CR1_IC_NC |
477 IRS_CR1_OC_NC |
478 IRS_CR1_SH_NS;
479 }
480 IRS_CFG_WRITE_4(irs, IRS_CR1, cr1);
481 IRS_CFG_WRITE_4(irs, IRS_CR0, IRS_CR0_IRSEN);
482 gicv5_wait_irs_cr0_idle(irs);
483
484 idr2 = IRS_CFG_READ_4(irs, IRS_IDR2);
485
486 two_levels = (idr2 & IRS_IDR2_IST_LEVELS) != 0;
487 lpi_id_bits = IRS_IDR2_ID_BITS(idr2);
488
489 if (!two_levels) {
490 /*
491 * Limit the size of the table as we need to entierly allocate
492 * it for the linear table
493 */
494 lpi_id_bits = MIN(lpi_id_bits, GICV5_LPI_ID_BITS_MAX);
495 /* Ensure lpi_id_bits is at least the mnimum value */
496 lpi_id_bits = MAX(lpi_id_bits, IRS_IDR2_MIN_LPI_ID_BITS(idr2));
497 }
498
499 icc_idr0 = READ_SPECIALREG(ICC_IDR0_EL1);
500 switch(icc_idr0 & ICC_IDR0_ID_BITS_MASK) {
501 case ICC_IDR0_ID_BITS_16:
502 lpi_id_bits = MIN(lpi_id_bits, 16);
503 break;
504 default:
505 case ICC_IDR0_ID_BITS_24:
506 lpi_id_bits = MIN(lpi_id_bits, 24);
507 break;
508 }
509
510 /* The IST entries contain metadata so the size will be larger */
511 if ((idr2 & IRS_IRD2_ISTMD) != 0) {
512 uint64_t istmd_sz;
513
514 istmd_sz = (idr2 & IRS_IDR2_ISTMD_SZ_MASK) >>
515 IRS_IDR2_ISTMD_SZ_SHIFT;
516 if (lpi_id_bits < istmd_sz) {
517 istsz = IRS_IST_CFGR_ISTSZ_8_VAL;
518 } else {
519 istsz = IRS_IST_CFGR_ISTSZ_16_VAL;
520 }
521 } else {
522 /* The default ITS entry size is 4 bytes */
523 istsz = IRS_IST_CFGR_ISTSZ_4_VAL;
524 }
525
526 if (two_levels) {
527 if ((idr2 & IRS_IDR2_IST_L2SZ_64K) != 0)
528 l2sz = IRS_IST_CFGR_L2SZ_64K_VAL;
529 else if ((idr2 & IRS_IDR2_IST_L2SZ_16K) != 0)
530 l2sz = IRS_IST_CFGR_L2SZ_16K_VAL;
531 else
532 l2sz = IRS_IST_CFGR_L2SZ_4K_VAL;
533 }
534
535 /*
536 * Use 2 level tables if able, and the size is large enough for them
537 * to be worth it. This is based on the calculation in the GICv5
538 * spec (ARM-AES-0070) 00EAC0 section 10.2.1.14 IRS_IST_CFGR.
539 */
540 if (two_levels &&
541 lpi_id_bits > L2_ISTE_LOG2_ENTRIES(istsz, l2sz)) {
542 gicv5_irs_alloc_2level(sc, irs, &cfgr, lpi_id_bits, istsz,
543 l2sz);
544 } else {
545 gicv5_irs_alloc_linear(sc, irs, &cfgr, lpi_id_bits, istsz);
546 }
547
548 if (idx == 0)
549 sc->gic_nlpis = 1u << lpi_id_bits;
550 else
551 sc->gic_nlpis = MIN(sc->gic_nlpis, 1u << lpi_id_bits);
552
553 spi_count = IRS_CFG_READ_4(irs, IRS_IDR5) & IRS_IDR5_SPI_RANGE;
554 if (sc->gic_irs_irqs == NULL) {
555 KASSERT(idx == 0,
556 ("%s: Null IRS table on no-zero index (idx = %d)",
557 __func__, idx));
558 sc->gic_spi_count = spi_count;
559 sc->gic_irs_irqs = mallocarray(spi_count,
560 sizeof(struct gicv5_irqsrc), M_DEVBUF, M_WAITOK | M_ZERO);
561 }
562
563 /* Read and check the IRS SPI details */
564 irs->irs_spi_start =
565 IRS_CFG_READ_4(irs, IRS_IDR7) & IRS_IDR7_SPI_BASE;
566
567 irs->irs_spi_count =
568 IRS_CFG_READ_4(irs, IRS_IDR6) & IRS_IDR6_SPI_IRS_RANGE;
569
570 spi_end = irs->irs_spi_start + irs->irs_spi_count;
571 if (spi_end > spi_count)
572 panic("%s: IRS %u has SPIs past global count (%u > %u)\n",
573 device_get_nameunit(dev), idx, spi_end, spi_count);
574
575 gicv5_irs_init_ist(sc, irs, cfgr);
576
577 /*
578 * Set a valid interrupt affinity ID, even if it's for a CPU not
579 * attached to this IRS.
580 */
581 iaffid = gicv5_iaffids[curcpu];
582 MPASS(iaffid >= 0);
583 for (u_int irq = irs->irs_spi_start; irq < spi_end; irq++) {
584 struct gicv5_irqsrc *gi;
585 uint64_t cdaff, cdpri;
586
587 gi = &sc->gic_irs_irqs[irq];
588 MPASS(gi->gi_irs == NULL);
589 gi->gi_isrc.gbi_space = GICv5_SPI;
590 gi->gi_isrc.gbi_ipi = false;
591 gi->gi_isrc.gbi_irq = irq;
592 gi->gi_irs = irs;
593 gi->gi_pol = INTR_POLARITY_CONFORM;
594 gi->gi_trig = INTR_TRIGGER_CONFORM;
595 error = intr_isrc_register(&gi->gi_isrc.gbi_isrc, dev,
596 0, "%s,s%u", name, irq);
597 if (error != 0)
598 panic("%s: Unable to register SPI irq src",
599 device_get_nameunit(dev));
600
601 /* Set the base priority */
602 cdpri = GIC_CDPRI_PRORITY(GICV5_PRI_LOWEST);
603 cdpri |= GIC_CDPRI_TYPE_SPI;
604 cdpri |= GIC_CDPRI_ID(irq);
605 gic_cdpri(cdpri);
606
607 /* Set the affinity */
608 cdaff = GIC_CDAFF_IAFFID(iaffid);
609 cdaff |= GIC_CDAFF_TYPE_SPI;
610 cdaff |= GIC_CDAFF_IRM_TARGETED;
611 cdaff |= GIC_CDAFF_ID(irq);
612 gic_cdaff(cdaff);
613 isb();
614 }
615
616 #ifdef INVARIANTS
617 irs->irs_ready = true;
618 #endif
619 }
620
621 void
gicv5_attach(device_t dev)622 gicv5_attach(device_t dev)
623 {
624 struct gicv5_softc *sc;
625 const char *name;
626 int error;
627
628 sc = device_get_softc(dev);
629 sc->gic_dev = dev;
630 name = device_get_nameunit(dev);
631
632 MPASS(gicv5_iaffids == NULL);
633 gicv5_iaffids = mallocarray(mp_maxid + 1, sizeof(*gicv5_iaffids),
634 M_DEVBUF, M_WAITOK);
635 for (int i = 0; i <= mp_maxid; i++)
636 gicv5_iaffids[i] = -1;
637 gicv5_iaffids[curcpu] =
638 ICC_IAFFIDR_IAFFID_VAL(READ_SPECIALREG(ICC_IAFFIDR_EL1));
639
640 for (u_int i = 0; i < sc->gic_nirs; i++)
641 gicv5_irs_attach(dev, sc->gic_irs[i], i);
642 if (bootverbose)
643 device_printf(dev, "Limited to %u LPIs\n", sc->gic_nlpis);
644
645 sc->gic_ppi_irqs = mallocarray(GICV5_PPI_COUNT,
646 sizeof(struct gicv5_irqsrc), M_DEVBUF, M_ZERO | M_WAITOK);
647 for (u_int irq = 0; irq < GICV5_PPI_COUNT; irq++) {
648 struct intr_irqsrc *isrc;
649
650 sc->gic_ppi_irqs[irq].gi_irs = NULL;
651 sc->gic_ppi_irqs[irq].gi_isrc.gbi_space = GICv5_PPI;
652 sc->gic_ppi_irqs[irq].gi_isrc.gbi_ipi = false;
653 sc->gic_ppi_irqs[irq].gi_isrc.gbi_irq = irq;
654 sc->gic_ppi_irqs[irq].gi_pol = INTR_POLARITY_CONFORM;
655 sc->gic_ppi_irqs[irq].gi_trig = INTR_TRIGGER_CONFORM;
656 isrc = &sc->gic_ppi_irqs[irq].gi_isrc.gbi_isrc;
657 error = intr_isrc_register(isrc, dev, INTR_ISRCF_PPI,
658 "%s,p%u", name, irq);
659 if (error != 0)
660 panic("%s: Unable to register PPI irq src",
661 device_get_nameunit(dev));
662 }
663
664 /* Assign LPIs to be used as IPIs */
665 sc->gic_ipi_irqs = mallocarray(INTR_IPI_COUNT,
666 sizeof(*sc->gic_ipi_irqs), M_DEVBUF, M_ZERO | M_WAITOK);
667 for (u_int ipi = 0; ipi < INTR_IPI_COUNT; ipi++) {
668 struct intr_irqsrc *isrc;
669
670 sc->gic_ipi_irqs[ipi].gi_irs = NULL;
671 sc->gic_ipi_irqs[ipi].gi_isrc.gbi_space = GICv5_LPI;
672 sc->gic_ipi_irqs[ipi].gi_isrc.gbi_ipi = true;
673 sc->gic_ipi_irqs[ipi].gi_isrc.gbi_irq = ipi + LPI_IPI_BASE;
674 sc->gic_ipi_irqs[ipi].gi_pol = INTR_POLARITY_HIGH;
675 sc->gic_ipi_irqs[ipi].gi_trig = INTR_TRIGGER_EDGE;
676 isrc = &sc->gic_ipi_irqs[ipi].gi_isrc.gbi_isrc;
677 error = intr_isrc_register(isrc, dev, INTR_ISRCF_IPI,
678 "%s,ipi%u", name, ipi);
679 if (error != 0)
680 panic("%s: Unable to register LPI irq src",
681 device_get_nameunit(dev));
682 }
683
684 WRITE_SPECIALREG(ICC_PPI_PRIORITYR0_EL1,
685 ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST));
686 WRITE_SPECIALREG(ICC_PPI_PRIORITYR1_EL1,
687 ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST));
688 WRITE_SPECIALREG(ICC_PPI_PRIORITYR2_EL1,
689 ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST));
690 WRITE_SPECIALREG(ICC_PPI_PRIORITYR3_EL1,
691 ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST));
692 WRITE_SPECIALREG(ICC_PPI_PRIORITYR4_EL1,
693 ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST));
694 WRITE_SPECIALREG(ICC_PPI_PRIORITYR5_EL1,
695 ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST));
696 WRITE_SPECIALREG(ICC_PPI_PRIORITYR6_EL1,
697 ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST));
698 WRITE_SPECIALREG(ICC_PPI_PRIORITYR7_EL1,
699 ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST));
700 WRITE_SPECIALREG(ICC_PPI_PRIORITYR8_EL1,
701 ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST));
702 WRITE_SPECIALREG(ICC_PPI_PRIORITYR9_EL1,
703 ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST));
704 WRITE_SPECIALREG(ICC_PPI_PRIORITYR10_EL1,
705 ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST));
706 WRITE_SPECIALREG(ICC_PPI_PRIORITYR11_EL1,
707 ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST));
708 WRITE_SPECIALREG(ICC_PPI_PRIORITYR12_EL1,
709 ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST));
710 WRITE_SPECIALREG(ICC_PPI_PRIORITYR13_EL1,
711 ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST));
712 WRITE_SPECIALREG(ICC_PPI_PRIORITYR14_EL1,
713 ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST));
714 WRITE_SPECIALREG(ICC_PPI_PRIORITYR15_EL1,
715 ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST));
716
717 WRITE_SPECIALREG(ICC_PPI_ENABLER0_EL1, ICC_PPI_ENABLER_NONE);
718 WRITE_SPECIALREG(ICC_PPI_ENABLER1_EL1, ICC_PPI_ENABLER_NONE);
719 isb();
720
721 /* Set the priority to the lowest value */
722 WRITE_SPECIALREG(ICC_PCR_EL1, ICC_PCR_PRIORITY_LOWEST);
723
724 /* Enable interrupts */
725 WRITE_SPECIALREG(ICC_CR0_EL1, ICC_CR0_EN);
726 isb();
727 }
728
729 bool
gicv5_add_child(device_t dev,struct gicv5_devinfo * di)730 gicv5_add_child(device_t dev, struct gicv5_devinfo *di)
731 {
732 device_t cdev;
733 struct gicv5_softc *sc;
734
735 cdev = device_add_child(dev, NULL, DEVICE_UNIT_ANY);
736 if (cdev == NULL)
737 return (false);
738
739 sc = device_get_softc(dev);
740 sc->gic_nchildren++;
741 device_set_ivars(cdev, di);
742
743 return (true);
744 }
745
746 static int
gicv5_print_child(device_t bus,device_t child)747 gicv5_print_child(device_t bus, device_t child)
748 {
749 struct resource_list *rl;
750 int retval = 0;
751
752 rl = BUS_GET_RESOURCE_LIST(bus, child);
753 KASSERT(rl != NULL, ("%s: No resource list", __func__));
754 retval += bus_print_child_header(bus, child);
755 retval += resource_list_print_type(rl, "mem", SYS_RES_MEMORY, "%#jx");
756 retval += bus_print_child_footer(bus, child);
757
758 return (retval);
759 }
760
761 static u_int
gicv5_lpi_count(struct gicv5_softc * sc)762 gicv5_lpi_count(struct gicv5_softc *sc)
763 {
764 MPASS(sc->gic_nlpis >= LPI_ITS_BASE);
765 return (sc->gic_nlpis - LPI_ITS_BASE);
766 }
767
768 static int
gicv5_read_ivar(device_t dev,device_t child,int which,uintptr_t * result)769 gicv5_read_ivar(device_t dev, device_t child, int which, uintptr_t *result)
770 {
771 struct gicv5_softc *sc;
772 struct gicv5_devinfo *di;
773 u_int nlpis;
774
775 switch (which) {
776 case GICV5_IVAR_LPI_START:
777 di = device_get_ivars(child);
778 if (di == NULL || di->di_irs == NULL)
779 return (EINVAL);
780
781 sc = device_get_softc(dev);
782 nlpis = gicv5_lpi_count(sc) / sc->gic_nchildren;
783 *result = LPI_ITS_BASE + device_get_unit(dev) * nlpis;
784 return (0);
785 case GICV3_IVAR_NIRQS:
786 di = device_get_ivars(child);
787 if (di == NULL || di->di_irs == NULL)
788 return (EINVAL);
789
790 sc = device_get_softc(dev);
791 *result = gicv5_lpi_count(sc) / sc->gic_nchildren;
792 return (0);
793 case GIC_IVAR_HW_REV:
794 *result = 5;
795 return (0);
796 case GIC_IVAR_BUS:
797 sc = device_get_softc(dev);
798 KASSERT(sc->gic_bus != GIC_BUS_UNKNOWN,
799 ("%s: Unknown bus type", __func__));
800 KASSERT(sc->gic_bus <= GIC_BUS_MAX,
801 ("%s: Invalid bus type %u", __func__, sc->gic_bus));
802 *result = sc->gic_bus;
803 return (0);
804 case GIC_IVAR_VGIC:
805 /* TODO when we have vgic support */
806 *result = 0;
807 return (0);
808 case GIC_IVAR_SUPPORT_LPIS:
809 di = device_get_ivars(child);
810 if (di == NULL || di->di_irs == NULL)
811 return (EINVAL);
812
813 *result =
814 (IRS_CFG_READ_4(di->di_irs, IRS_IDR2) & IRS_IDR2_LPI) != 0;
815 return (0);
816 }
817
818 return (ENOENT);
819 }
820
821 static int
gicv5_get_cpus(device_t dev,device_t child,enum cpu_sets op,size_t setsize,cpuset_t * cpuset)822 gicv5_get_cpus(device_t dev, device_t child, enum cpu_sets op, size_t setsize,
823 cpuset_t *cpuset)
824 {
825 struct gicv5_devinfo *di;
826
827 if (op != LOCAL_CPUS)
828 return (bus_generic_get_cpus(dev, child, op, setsize, cpuset));
829
830 di = device_get_ivars(child);
831 if (di == NULL)
832 return (bus_generic_get_cpus(dev, child, op, setsize, cpuset));
833
834 if (setsize != sizeof(cpuset_t))
835 return (EINVAL);
836
837 *cpuset = di->di_irs->irs_cpus;
838 return (0);
839 }
840
841 static struct resource_list *
gicv5_get_resource_list(device_t bus,device_t child)842 gicv5_get_resource_list(device_t bus, device_t child)
843 {
844 struct gicv5_devinfo *di;
845
846 di = device_get_ivars(child);
847 KASSERT(di != NULL, ("%s: No devinfo", __func__));
848
849 return (&di->di_rl);
850 }
851
852 static void
gicv5_eoi_intr(enum gicv5_irq_space space,uint32_t irq)853 gicv5_eoi_intr(enum gicv5_irq_space space, uint32_t irq)
854 {
855 uint64_t cddi;
856
857 /* Drop the priority of the specified interrupt */
858 cddi = (uint64_t)space << GIC_CDDI_Type_SHIFT;
859 cddi |= (uint64_t)irq << GIC_CDDI_ID_SHIFT;
860 gic_cddi(cddi);
861
862 /* Drop the running priority of the CPU */
863 gic_cdeoi();
864 }
865
866 static void
gicv5_eoi(struct gicv5_base_irqsrc * gbi)867 gicv5_eoi(struct gicv5_base_irqsrc *gbi)
868 {
869 uint32_t irq;
870 enum gicv5_irq_space space;
871
872 MPASS(!gbi->gbi_ipi);
873 space = gbi->gbi_space;
874 irq = gbi->gbi_irq;
875
876 gicv5_eoi_intr(space, irq);
877 }
878
879 int
gicv5_intr(void * arg)880 gicv5_intr(void *arg)
881 {
882 struct gicv5_softc *sc = arg;
883 struct gicv5_irqsrc *gi;
884 struct trapframe *tf;
885 uint64_t hppi;
886 u_int irq;
887
888 tf = curthread->td_intr_frame;
889 for (;;) {
890 hppi = gicr_cdia();
891 /* Ensure the interrupt activation has completed */
892 gsb_ack();
893 /* Ensure the gsb ack instruction has completed */
894 isb();
895
896 if ((hppi & ICC_HPPIR_HPPIV) == 0)
897 return (FILTER_HANDLED);
898
899 irq = (hppi & ICC_HPPIR_ID_MASK) >> ICC_HPPIR_ID_SHIFT;
900 switch (hppi & ICC_HPPIR_TYPE_MASK) {
901 case ICC_HPPIR_TYPE_PPI:
902 MPASS(irq < GICV5_PPI_COUNT);
903 gi = &sc->gic_ppi_irqs[irq];
904 if (intr_isrc_dispatch(&gi->gi_isrc.gbi_isrc, tf) != 0){
905 if (gi->gi_trig != INTR_TRIGGER_EDGE)
906 gicv5_eoi(&gi->gi_isrc);
907 gicv5_disable_intr(sc->gic_dev,
908 &gi->gi_isrc.gbi_isrc);
909 device_printf(sc->gic_dev,
910 "Stray PPI %u disabled\n", irq);
911 }
912 break;
913 case ICC_HPPIR_TYPE_LPI:
914 /* XXX */
915 if (LPI_IS_IPI(irq)) {
916 u_int ipi;
917
918 KASSERT(LPI_IPI_IDX(irq) < LPI_IPI_LIMIT,
919 ("%s: Invalid IPI LPI %u", __func__, irq));
920 ipi = LPI_TO_IPI(irq);
921 #ifdef SMP
922 intr_ipi_dispatch(ipi);
923 #else
924 device_printf(sc->gic_dev,
925 "IPI LPI %u on UP system detected\n", ipi);
926 #endif
927 gicv5_eoi_intr(GICv5_LPI, irq);
928 } else {
929 intr_child_irq_handler(sc->gic_pic, irq);
930 }
931 break;
932 case ICC_HPPIR_TYPE_SPI:
933 MPASS(irq < sc->gic_spi_count);
934 gi = &sc->gic_irs_irqs[irq];
935 if (intr_isrc_dispatch(&gi->gi_isrc.gbi_isrc, tf) != 0){
936 if (gi->gi_trig != INTR_TRIGGER_EDGE)
937 gicv5_eoi(&gi->gi_isrc);
938 gicv5_disable_intr(sc->gic_dev,
939 &gi->gi_isrc.gbi_isrc);
940 device_printf(sc->gic_dev,
941 "Stray SPI %u disabled\n", irq);
942 }
943 break;
944 default:
945 panic("%s: Invalid interrupt type %lx", __func__,
946 (hppi & ICC_HPPIR_TYPE_MASK) >>
947 ICC_HPPIR_TYPE_SHIFT);
948 }
949 }
950 }
951
952 static void
gicv5_disable_intr_action(void * argp)953 gicv5_disable_intr_action(void *argp)
954 {
955 struct gicv5_base_irqsrc *gbi = argp;
956 uint64_t reg;
957 uint32_t irq;
958
959 MPASS(!gbi->gbi_ipi);
960 irq = gbi->gbi_irq;
961
962 if (irq < GICV5_PPIS_PER_REG)
963 reg = READ_SPECIALREG(ICC_PPI_ENABLER0_EL1);
964 else
965 reg = READ_SPECIALREG(ICC_PPI_ENABLER1_EL1);
966
967 reg &= ~ICC_PPI_ENABLER_MASK(irq);
968 reg |= ICC_PPI_ENABLER_DIS(irq);
969
970 if (irq < GICV5_PPIS_PER_REG)
971 WRITE_SPECIALREG(ICC_PPI_ENABLER0_EL1, reg);
972 else
973 WRITE_SPECIALREG(ICC_PPI_ENABLER1_EL1, reg);
974 isb();
975 }
976
977 static void
gicv5_disable_intr(device_t dev,struct intr_irqsrc * isrc)978 gicv5_disable_intr(device_t dev, struct intr_irqsrc *isrc)
979 {
980 struct gicv5_base_irqsrc *gbi = (struct gicv5_base_irqsrc *)isrc;
981 uint32_t irq;
982
983 /* We don't disable IPIs */
984 MPASS(!gbi->gbi_ipi);
985 irq = gbi->gbi_irq;
986
987 switch (gbi->gbi_space) {
988 case GICv5_PPI:
989 MPASS((isrc->isrc_flags & INTR_ISRCF_PPI) != 0);
990
991 smp_rendezvous(NULL, gicv5_disable_intr_action, NULL, gbi);
992 break;
993 case GICv5_SPI:
994 gic_cddis(GIC_CDDIS_TYPE_SPI | (irq << GIC_CDDIS_ID_SHIFT));
995 isb();
996 break;
997 case GICv5_LPI:
998 gic_cddis(GIC_CDDIS_TYPE_LPI | (irq << GIC_CDDIS_ID_SHIFT));
999 isb();
1000 break;
1001 default:
1002 panic("%s: Invalid interrupt space 0x%x", __func__,
1003 gbi->gbi_space);
1004 }
1005 }
1006
1007 static void
gicv5_enable_intr_action(void * argp)1008 gicv5_enable_intr_action(void *argp)
1009 {
1010 struct gicv5_base_irqsrc *gbi = argp;
1011 uint64_t reg;
1012 uint32_t irq;
1013
1014 MPASS(!gbi->gbi_ipi);
1015 MPASS(gbi->gbi_space == GICv5_PPI);
1016
1017 irq = gbi->gbi_irq;
1018
1019 if (irq < GICV5_PPIS_PER_REG)
1020 reg = READ_SPECIALREG(ICC_PPI_ENABLER0_EL1);
1021 else
1022 reg = READ_SPECIALREG(ICC_PPI_ENABLER1_EL1);
1023
1024 reg &= ~ICC_PPI_ENABLER_MASK(irq);
1025 reg |= ICC_PPI_ENABLER_EN(irq);
1026
1027 if (irq < GICV5_PPIS_PER_REG)
1028 WRITE_SPECIALREG(ICC_PPI_ENABLER0_EL1, reg);
1029 else
1030 WRITE_SPECIALREG(ICC_PPI_ENABLER1_EL1, reg);
1031 isb();
1032 }
1033
1034 static void
gicv5_enable_ipi(struct gicv5_base_irqsrc * gbi)1035 gicv5_enable_ipi(struct gicv5_base_irqsrc *gbi)
1036 {
1037 uint32_t irq;
1038
1039 MPASS(gbi->gbi_ipi);
1040 MPASS(gbi->gbi_space == GICv5_LPI);
1041
1042 irq = IPI_TO_LPI(gbi->gbi_irq, PCPU_GET(cpuid));
1043 MPASS(irq < LPI_IPI_LIMIT);
1044
1045 gic_cden(GIC_CDEN_TYPE_LPI | (irq << GIC_CDEN_ID_SHIFT));
1046 isb();
1047 }
1048
1049 static void
gicv5_enable_intr(device_t dev,struct intr_irqsrc * isrc)1050 gicv5_enable_intr(device_t dev, struct intr_irqsrc *isrc)
1051 {
1052 struct gicv5_base_irqsrc *gbi = (struct gicv5_base_irqsrc *)isrc;
1053 uint32_t irq;
1054
1055 if (gbi->gbi_ipi) {
1056 gicv5_enable_ipi(gbi);
1057 return;
1058 }
1059
1060 irq = gbi->gbi_irq;
1061
1062 switch (gbi->gbi_space) {
1063 case GICv5_PPI:
1064 MPASS((isrc->isrc_flags & INTR_ISRCF_PPI) != 0);
1065
1066 smp_rendezvous(NULL, gicv5_enable_intr_action, NULL, gbi);
1067 break;
1068 case GICv5_SPI:
1069 gic_cden(GIC_CDEN_TYPE_SPI | (irq << GIC_CDEN_ID_SHIFT));
1070 isb();
1071 break;
1072 case GICv5_LPI:
1073 gic_cden(GIC_CDEN_TYPE_LPI | (irq << GIC_CDEN_ID_SHIFT));
1074 isb();
1075 break;
1076 default:
1077 panic("%s: Invalid interrupt space 0x%x", __func__,
1078 gbi->gbi_space);
1079 }
1080 }
1081
1082 #ifdef FDT
1083 static int
gic_map_fdt(device_t dev,u_int ncells,pcell_t * cells,bool * ppi,u_int * irqp,enum intr_polarity * polp,enum intr_trigger * trigp)1084 gic_map_fdt(device_t dev, u_int ncells, pcell_t *cells, bool *ppi, u_int *irqp,
1085 enum intr_polarity *polp, enum intr_trigger *trigp)
1086 {
1087 uint64_t reg;
1088 u_int irq;
1089 int type;
1090
1091 if (ncells < 3)
1092 return (EINVAL);
1093
1094 /*
1095 * The 1st cell is the interrupt type:
1096 * 1 = PPI
1097 * 2 = LPI
1098 * 3 = SPI
1099 * The 2nd cell contains the interrupt number
1100 * The 3rd cell is the flags, encoded as follows:
1101 * bits[3:0] trigger type and level flags
1102 * 1 = edge triggered
1103 * 2 = edge triggered (PPI only)
1104 * 4 = level-sensitive
1105 * 8 = level-sensitive (PPI only)
1106 */
1107 switch (cells[0]) {
1108 case 1:
1109 *ppi = true;
1110 break;
1111 /* case 2: LPI */
1112 case 3:
1113 *ppi = false;
1114 break;
1115 default:
1116 device_printf(dev, "unsupported interrupt type "
1117 "configuration:");
1118 for (u_int i = 0; i < ncells; i++)
1119 printf(" %x", cells[i]);
1120 printf("\n");
1121 return (EINVAL);
1122 }
1123
1124 irq = cells[1];
1125
1126 if (ppi) {
1127 /* PPIs are hard coded, ignore cells[2] */
1128 if (irq < GICV5_PPIS_PER_REG)
1129 reg = READ_SPECIALREG(ICC_PPI_HMR0_EL1);
1130 else
1131 reg = READ_SPECIALREG(ICC_PPI_HMR1_EL1);
1132
1133 if ((reg & ICC_PPI_HMR_MASK(irq)) == ICC_PPI_HMR_EDGE(irq))
1134 type = FDT_INTR_EDGE_RISING;
1135 else
1136 type = FDT_INTR_LEVEL_LOW;
1137 } else {
1138 type = cells[2] & FDT_INTR_MASK;
1139 }
1140
1141 switch (type) {
1142 case FDT_INTR_EDGE_RISING:
1143 *trigp = INTR_TRIGGER_EDGE;
1144 *polp = INTR_POLARITY_HIGH;
1145 break;
1146 case FDT_INTR_EDGE_FALLING:
1147 *trigp = INTR_TRIGGER_EDGE;
1148 *polp = INTR_POLARITY_LOW;
1149 break;
1150 case FDT_INTR_LEVEL_HIGH:
1151 *trigp = INTR_TRIGGER_LEVEL;
1152 *polp = INTR_POLARITY_HIGH;
1153 break;
1154 case FDT_INTR_LEVEL_LOW:
1155 *trigp = INTR_TRIGGER_LEVEL;
1156 *polp = INTR_POLARITY_LOW;
1157 break;
1158 default:
1159 device_printf(dev, "unsupported trigger/polarity "
1160 "configuration 0x%02x\n", cells[2]);
1161 return (EINVAL);
1162 }
1163
1164 *irqp = irq;
1165 return (0);
1166 }
1167 #endif
1168
1169 static int
do_gicv5_map_intr(device_t dev,struct intr_map_data * data,bool * ppip,u_int * irqp,enum intr_polarity * polp,enum intr_trigger * trigp)1170 do_gicv5_map_intr(device_t dev, struct intr_map_data *data, bool *ppip,
1171 u_int *irqp, enum intr_polarity *polp, enum intr_trigger *trigp)
1172 {
1173 struct gicv5_softc *sc;
1174 enum intr_polarity pol;
1175 enum intr_trigger trig;
1176 #ifdef FDT
1177 struct intr_map_data_fdt *daf;
1178 #endif
1179 u_int irq;
1180 bool ppi;
1181
1182 sc = device_get_softc(dev);
1183
1184 switch (data->type) {
1185 #ifdef FDT
1186 case INTR_MAP_DATA_FDT:
1187 daf = (struct intr_map_data_fdt *)data;
1188 if (gic_map_fdt(dev, daf->ncells, daf->cells, &ppi, &irq, &pol,
1189 &trig) != 0)
1190 return (EINVAL);
1191 break;
1192 #endif
1193 default:
1194 return (EINVAL);
1195 }
1196
1197 if (ppi) {
1198 if (irq >= GICV5_PPI_COUNT)
1199 return (EINVAL);
1200 } else {
1201 if (irq > sc->gic_spi_count)
1202 return (EINVAL);
1203 }
1204 switch (pol) {
1205 case INTR_POLARITY_CONFORM:
1206 case INTR_POLARITY_LOW:
1207 case INTR_POLARITY_HIGH:
1208 break;
1209 default:
1210 return (EINVAL);
1211 }
1212 switch (trig) {
1213 case INTR_TRIGGER_CONFORM:
1214 case INTR_TRIGGER_EDGE:
1215 case INTR_TRIGGER_LEVEL:
1216 break;
1217 default:
1218 return (EINVAL);
1219 }
1220
1221 *ppip = ppi;
1222 *irqp = irq;
1223 if (polp != NULL)
1224 *polp = pol;
1225 if (trigp != NULL)
1226 *trigp = trig;
1227 return (0);
1228 }
1229
1230 static int
gicv5_map_intr(device_t dev,struct intr_map_data * data,struct intr_irqsrc ** isrcp)1231 gicv5_map_intr(device_t dev, struct intr_map_data *data,
1232 struct intr_irqsrc **isrcp)
1233 {
1234 struct gicv5_softc *sc;
1235 u_int irq;
1236 int error;
1237 bool ppi;
1238
1239 error = do_gicv5_map_intr(dev, data, &ppi, &irq, NULL, NULL);
1240 if (error == 0) {
1241 sc = device_get_softc(dev);
1242 if (ppi)
1243 *isrcp = &sc->gic_ppi_irqs[irq].gi_isrc.gbi_isrc;
1244 else
1245 *isrcp = &sc->gic_irs_irqs[irq].gi_isrc.gbi_isrc;
1246 }
1247 return (error);
1248 }
1249
1250 static int
gicv5_setup_intr(device_t dev,struct intr_irqsrc * isrc,struct resource * res,struct intr_map_data * data)1251 gicv5_setup_intr(device_t dev, struct intr_irqsrc *isrc,
1252 struct resource *res, struct intr_map_data *data)
1253 {
1254 struct gicv5_irqsrc *gi = (struct gicv5_irqsrc *)isrc;
1255 enum intr_trigger trig;
1256 enum intr_polarity pol;
1257 struct gicv5_irs *irs;
1258 int error;
1259 u_int irq;
1260 bool ppi;
1261
1262 if (data == NULL)
1263 return (ENOTSUP);
1264
1265 error = do_gicv5_map_intr(dev, data, &ppi, &irq, &pol, &trig);
1266 if (error != 0)
1267 return (error);
1268
1269 /* This shouldn't return IPIs */
1270 MPASS(!gi->gi_isrc.gbi_ipi);
1271
1272 if (gi->gi_isrc.gbi_irq != irq || pol == INTR_POLARITY_CONFORM ||
1273 trig == INTR_TRIGGER_CONFORM)
1274 return (EINVAL);
1275
1276 if (((isrc->isrc_flags & INTR_ISRCF_PPI) != 0) != ppi)
1277 return (EINVAL);
1278
1279 /* Compare config if this is not first setup. */
1280 if (isrc->isrc_handlers != 0) {
1281 if (pol != gi->gi_pol || trig != gi->gi_trig)
1282 return (EINVAL);
1283 else
1284 return (0);
1285 }
1286
1287 gi->gi_pol = pol;
1288 gi->gi_trig = trig;
1289
1290 switch (gi->gi_isrc.gbi_space) {
1291 default:
1292 panic("%s: Invalid IRQ space %#x", __func__,
1293 gi->gi_isrc.gbi_space);
1294 case GICv5_PPI:
1295 MPASS(ppi);
1296 MPASS(gi->gi_irs == NULL);
1297 MPASS(irq < GICV5_PPI_COUNT);
1298 CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu);
1299
1300 break;
1301 case GICv5_SPI:
1302 MPASS(!ppi);
1303 irs = gi->gi_irs;
1304
1305 mtx_lock_spin(&irs->irs_lock);
1306 /*
1307 * This depends on intr_setup_irq holding the isrc_table_lock
1308 * to serialise access to this register
1309 */
1310 IRS_CFG_WRITE_4(irs, IRS_SPI_SELR, irq);
1311 error = gicv5_wait_irs_spi_status_idle(irs);
1312 if (error != 0) {
1313 mtx_unlock_spin(&irs->irs_lock);
1314 return (error);
1315 }
1316
1317 /* Set the trigger mode */
1318 if (trig == INTR_TRIGGER_EDGE)
1319 IRS_CFG_WRITE_4(irs, IRS_SPI_CFGR,
1320 IRS_SPI_CFGR_TM_EDGE);
1321 else
1322 IRS_CFG_WRITE_4(irs, IRS_SPI_CFGR,
1323 IRS_SPI_CFGR_TM_LEVEL);
1324
1325 error = gicv5_wait_irs_spi_status_idle(irs);
1326 mtx_unlock_spin(&irs->irs_lock);
1327 return (error);
1328 }
1329
1330 return (0);
1331 }
1332
1333 static int
gicv5_teardown_intr(device_t dev,struct intr_irqsrc * isrc,struct resource * res,struct intr_map_data * data)1334 gicv5_teardown_intr(device_t dev, struct intr_irqsrc *isrc,
1335 struct resource *res, struct intr_map_data *data)
1336 {
1337 return (0);
1338 }
1339
1340 static void
gicv5_post_filter(device_t dev,struct intr_irqsrc * isrc)1341 gicv5_post_filter(device_t dev, struct intr_irqsrc *isrc)
1342 {
1343 struct gicv5_base_irqsrc *gbi = (struct gicv5_base_irqsrc *)isrc;
1344
1345 /* No ithreads for IPIs */
1346 MPASS(!gbi->gbi_ipi);
1347
1348 gicv5_eoi(gbi);
1349 }
1350
1351 static void
gicv5_pre_ithread(device_t dev,struct intr_irqsrc * isrc)1352 gicv5_pre_ithread(device_t dev, struct intr_irqsrc *isrc)
1353 {
1354 struct gicv5_base_irqsrc *gbi = (struct gicv5_base_irqsrc *)isrc;
1355
1356 /* No ithreads for IPIs */
1357 MPASS(!gbi->gbi_ipi);
1358
1359 switch (gbi->gbi_space) {
1360 case GICv5_PPI:
1361 gicv5_disable_intr_action(isrc);
1362 break;
1363 default:
1364 gicv5_disable_intr(dev, isrc);
1365 break;
1366 }
1367 gicv5_eoi(gbi);
1368 }
1369
1370 static void
gicv5_post_ithread(device_t dev,struct intr_irqsrc * isrc)1371 gicv5_post_ithread(device_t dev, struct intr_irqsrc *isrc)
1372 {
1373 struct gicv5_base_irqsrc *gbi = (struct gicv5_base_irqsrc *)isrc;
1374
1375 /* No ithreads for IPIs */
1376 MPASS(!gbi->gbi_ipi);
1377
1378 switch (gbi->gbi_space) {
1379 case GICv5_PPI:
1380 gicv5_enable_intr_action(isrc);
1381 break;
1382 default:
1383 gicv5_enable_intr(dev, isrc);
1384 break;
1385 }
1386 }
1387
1388 static int
gicv5_bind_intr(device_t dev,struct intr_irqsrc * isrc)1389 gicv5_bind_intr(device_t dev, struct intr_irqsrc *isrc)
1390 {
1391 struct gicv5_irqsrc *gi = (struct gicv5_irqsrc *)isrc;
1392 struct gicv5_base_irqsrc *gbi = &gi->gi_isrc;
1393 struct gicv5_irs *irs;
1394 uint64_t cdaff, cdpri;
1395 uint32_t irq;
1396 int cpu, iaffid;
1397
1398 /* IPIs are already bound */
1399 MPASS(!gbi->gbi_ipi);
1400
1401 if (CPU_EMPTY(&isrc->isrc_cpu)) {
1402 irs = gi->gi_irs;
1403 cpu = irs->irs_next_irq_cpu =
1404 intr_irq_next_cpu(irs->irs_next_irq_cpu, &irs->irs_cpus);
1405 } else {
1406 cpu = CPU_FFS(&isrc->isrc_cpu) - 1;
1407 }
1408
1409 MPASS(cpu <= mp_maxid);
1410 iaffid = gicv5_iaffids[cpu];
1411 MPASS(iaffid >= 0);
1412
1413 irq = gbi->gbi_irq;
1414
1415 /* TODO: Where should priority be set? */
1416 cdpri = GIC_CDPRI_PRORITY(GICV5_PRI_LOWEST);
1417 cdpri |= GIC_CDPRI_ID(irq);
1418
1419 cdaff = GIC_CDAFF_IAFFID(iaffid);
1420 cdaff |= GIC_CDAFF_IRM_TARGETED;
1421 cdaff |= GIC_CDAFF_ID(irq);
1422
1423 switch (gbi->gbi_space) {
1424 default:
1425 panic("%s: Invalid space %x", __func__, gbi->gbi_space);
1426 case GICv5_SPI:
1427 cdpri |= GIC_CDPRI_TYPE_SPI;
1428 cdaff |= GIC_CDAFF_TYPE_SPI;
1429 break;
1430 case GICv5_LPI:
1431 cdpri |= GIC_CDPRI_TYPE_LPI;
1432 cdaff |= GIC_CDAFF_TYPE_LPI;
1433 break;
1434 }
1435
1436 gic_cdpri(cdpri);
1437 gic_cdaff(cdaff);
1438 isb();
1439
1440 return (0);
1441 }
1442
1443 #ifdef SMP
1444 static void
gicv5_init_secondary(device_t dev,uint32_t rootnum)1445 gicv5_init_secondary(device_t dev, uint32_t rootnum)
1446 {
1447 struct gicv5_softc *sc;
1448 u_int cpu;
1449
1450 sc = device_get_softc(dev);
1451 cpu = curcpu;
1452
1453 WRITE_SPECIALREG(ICC_PPI_PRIORITYR0_EL1,
1454 ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST));
1455 WRITE_SPECIALREG(ICC_PPI_PRIORITYR1_EL1,
1456 ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST));
1457 WRITE_SPECIALREG(ICC_PPI_PRIORITYR2_EL1,
1458 ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST));
1459 WRITE_SPECIALREG(ICC_PPI_PRIORITYR3_EL1,
1460 ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST));
1461 WRITE_SPECIALREG(ICC_PPI_PRIORITYR4_EL1,
1462 ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST));
1463 WRITE_SPECIALREG(ICC_PPI_PRIORITYR5_EL1,
1464 ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST));
1465 WRITE_SPECIALREG(ICC_PPI_PRIORITYR6_EL1,
1466 ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST));
1467 WRITE_SPECIALREG(ICC_PPI_PRIORITYR7_EL1,
1468 ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST));
1469 WRITE_SPECIALREG(ICC_PPI_PRIORITYR8_EL1,
1470 ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST));
1471 WRITE_SPECIALREG(ICC_PPI_PRIORITYR9_EL1,
1472 ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST));
1473 WRITE_SPECIALREG(ICC_PPI_PRIORITYR10_EL1,
1474 ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST));
1475 WRITE_SPECIALREG(ICC_PPI_PRIORITYR11_EL1,
1476 ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST));
1477 WRITE_SPECIALREG(ICC_PPI_PRIORITYR12_EL1,
1478 ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST));
1479 WRITE_SPECIALREG(ICC_PPI_PRIORITYR13_EL1,
1480 ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST));
1481 WRITE_SPECIALREG(ICC_PPI_PRIORITYR14_EL1,
1482 ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST));
1483 WRITE_SPECIALREG(ICC_PPI_PRIORITYR15_EL1,
1484 ICC_PPI_PRIORITYR_PRIORITY_ALL(GICV5_PRI_LOWEST));
1485
1486 /* Disable all PPIs, then enable as needed */
1487 WRITE_SPECIALREG(ICC_PPI_ENABLER0_EL1, ICC_PPI_ENABLER_NONE);
1488 WRITE_SPECIALREG(ICC_PPI_ENABLER1_EL1, ICC_PPI_ENABLER_NONE);
1489 isb();
1490
1491 /* Set the priority to the lowest value */
1492 WRITE_SPECIALREG(ICC_PCR_EL1, ICC_PCR_PRIORITY_LOWEST);
1493
1494 /* Enable interrupts */
1495 WRITE_SPECIALREG(ICC_CR0_EL1, ICC_CR0_EN);
1496 isb();
1497
1498 /* Enable IPIs */
1499 for (u_int irq = 0; irq < INTR_IPI_COUNT; irq++) {
1500 struct gicv5_base_irqsrc *gbi;
1501
1502 gbi = &sc->gic_ipi_irqs[irq].gi_isrc;
1503 if (intr_isrc_init_on_cpu(&gbi->gbi_isrc, cpu)) {
1504 gicv5_enable_ipi(gbi);
1505 }
1506 }
1507
1508 /* Enable PPIs */
1509 for (u_int irq = 0; irq < GICV5_PPI_COUNT; irq++) {
1510 struct intr_irqsrc *isrc;
1511
1512 isrc = &sc->gic_ppi_irqs[irq].gi_isrc.gbi_isrc;
1513 if (intr_isrc_init_on_cpu(isrc, cpu)) {
1514 gicv5_enable_intr_action(isrc);
1515 }
1516 }
1517
1518 /* Set the priority to the lowest value */
1519 WRITE_SPECIALREG(ICC_PCR_EL1, ICC_PCR_PRIORITY_LOWEST);
1520
1521 /* Enable interrupts */
1522 WRITE_SPECIALREG(ICC_CR0_EL1, ICC_CR0_EN);
1523 isb();
1524 }
1525
1526 static void
gicv5_ipi_send(device_t dev,struct intr_irqsrc * isrc,cpuset_t cpus,u_int ipi)1527 gicv5_ipi_send(device_t dev, struct intr_irqsrc *isrc, cpuset_t cpus,
1528 u_int ipi)
1529 {
1530 struct gicv5_irqsrc *gi = (struct gicv5_irqsrc *)isrc;
1531 uint64_t val;
1532 u_int cpu, irq;
1533
1534 MPASS(gi->gi_isrc.gbi_ipi);
1535
1536 val = 0;
1537 switch (gi->gi_isrc.gbi_space) {
1538 default:
1539 panic("%s: Invalid space: %x", __func__, gi->gi_isrc.gbi_space);
1540 case GICv5_LPI:
1541 val |= GIC_CDPEND_TYPE_LPI;
1542 break;
1543 }
1544 val |= GIC_CDPEND_PENDING_SET;
1545 CPU_FOREACH_ISSET(cpu, &cpus) {
1546 irq = IPI_TO_LPI(gi->gi_isrc.gbi_irq, cpu);
1547 gic_cdpend(val | GIC_CDPEND_ID(irq));
1548 isb();
1549 }
1550 }
1551
1552 static int
gicv5_ipi_setup(device_t dev,u_int ipi,struct intr_irqsrc ** isrcp)1553 gicv5_ipi_setup(device_t dev, u_int ipi, struct intr_irqsrc **isrcp)
1554 {
1555 struct gicv5_softc *sc;
1556 struct gicv5_irqsrc *gi;
1557 u_int irq;
1558 int iaffid;
1559
1560 sc = device_get_softc(dev);
1561 gi = &sc->gic_ipi_irqs[ipi];
1562 MPASS(gi->gi_isrc.gbi_ipi);
1563
1564 for (u_int cpu = 0; cpu <= mp_maxid; cpu++) {
1565 if (CPU_ABSENT(cpu))
1566 continue;
1567 iaffid = gicv5_iaffids[cpu];
1568 KASSERT(iaffid >= 0,
1569 ("%s: No iaffid for cpu %u", __func__, cpu));
1570 CPU_SET(cpu, &gi->gi_isrc.gbi_isrc.isrc_cpu);
1571 irq = IPI_TO_LPI(ipi, cpu);
1572
1573 gic_cdpri(GIC_CDPRI_PRORITY(GICV5_PRI_LOWEST) |
1574 GIC_CDPRI_TYPE_LPI | GIC_CDPRI_ID(irq));
1575 gic_cdaff(GIC_CDAFF_IAFFID(iaffid) | GIC_CDAFF_TYPE_LPI |
1576 GIC_CDAFF_ID(irq));
1577 isb();
1578 }
1579
1580 *isrcp = &gi->gi_isrc.gbi_isrc;
1581 return (0);
1582 }
1583
1584 static cpu_feat_en
gicv5_feat_check(const struct cpu_feat * feat __unused,u_int midr __unused)1585 gicv5_feat_check(const struct cpu_feat *feat __unused, u_int midr __unused)
1586 {
1587 if (gicv5_iaffids == NULL)
1588 return (FEAT_ALWAYS_DISABLE);
1589
1590 return (FEAT_ALWAYS_ENABLE);
1591 }
1592
1593 static bool
gicv5_feat_enable(const struct cpu_feat * feat __unused,cpu_feat_errata errata_status __unused,u_int * errata_list __unused,u_int errata_count __unused)1594 gicv5_feat_enable(const struct cpu_feat *feat __unused,
1595 cpu_feat_errata errata_status __unused, u_int *errata_list __unused,
1596 u_int errata_count __unused)
1597 {
1598 u_int cpu;
1599
1600 /* This is handled by attach */
1601 cpu = curcpu;
1602 if (cpu == 0)
1603 return (true);
1604
1605 MPASS(cpu <= mp_maxid);
1606 MPASS(gicv5_iaffids[cpu] == -1);
1607 gicv5_iaffids[cpu] =
1608 ICC_IAFFIDR_IAFFID_VAL(READ_SPECIALREG(ICC_IAFFIDR_EL1));
1609
1610 return (true);
1611 }
1612
1613 CPU_FEAT(gicv5, "GICv5",
1614 gicv5_feat_check, NULL, gicv5_feat_enable, NULL,
1615 CPU_FEAT_AFTER_DEV | CPU_FEAT_PER_CPU);
1616 #endif
1617