1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2024 Himanshu Chauhan <hchauhan@thechauhan.dev>
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28 #include <sys/param.h>
29 #include <sys/systm.h>
30 #include <sys/bus.h>
31 #include <sys/kernel.h>
32 #include <sys/ktr.h>
33 #include <sys/module.h>
34 #include <sys/proc.h>
35 #include <sys/rman.h>
36 #include <sys/smp.h>
37
38 #include <machine/bus.h>
39 #include <machine/intr.h>
40 #include <machine/riscvreg.h>
41
42 #include <dev/ofw/openfirm.h>
43 #include <dev/ofw/ofw_bus.h>
44 #include <dev/ofw/ofw_bus_subr.h>
45
46 #include "pic_if.h"
47
48 #define APLIC_MAX_IRQS 1023
49
50 /* Smaller priority number means higher priority */
51 #define APLIC_INTR_DEF_PRIO 1
52
53 static pic_disable_intr_t aplic_disable_intr;
54 static pic_enable_intr_t aplic_enable_intr;
55 static pic_map_intr_t aplic_map_intr;
56 static pic_setup_intr_t aplic_setup_intr;
57 static pic_post_ithread_t aplic_post_ithread;
58 static pic_pre_ithread_t aplic_pre_ithread;
59 static pic_bind_intr_t aplic_bind_intr;
60
61 struct aplic_irqsrc {
62 struct intr_irqsrc isrc;
63 u_int irq;
64 };
65
66 struct aplic_softc {
67 device_t dev;
68 struct resource *mem_res;
69 struct resource *irq_res;
70 void *ih;
71 struct aplic_irqsrc isrcs[APLIC_MAX_IRQS + 1];
72 unsigned int hart_indices[MAXCPU];
73 int ndev;
74 };
75
76 #define APLIC_DOMAIN_CFG_IE (1UL << 8) /* Enable domain IRQs */
77 #define APLIC_DOMAIN_CFG_DM (1UL << 2) /* IRQ delivery mode */
78 #define APLIC_DOMAIN_CFG_BE (1UL << 0) /* Endianess */
79
80 #define APLIC_MODE_DIRECT 0 /* Direct delivery mode */
81 #define APLIC_MODE_MSI 1 /* MSI delivery mode */
82
83 #define APLIC_SRC_CFG_DLGT (1UL << 10) /* Source delegation */
84 #define APLIC_SRC_CFG_SM_SHIFT 0
85 #define APLIC_SRC_CFG_SM_MASK (0x7UL << APLIC_SRC_CFG_SM_SHIFT)
86
87 #define APLIC_SRC_CFG_SM_INACTIVE 0 /* APLIC inactive in domain */
88 #define APLIC_SRC_CFG_SM_DETACHED 1 /* Detached from source wire */
89 #define APLIC_SRC_CFG_SM_EDGE_RSE 4 /* Asserted on rising edge */
90 #define APLIC_SRC_CFG_SM_EDGE_FLL 5 /* Asserted on falling edge */
91 #define APLIC_SRC_CFG_SM_LVL_HI 6 /* Asserted when high */
92 #define APLIC_SRC_CFG_SM_LVL_LO 7 /* Asserted when low */
93
94 /* Register offsets in APLIC configuration space */
95 #define APLIC_DOMAIN_CFG 0x0000
96 #define APLIC_SRC_CFG(_idx) (0x0004 + (((_idx) - 1) * 4))
97 #define APLIC_TARGET(_idx) (0x3004 + (((_idx) - 1) * 4))
98 #define APLIC_MMSIADDRCFG 0x1BC0
99 #define APLIC_MMSIADDRCFGH 0x1BC4
100 #define APLIC_SMSIADDRCFG 0x1BC8
101 #define APLIC_SMSIADDRCFGH 0x1BCC
102 #define APLIC_SETIPNUM 0x1CDC
103 #define APLIC_CLRIPNUM 0x1DDC
104 #define APLIC_SETIENUM 0x1EDC
105 #define APLIC_CLRIENUM 0x1FDC
106 #define APLIC_SETIPNUM_LE 0x2000
107 #define APLIC_SETIPNUM_BE 0x2004
108 #define APLIC_GENMSI 0x3000
109 #define APLIC_IDC_BASE 0x4000
110
111 #define APLIC_SETIE_BASE 0x1E00
112 #define APLIC_CLRIE_BASE 0x1F00
113
114 /* Interrupt delivery control structure */
115 #define APLIC_IDC_IDELIVERY_OFFS 0x0000
116 #define APLIC_IDC_IFORCE_OFFS 0x0004
117 #define APLIC_IDC_ITHRESHOLD_OFFS 0x0008
118 #define APLIC_IDC_TOPI_OFFS 0x0018
119 #define APLIC_IDC_CLAIMI_OFFS 0x001C
120
121 #define APLIC_IDC_SZ 0x20
122
123 #define APLIC_IDC_IDELIVERY_DISABLE 0
124 #define APLIC_IDC_IDELIVERY_ENABLE 1
125 #define APLIC_IDC_ITHRESHOLD_DISABLE 0
126
127 #define APLIC_IDC_CLAIMI_PRIO_MASK 0xff
128 #define APLIC_IDC_CLAIMI_IRQ_SHIFT 16
129 #define APLIC_IDC_CLAIMI_IRQ_MASK 0x3ff
130
131 #define APLIC_IDC_CLAIMI_IRQ(_claimi) \
132 (((_claimi) >> APLIC_IDC_CLAIMI_IRQ_SHIFT) \
133 & APLIC_IDC_CLAIMI_IRQ_MASK)
134
135 #define APLIC_IDC_CLAIMI_PRIO(_claimi) ((_claimi) & APLIC_IDC_CLAIMI_PRIO_MASK)
136
137 #define APLIC_IDC_REG(_sc, _cpu, _field) \
138 (APLIC_IDC_BASE + APLIC_IDC_##_field##_OFFS + \
139 ((_sc->hart_indices[_cpu]) * APLIC_IDC_SZ))
140
141 #define APLIC_IDC_IDELIVERY(_sc, _cpu) \
142 APLIC_IDC_REG(_sc, _cpu, IDELIVERY)
143
144 #define APLIC_IDC_IFORCE(_sc, _cpu) \
145 APLIC_IDC_REG(_sc, _cpu, IFORCE)
146
147 #define APLIC_IDC_ITHRESHOLD(_sc, _cpu) \
148 APLIC_IDC_REG(_sc, _cpu, ITHRESHOLD)
149
150 #define APLIC_IDC_TOPI(_sc, _cpu) \
151 APLIC_IDC_REG(_sc, _cpu, TOPI)
152
153 #define APLIC_IDC_CLAIMI(_sc, _cpu) \
154 APLIC_IDC_REG(_sc, _cpu, CLAIMI)
155
156 #define APLIC_MK_IRQ_TARGET(_sc, _cpu, _prio) \
157 (_sc->hart_indices[_cpu] << 18 | ((_prio) & 0xff))
158
159 #define aplic_read(sc, reg) bus_read_4(sc->mem_res, (reg))
160 #define aplic_write(sc, reg, val) bus_write_4(sc->mem_res, (reg), (val))
161
162 static u_int aplic_irq_cpu;
163
164 static inline int
riscv_cpu_to_hartid(int cpu)165 riscv_cpu_to_hartid(int cpu)
166 {
167 return pcpu_find(cpu)->pc_hart;
168 }
169
170 static inline int
riscv_hartid_to_cpu(int hartid)171 riscv_hartid_to_cpu(int hartid)
172 {
173 int cpu;
174
175 CPU_FOREACH(cpu) {
176 if (riscv_cpu_to_hartid(cpu) == hartid)
177 return cpu;
178 }
179
180 return (-1);
181 }
182
183 static int
fdt_get_hartid(device_t dev,phandle_t aplic)184 fdt_get_hartid(device_t dev, phandle_t aplic)
185 {
186 int hartid;
187
188 /* Check the interrupt controller layout. */
189 if (OF_searchencprop(aplic, "#interrupt-cells", &hartid,
190 sizeof(hartid)) == -1) {
191 device_printf(dev,
192 "Could not find #interrupt-cells for phandle %u\n", aplic);
193 return (-1);
194 }
195
196 /*
197 * The parent of the interrupt-controller is the CPU we are
198 * interested in, so search for its hart ID.
199 */
200 if (OF_searchencprop(OF_parent(aplic), "reg", (pcell_t *)&hartid,
201 sizeof(hartid)) == -1) {
202 device_printf(dev, "Could not find hartid\n");
203 return (-1);
204 }
205
206 return (hartid);
207 }
208
209 static inline void
aplic_irq_dispatch(struct aplic_softc * sc,u_int irq,u_int prio,struct trapframe * tf)210 aplic_irq_dispatch(struct aplic_softc *sc, u_int irq, u_int prio,
211 struct trapframe *tf)
212 {
213 struct aplic_irqsrc *src;
214
215 src = &sc->isrcs[irq];
216
217 if (intr_isrc_dispatch(&src->isrc, tf) != 0)
218 if (bootverbose)
219 device_printf(sc->dev, "Stray irq %u detected\n", irq);
220 }
221
222 static int
aplic_intr(void * arg)223 aplic_intr(void *arg)
224 {
225 struct aplic_softc *sc;
226 struct trapframe *tf;
227 uint32_t claimi;
228 u_int prio, irq;
229 int cpu;
230
231 sc = arg;
232 cpu = PCPU_GET(cpuid);
233
234 /* Claim all pending interrupts. */
235 while ((claimi = aplic_read(sc, APLIC_IDC_CLAIMI(sc, cpu))) != 0) {
236 prio = APLIC_IDC_CLAIMI_PRIO(claimi);
237 irq = APLIC_IDC_CLAIMI_IRQ(claimi);
238
239 KASSERT((irq != 0), ("Invalid IRQ 0"));
240
241 tf = curthread->td_intr_frame;
242 aplic_irq_dispatch(sc, irq, prio, tf);
243 }
244
245 return (FILTER_HANDLED);
246 }
247
248 static void
aplic_disable_intr(device_t dev,struct intr_irqsrc * isrc)249 aplic_disable_intr(device_t dev, struct intr_irqsrc *isrc)
250 {
251 struct aplic_softc *sc;
252 struct aplic_irqsrc *src;
253
254 sc = device_get_softc(dev);
255 src = (struct aplic_irqsrc *)isrc;
256
257 /* Disable the interrupt source */
258 aplic_write(sc, APLIC_CLRIENUM, src->irq);
259 }
260
261 static void
aplic_enable_intr(device_t dev,struct intr_irqsrc * isrc)262 aplic_enable_intr(device_t dev, struct intr_irqsrc *isrc)
263 {
264 struct aplic_softc *sc;
265 struct aplic_irqsrc *src;
266
267 sc = device_get_softc(dev);
268 src = (struct aplic_irqsrc *)isrc;
269
270 /* Enable the interrupt source */
271 aplic_write(sc, APLIC_SETIENUM, src->irq);
272 }
273
274 static int
aplic_map_intr(device_t dev,struct intr_map_data * data,struct intr_irqsrc ** isrcp)275 aplic_map_intr(device_t dev, struct intr_map_data *data,
276 struct intr_irqsrc **isrcp)
277 {
278 struct intr_map_data_fdt *daf;
279 struct aplic_softc *sc;
280
281 sc = device_get_softc(dev);
282
283 if (data->type != INTR_MAP_DATA_FDT)
284 return (ENOTSUP);
285
286 daf = (struct intr_map_data_fdt *)data;
287 if (daf->ncells != 2 || daf->cells[0] > sc->ndev) {
288 device_printf(dev, "Invalid cell data\n");
289 return (EINVAL);
290 }
291
292 *isrcp = &sc->isrcs[daf->cells[0]].isrc;
293
294 return (0);
295 }
296
297 static int
aplic_probe(device_t dev)298 aplic_probe(device_t dev)
299 {
300 if (!ofw_bus_status_okay(dev))
301 return (ENXIO);
302
303 if (!ofw_bus_is_compatible(dev, "riscv,aplic"))
304 return (ENXIO);
305
306 device_set_desc(dev, "Advanced Platform-Level Interrupt Controller");
307
308 return (BUS_PROBE_DEFAULT);
309 }
310
311 /*
312 * Setup APLIC in direct mode.
313 */
314 static int
aplic_setup_direct_mode(device_t dev)315 aplic_setup_direct_mode(device_t dev)
316 {
317 struct aplic_irqsrc *isrcs;
318 struct aplic_softc *sc;
319 struct intr_pic *pic;
320 const char *name;
321 phandle_t node, xref, iparent;
322 pcell_t *cells, cell;
323 int error = ENXIO;
324 u_int irq;
325 int cpu, hartid, rid, i, nintr, idc;
326 device_t rootdev;
327
328 sc = device_get_softc(dev);
329 node = ofw_bus_get_node(dev);
330
331 sc->dev = dev;
332
333 if ((OF_getencprop(node, "riscv,num-sources", &sc->ndev,
334 sizeof(sc->ndev))) < 0) {
335 device_printf(dev, "Error: could not get number of devices\n");
336 return (error);
337 }
338
339 if (sc->ndev > APLIC_MAX_IRQS) {
340 device_printf(dev, "Error: invalid ndev (%d)\n", sc->ndev);
341 return (error);
342 }
343
344 /* Request memory resources */
345 rid = 0;
346 sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
347 RF_ACTIVE);
348 if (sc->mem_res == NULL) {
349 device_printf(dev,
350 "Error: could not allocate memory resources\n");
351 return (error);
352 }
353
354 /* Set APLIC in direct mode and enable all interrupts */
355 aplic_write(sc, APLIC_DOMAIN_CFG,
356 APLIC_MODE_DIRECT | APLIC_DOMAIN_CFG_IE);
357
358 /* Register the interrupt sources */
359 isrcs = sc->isrcs;
360 name = device_get_nameunit(sc->dev);
361 for (irq = 1; irq <= sc->ndev; irq++) {
362 isrcs[irq].irq = irq;
363
364 error = intr_isrc_register(&isrcs[irq].isrc, sc->dev,
365 0, "%s,%u", name, irq);
366 if (error != 0)
367 goto fail;
368
369 aplic_write(sc, APLIC_SRC_CFG(irq),
370 APLIC_SRC_CFG_SM_DETACHED);
371 }
372
373 nintr = OF_getencprop_alloc_multi(node, "interrupts-extended",
374 sizeof(uint32_t), (void **)&cells);
375 if (nintr <= 0) {
376 device_printf(dev, "Could not read interrupts-extended\n");
377 goto fail;
378 }
379
380 /* interrupts-extended is a list of phandles and interrupt types. */
381 for (i = 0, idc = 0; i < nintr; i += 2, idc++) {
382 /* Skip M-mode external interrupts */
383 if (cells[i + 1] != IRQ_EXTERNAL_SUPERVISOR)
384 continue;
385
386 /* Get the hart ID from the CLIC's phandle. */
387 hartid = fdt_get_hartid(dev, OF_node_from_xref(cells[i]));
388 if (hartid < 0) {
389 OF_prop_free(cells);
390 goto fail;
391 }
392
393 /* Get the corresponding cpuid. */
394 cpu = riscv_hartid_to_cpu(hartid);
395 if (cpu < 0) {
396 device_printf(dev, "Invalid cpu for hart %d\n", hartid);
397 OF_prop_free(cells);
398 goto fail;
399 }
400
401 sc->hart_indices[cpu] = idc;
402 }
403 OF_prop_free(cells);
404
405 /* Turn off the interrupt delivery for all CPUs within or out domain */
406 CPU_FOREACH(cpu) {
407 aplic_write(sc, APLIC_IDC_IDELIVERY(sc, cpu),
408 APLIC_IDC_IDELIVERY_DISABLE);
409 aplic_write(sc, APLIC_IDC_ITHRESHOLD(sc, cpu),
410 APLIC_IDC_ITHRESHOLD_DISABLE);
411 }
412
413 rootdev = intr_irq_root_device(INTR_ROOT_IRQ);
414 iparent = OF_xref_from_node(ofw_bus_get_node(rootdev));
415 cell = IRQ_EXTERNAL_SUPERVISOR;
416 irq = ofw_bus_map_intr(dev, iparent, 1, &cell);
417 error = bus_set_resource(dev, SYS_RES_IRQ, 0, irq, 1);
418 if (error != 0) {
419 device_printf(dev, "Unable to register IRQ resource\n");
420 return (ENXIO);
421 }
422
423 rid = 0;
424 sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
425 RF_ACTIVE);
426 if (sc->irq_res == NULL) {
427 device_printf(dev,
428 "Error: could not allocate IRQ resources\n");
429 return (ENXIO);
430 }
431
432 xref = OF_xref_from_node(node);
433 pic = intr_pic_register(sc->dev, xref);
434 if (pic == NULL) {
435 error = ENXIO;
436 goto fail;
437 }
438
439 error = bus_setup_intr(dev, sc->irq_res, INTR_TYPE_CLK,
440 aplic_intr, NULL, sc, &sc->ih);
441 if (error != 0) {
442 device_printf(dev, "Unable to setup IRQ resource\n");
443 return (ENXIO);
444 }
445
446 fail:
447 return (error);
448 }
449
450 static int
aplic_attach(device_t dev)451 aplic_attach(device_t dev)
452 {
453 int rc;
454
455 /* APLIC with IMSIC on hart is not supported */
456 if (ofw_bus_has_prop(dev, "msi-parent")) {
457 device_printf(dev, "APLIC with IMSIC is unsupported\n");
458 return (ENXIO);
459 }
460
461 rc = aplic_setup_direct_mode(dev);
462
463 return (rc);
464 }
465
466 static void
aplic_pre_ithread(device_t dev,struct intr_irqsrc * isrc)467 aplic_pre_ithread(device_t dev, struct intr_irqsrc *isrc)
468 {
469 aplic_disable_intr(dev, isrc);
470 }
471
472 static void
aplic_post_ithread(device_t dev,struct intr_irqsrc * isrc)473 aplic_post_ithread(device_t dev, struct intr_irqsrc *isrc)
474 {
475 aplic_enable_intr(dev, isrc);
476 }
477
478 static int
aplic_setup_intr(device_t dev,struct intr_irqsrc * isrc,struct resource * res,struct intr_map_data * data)479 aplic_setup_intr(device_t dev, struct intr_irqsrc *isrc, struct resource *res,
480 struct intr_map_data *data)
481 {
482 struct aplic_irqsrc *src;
483 struct aplic_softc *sc;
484
485 CPU_ZERO(&isrc->isrc_cpu);
486
487 sc = device_get_softc(dev);
488 src = (struct aplic_irqsrc *)isrc;
489
490 aplic_write(sc, APLIC_SRC_CFG(src->irq), APLIC_SRC_CFG_SM_EDGE_RSE);
491
492 /*
493 * In uniprocessor system, bind_intr will not be called.
494 * So bind the interrupt on this CPU. If secondary CPUs
495 * are present, then bind_intr will be called again and
496 * interrupts will rebind to those CPUs.
497 */
498 aplic_bind_intr(dev, isrc);
499
500 return (0);
501 }
502
503 static int
aplic_bind_intr(device_t dev,struct intr_irqsrc * isrc)504 aplic_bind_intr(device_t dev, struct intr_irqsrc *isrc)
505 {
506 struct aplic_softc *sc;
507 struct aplic_irqsrc *src;
508 uint32_t cpu, hartid;
509
510 sc = device_get_softc(dev);
511 src = (struct aplic_irqsrc *)isrc;
512
513 /* Disable the interrupt source */
514 aplic_write(sc, APLIC_CLRIENUM, src->irq);
515
516 if (CPU_EMPTY(&isrc->isrc_cpu)) {
517 cpu = aplic_irq_cpu = intr_irq_next_cpu(aplic_irq_cpu,
518 &all_cpus);
519 CPU_SETOF(cpu, &isrc->isrc_cpu);
520 } else {
521 cpu = CPU_FFS(&isrc->isrc_cpu) - 1;
522 }
523
524 hartid = riscv_cpu_to_hartid(cpu);
525
526 if (bootverbose)
527 device_printf(dev, "Bind irq %d to cpu%d (hart %d)\n", src->irq,
528 cpu, hartid);
529
530 aplic_write(sc, APLIC_TARGET(src->irq),
531 APLIC_MK_IRQ_TARGET(sc, cpu, APLIC_INTR_DEF_PRIO));
532 aplic_write(sc, APLIC_IDC_IDELIVERY(sc, cpu),
533 APLIC_IDC_IDELIVERY_ENABLE);
534 aplic_enable_intr(dev, isrc);
535
536 return (0);
537 }
538
539 static device_method_t aplic_methods[] = {
540 DEVMETHOD(device_probe, aplic_probe),
541 DEVMETHOD(device_attach, aplic_attach),
542
543 DEVMETHOD(pic_disable_intr, aplic_disable_intr),
544 DEVMETHOD(pic_enable_intr, aplic_enable_intr),
545 DEVMETHOD(pic_map_intr, aplic_map_intr),
546 DEVMETHOD(pic_pre_ithread, aplic_pre_ithread),
547 DEVMETHOD(pic_post_ithread, aplic_post_ithread),
548 DEVMETHOD(pic_post_filter, aplic_post_ithread),
549 DEVMETHOD(pic_setup_intr, aplic_setup_intr),
550 DEVMETHOD(pic_bind_intr, aplic_bind_intr),
551
552 DEVMETHOD_END
553 };
554
555 DEFINE_CLASS_0(aplic, aplic_driver, aplic_methods, sizeof(struct aplic_softc));
556
557 EARLY_DRIVER_MODULE(aplic, simplebus, aplic_driver, 0, 0,
558 BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE);
559