1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2017 SiFive
4 * Copyright (C) 2018 Christoph Hellwig
5 */
6 #define pr_fmt(fmt) "riscv-plic: " fmt
7 #include <linux/acpi.h>
8 #include <linux/cpu.h>
9 #include <linux/interrupt.h>
10 #include <linux/io.h>
11 #include <linux/irq.h>
12 #include <linux/irqchip.h>
13 #include <linux/irqchip/chained_irq.h>
14 #include <linux/irqdomain.h>
15 #include <linux/module.h>
16 #include <linux/of.h>
17 #include <linux/of_address.h>
18 #include <linux/of_irq.h>
19 #include <linux/platform_device.h>
20 #include <linux/spinlock.h>
21 #include <linux/syscore_ops.h>
22 #include <asm/smp.h>
23
24 /*
25 * This driver implements a version of the RISC-V PLIC with the actual layout
26 * specified in chapter 8 of the SiFive U5 Coreplex Series Manual:
27 *
28 * https://static.dev.sifive.com/U54-MC-RVCoreIP.pdf
29 *
30 * The largest number supported by devices marked as 'sifive,plic-1.0.0', is
31 * 1024, of which device 0 is defined as non-existent by the RISC-V Privileged
32 * Spec.
33 */
34
35 #define MAX_DEVICES 1024
36 #define MAX_CONTEXTS 15872
37
38 /*
39 * Each interrupt source has a priority register associated with it.
40 * We always hardwire it to one in Linux.
41 */
42 #define PRIORITY_BASE 0
43 #define PRIORITY_PER_ID 4
44
45 /*
46 * Each hart context has a vector of interrupt enable bits associated with it.
47 * There's one bit for each interrupt source.
48 */
49 #define CONTEXT_ENABLE_BASE 0x2000
50 #define CONTEXT_ENABLE_SIZE 0x80
51
52 #define PENDING_BASE 0x1000
53
54 /*
55 * Each hart context has a set of control registers associated with it. Right
56 * now there's only two: a source priority threshold over which the hart will
57 * take an interrupt, and a register to claim interrupts.
58 */
59 #define CONTEXT_BASE 0x200000
60 #define CONTEXT_SIZE 0x1000
61 #define CONTEXT_THRESHOLD 0x00
62 #define CONTEXT_CLAIM 0x04
63
64 #define PLIC_DISABLE_THRESHOLD 0x7
65 #define PLIC_ENABLE_THRESHOLD 0
66
67 #define PLIC_QUIRK_EDGE_INTERRUPT 0
68 #define PLIC_QUIRK_CP100_CLAIM_REGISTER_ERRATUM 1
69
70 struct plic_priv {
71 struct fwnode_handle *fwnode;
72 struct cpumask lmask;
73 struct irq_domain *irqdomain;
74 void __iomem *regs;
75 unsigned long plic_quirks;
76 unsigned int nr_irqs;
77 unsigned long *prio_save;
78 u32 gsi_base;
79 int acpi_plic_id;
80 };
81
82 struct plic_handler {
83 bool present;
84 void __iomem *hart_base;
85 /*
86 * Protect mask operations on the registers given that we can't
87 * assume atomic memory operations work on them.
88 */
89 raw_spinlock_t enable_lock;
90 void __iomem *enable_base;
91 u32 *enable_save;
92 struct plic_priv *priv;
93 };
94 static int plic_parent_irq __ro_after_init;
95 static bool plic_global_setup_done __ro_after_init;
96 static DEFINE_PER_CPU(struct plic_handler, plic_handlers);
97
98 static int plic_irq_set_type(struct irq_data *d, unsigned int type);
99
__plic_toggle(struct plic_handler * handler,int hwirq,int enable)100 static void __plic_toggle(struct plic_handler *handler, int hwirq, int enable)
101 {
102 u32 __iomem *base = handler->enable_base;
103 u32 hwirq_mask = 1 << (hwirq % 32);
104 int group = hwirq / 32;
105 u32 value;
106
107 value = readl(base + group);
108
109 if (enable)
110 value |= hwirq_mask;
111 else
112 value &= ~hwirq_mask;
113
114 handler->enable_save[group] = value;
115 writel(value, base + group);
116 }
117
plic_toggle(struct plic_handler * handler,int hwirq,int enable)118 static void plic_toggle(struct plic_handler *handler, int hwirq, int enable)
119 {
120 unsigned long flags;
121
122 raw_spin_lock_irqsave(&handler->enable_lock, flags);
123 __plic_toggle(handler, hwirq, enable);
124 raw_spin_unlock_irqrestore(&handler->enable_lock, flags);
125 }
126
plic_irq_toggle(const struct cpumask * mask,struct irq_data * d,int enable)127 static inline void plic_irq_toggle(const struct cpumask *mask,
128 struct irq_data *d, int enable)
129 {
130 int cpu;
131
132 for_each_cpu(cpu, mask) {
133 struct plic_handler *handler = per_cpu_ptr(&plic_handlers, cpu);
134
135 plic_toggle(handler, d->hwirq, enable);
136 }
137 }
138
plic_irq_unmask(struct irq_data * d)139 static void plic_irq_unmask(struct irq_data *d)
140 {
141 struct plic_priv *priv = irq_data_get_irq_chip_data(d);
142
143 writel(1, priv->regs + PRIORITY_BASE + d->hwirq * PRIORITY_PER_ID);
144 }
145
plic_irq_mask(struct irq_data * d)146 static void plic_irq_mask(struct irq_data *d)
147 {
148 struct plic_priv *priv = irq_data_get_irq_chip_data(d);
149
150 writel(0, priv->regs + PRIORITY_BASE + d->hwirq * PRIORITY_PER_ID);
151 }
152
plic_irq_enable(struct irq_data * d)153 static void plic_irq_enable(struct irq_data *d)
154 {
155 plic_irq_toggle(irq_data_get_effective_affinity_mask(d), d, 1);
156 plic_irq_unmask(d);
157 }
158
plic_irq_disable(struct irq_data * d)159 static void plic_irq_disable(struct irq_data *d)
160 {
161 plic_irq_toggle(irq_data_get_effective_affinity_mask(d), d, 0);
162 }
163
plic_irq_eoi(struct irq_data * d)164 static void plic_irq_eoi(struct irq_data *d)
165 {
166 struct plic_handler *handler = this_cpu_ptr(&plic_handlers);
167
168 if (unlikely(irqd_irq_disabled(d))) {
169 plic_toggle(handler, d->hwirq, 1);
170 writel(d->hwirq, handler->hart_base + CONTEXT_CLAIM);
171 plic_toggle(handler, d->hwirq, 0);
172 } else {
173 writel(d->hwirq, handler->hart_base + CONTEXT_CLAIM);
174 }
175 }
176
177 #ifdef CONFIG_SMP
plic_set_affinity(struct irq_data * d,const struct cpumask * mask_val,bool force)178 static int plic_set_affinity(struct irq_data *d,
179 const struct cpumask *mask_val, bool force)
180 {
181 unsigned int cpu;
182 struct plic_priv *priv = irq_data_get_irq_chip_data(d);
183
184 if (force)
185 cpu = cpumask_first_and(&priv->lmask, mask_val);
186 else
187 cpu = cpumask_first_and_and(&priv->lmask, mask_val, cpu_online_mask);
188
189 if (cpu >= nr_cpu_ids)
190 return -EINVAL;
191
192 /* Invalidate the original routing entry */
193 plic_irq_toggle(irq_data_get_effective_affinity_mask(d), d, 0);
194
195 irq_data_update_effective_affinity(d, cpumask_of(cpu));
196
197 /* Setting the new routing entry if irq is enabled */
198 if (!irqd_irq_disabled(d))
199 plic_irq_toggle(irq_data_get_effective_affinity_mask(d), d, 1);
200
201 return IRQ_SET_MASK_OK_DONE;
202 }
203 #endif
204
205 static struct irq_chip plic_edge_chip = {
206 .name = "SiFive PLIC",
207 .irq_enable = plic_irq_enable,
208 .irq_disable = plic_irq_disable,
209 .irq_ack = plic_irq_eoi,
210 .irq_mask = plic_irq_mask,
211 .irq_unmask = plic_irq_unmask,
212 #ifdef CONFIG_SMP
213 .irq_set_affinity = plic_set_affinity,
214 #endif
215 .irq_set_type = plic_irq_set_type,
216 .flags = IRQCHIP_SKIP_SET_WAKE |
217 IRQCHIP_AFFINITY_PRE_STARTUP,
218 };
219
220 static struct irq_chip plic_chip = {
221 .name = "SiFive PLIC",
222 .irq_enable = plic_irq_enable,
223 .irq_disable = plic_irq_disable,
224 .irq_mask = plic_irq_mask,
225 .irq_unmask = plic_irq_unmask,
226 .irq_eoi = plic_irq_eoi,
227 #ifdef CONFIG_SMP
228 .irq_set_affinity = plic_set_affinity,
229 #endif
230 .irq_set_type = plic_irq_set_type,
231 .flags = IRQCHIP_SKIP_SET_WAKE |
232 IRQCHIP_AFFINITY_PRE_STARTUP,
233 };
234
plic_irq_set_type(struct irq_data * d,unsigned int type)235 static int plic_irq_set_type(struct irq_data *d, unsigned int type)
236 {
237 struct plic_priv *priv = irq_data_get_irq_chip_data(d);
238
239 if (!test_bit(PLIC_QUIRK_EDGE_INTERRUPT, &priv->plic_quirks))
240 return IRQ_SET_MASK_OK_NOCOPY;
241
242 switch (type) {
243 case IRQ_TYPE_EDGE_RISING:
244 irq_set_chip_handler_name_locked(d, &plic_edge_chip,
245 handle_edge_irq, NULL);
246 break;
247 case IRQ_TYPE_LEVEL_HIGH:
248 irq_set_chip_handler_name_locked(d, &plic_chip,
249 handle_fasteoi_irq, NULL);
250 break;
251 default:
252 return -EINVAL;
253 }
254
255 return IRQ_SET_MASK_OK;
256 }
257
plic_irq_suspend(void * data)258 static int plic_irq_suspend(void *data)
259 {
260 struct plic_priv *priv;
261
262 priv = per_cpu_ptr(&plic_handlers, smp_processor_id())->priv;
263
264 /* irq ID 0 is reserved */
265 for (unsigned int i = 1; i < priv->nr_irqs; i++) {
266 __assign_bit(i, priv->prio_save,
267 readl(priv->regs + PRIORITY_BASE + i * PRIORITY_PER_ID));
268 }
269
270 return 0;
271 }
272
plic_irq_resume(void * data)273 static void plic_irq_resume(void *data)
274 {
275 unsigned int i, index, cpu;
276 unsigned long flags;
277 u32 __iomem *reg;
278 struct plic_priv *priv;
279
280 priv = per_cpu_ptr(&plic_handlers, smp_processor_id())->priv;
281
282 /* irq ID 0 is reserved */
283 for (i = 1; i < priv->nr_irqs; i++) {
284 index = BIT_WORD(i);
285 writel((priv->prio_save[index] & BIT_MASK(i)) ? 1 : 0,
286 priv->regs + PRIORITY_BASE + i * PRIORITY_PER_ID);
287 }
288
289 for_each_present_cpu(cpu) {
290 struct plic_handler *handler = per_cpu_ptr(&plic_handlers, cpu);
291
292 if (!handler->present)
293 continue;
294
295 raw_spin_lock_irqsave(&handler->enable_lock, flags);
296 for (i = 0; i < DIV_ROUND_UP(priv->nr_irqs, 32); i++) {
297 reg = handler->enable_base + i * sizeof(u32);
298 writel(handler->enable_save[i], reg);
299 }
300 raw_spin_unlock_irqrestore(&handler->enable_lock, flags);
301 }
302 }
303
304 static const struct syscore_ops plic_irq_syscore_ops = {
305 .suspend = plic_irq_suspend,
306 .resume = plic_irq_resume,
307 };
308
309 static struct syscore plic_irq_syscore = {
310 .ops = &plic_irq_syscore_ops,
311 };
312
plic_irqdomain_map(struct irq_domain * d,unsigned int irq,irq_hw_number_t hwirq)313 static int plic_irqdomain_map(struct irq_domain *d, unsigned int irq,
314 irq_hw_number_t hwirq)
315 {
316 struct plic_priv *priv = d->host_data;
317
318 irq_domain_set_info(d, irq, hwirq, &plic_chip, d->host_data,
319 handle_fasteoi_irq, NULL, NULL);
320 irq_set_noprobe(irq);
321 irq_set_affinity(irq, &priv->lmask);
322 return 0;
323 }
324
plic_irq_domain_translate(struct irq_domain * d,struct irq_fwspec * fwspec,unsigned long * hwirq,unsigned int * type)325 static int plic_irq_domain_translate(struct irq_domain *d,
326 struct irq_fwspec *fwspec,
327 unsigned long *hwirq,
328 unsigned int *type)
329 {
330 struct plic_priv *priv = d->host_data;
331
332 /* For DT, gsi_base is always zero. */
333 if (fwspec->param[0] >= priv->gsi_base)
334 fwspec->param[0] = fwspec->param[0] - priv->gsi_base;
335
336 if (test_bit(PLIC_QUIRK_EDGE_INTERRUPT, &priv->plic_quirks))
337 return irq_domain_translate_twocell(d, fwspec, hwirq, type);
338
339 return irq_domain_translate_onecell(d, fwspec, hwirq, type);
340 }
341
plic_irq_domain_alloc(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs,void * arg)342 static int plic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
343 unsigned int nr_irqs, void *arg)
344 {
345 int i, ret;
346 irq_hw_number_t hwirq;
347 unsigned int type;
348 struct irq_fwspec *fwspec = arg;
349
350 ret = plic_irq_domain_translate(domain, fwspec, &hwirq, &type);
351 if (ret)
352 return ret;
353
354 for (i = 0; i < nr_irqs; i++) {
355 ret = plic_irqdomain_map(domain, virq + i, hwirq + i);
356 if (ret)
357 return ret;
358 }
359
360 return 0;
361 }
362
363 static const struct irq_domain_ops plic_irqdomain_ops = {
364 .translate = plic_irq_domain_translate,
365 .alloc = plic_irq_domain_alloc,
366 .free = irq_domain_free_irqs_top,
367 };
368
369 /*
370 * Handling an interrupt is a two-step process: first you claim the interrupt
371 * by reading the claim register, then you complete the interrupt by writing
372 * that source ID back to the same claim register. This automatically enables
373 * and disables the interrupt, so there's nothing else to do.
374 */
plic_handle_irq(struct irq_desc * desc)375 static void plic_handle_irq(struct irq_desc *desc)
376 {
377 struct plic_handler *handler = this_cpu_ptr(&plic_handlers);
378 struct irq_chip *chip = irq_desc_get_chip(desc);
379 void __iomem *claim = handler->hart_base + CONTEXT_CLAIM;
380 irq_hw_number_t hwirq;
381
382 WARN_ON_ONCE(!handler->present);
383
384 chained_irq_enter(chip, desc);
385
386 while ((hwirq = readl(claim))) {
387 int err = generic_handle_domain_irq(handler->priv->irqdomain,
388 hwirq);
389 if (unlikely(err)) {
390 pr_warn_ratelimited("%pfwP: can't find mapping for hwirq %lu\n",
391 handler->priv->fwnode, hwirq);
392 }
393 }
394
395 chained_irq_exit(chip, desc);
396 }
397
cp100_isolate_pending_irq(int nr_irq_groups,struct plic_handler * handler)398 static u32 cp100_isolate_pending_irq(int nr_irq_groups, struct plic_handler *handler)
399 {
400 u32 __iomem *pending = handler->priv->regs + PENDING_BASE;
401 u32 __iomem *enable = handler->enable_base;
402 u32 pending_irqs = 0;
403 int i, j;
404
405 /* Look for first pending interrupt */
406 for (i = 0; i < nr_irq_groups; i++) {
407 /* Any pending interrupts would be annihilated, so skip checking them */
408 if (!handler->enable_save[i])
409 continue;
410
411 pending_irqs = handler->enable_save[i] & readl_relaxed(pending + i);
412 if (pending_irqs)
413 break;
414 }
415
416 if (!pending_irqs)
417 return 0;
418
419 /* Isolate lowest set bit */
420 pending_irqs &= -pending_irqs;
421
422 /* Disable all interrupts but the first pending one */
423 for (j = 0; j < nr_irq_groups; j++) {
424 u32 new_mask = j == i ? pending_irqs : 0;
425
426 if (new_mask != handler->enable_save[j])
427 writel_relaxed(new_mask, enable + j);
428 }
429 return pending_irqs;
430 }
431
cp100_get_hwirq(struct plic_handler * handler,void __iomem * claim)432 static irq_hw_number_t cp100_get_hwirq(struct plic_handler *handler, void __iomem *claim)
433 {
434 int nr_irq_groups = DIV_ROUND_UP(handler->priv->nr_irqs, 32);
435 u32 __iomem *enable = handler->enable_base;
436 irq_hw_number_t hwirq = 0;
437 u32 iso_mask;
438 int i;
439
440 guard(raw_spinlock)(&handler->enable_lock);
441
442 /* Existing enable state is already cached in enable_save */
443 iso_mask = cp100_isolate_pending_irq(nr_irq_groups, handler);
444 if (!iso_mask)
445 return 0;
446
447 /*
448 * Interrupts delievered to hardware still become pending, but only
449 * interrupts that are both pending and enabled can be claimed.
450 * Clearing the enable bit for all interrupts but the first pending
451 * one avoids a hardware bug that occurs during read from the claim
452 * register with more than one eligible interrupt.
453 */
454 hwirq = readl(claim);
455
456 /* Restore previous state */
457 for (i = 0; i < nr_irq_groups; i++) {
458 u32 written = i == hwirq / 32 ? iso_mask : 0;
459 u32 stored = handler->enable_save[i];
460
461 if (stored != written)
462 writel_relaxed(stored, enable + i);
463 }
464 return hwirq;
465 }
466
plic_handle_irq_cp100(struct irq_desc * desc)467 static void plic_handle_irq_cp100(struct irq_desc *desc)
468 {
469 struct plic_handler *handler = this_cpu_ptr(&plic_handlers);
470 struct irq_chip *chip = irq_desc_get_chip(desc);
471 void __iomem *claim = handler->hart_base + CONTEXT_CLAIM;
472 irq_hw_number_t hwirq;
473
474 WARN_ON_ONCE(!handler->present);
475
476 chained_irq_enter(chip, desc);
477
478 while ((hwirq = cp100_get_hwirq(handler, claim))) {
479 int err = generic_handle_domain_irq(handler->priv->irqdomain, hwirq);
480
481 if (unlikely(err)) {
482 pr_warn_ratelimited("%pfwP: can't find mapping for hwirq %lu\n",
483 handler->priv->fwnode, hwirq);
484 }
485 }
486
487 chained_irq_exit(chip, desc);
488 }
489
plic_set_threshold(struct plic_handler * handler,u32 threshold)490 static void plic_set_threshold(struct plic_handler *handler, u32 threshold)
491 {
492 /* priority must be > threshold to trigger an interrupt */
493 writel(threshold, handler->hart_base + CONTEXT_THRESHOLD);
494 }
495
plic_dying_cpu(unsigned int cpu)496 static int plic_dying_cpu(unsigned int cpu)
497 {
498 if (plic_parent_irq)
499 disable_percpu_irq(plic_parent_irq);
500
501 return 0;
502 }
503
plic_starting_cpu(unsigned int cpu)504 static int plic_starting_cpu(unsigned int cpu)
505 {
506 struct plic_handler *handler = this_cpu_ptr(&plic_handlers);
507
508 if (plic_parent_irq)
509 enable_percpu_irq(plic_parent_irq,
510 irq_get_trigger_type(plic_parent_irq));
511 else
512 pr_warn("%pfwP: cpu%d: parent irq not available\n",
513 handler->priv->fwnode, cpu);
514 plic_set_threshold(handler, PLIC_ENABLE_THRESHOLD);
515
516 return 0;
517 }
518
519 static const struct of_device_id plic_match[] = {
520 { .compatible = "sifive,plic-1.0.0" },
521 { .compatible = "riscv,plic0" },
522 { .compatible = "andestech,nceplic100",
523 .data = (const void *)BIT(PLIC_QUIRK_EDGE_INTERRUPT) },
524 { .compatible = "thead,c900-plic",
525 .data = (const void *)BIT(PLIC_QUIRK_EDGE_INTERRUPT) },
526 { .compatible = "ultrarisc,cp100-plic",
527 .data = (const void *)BIT(PLIC_QUIRK_CP100_CLAIM_REGISTER_ERRATUM) },
528 {}
529 };
530
531 #ifdef CONFIG_ACPI
532
533 static const struct acpi_device_id plic_acpi_match[] = {
534 { "RSCV0001", 0 },
535 {}
536 };
537 MODULE_DEVICE_TABLE(acpi, plic_acpi_match);
538
539 #endif
plic_parse_nr_irqs_and_contexts(struct fwnode_handle * fwnode,u32 * nr_irqs,u32 * nr_contexts,u32 * gsi_base,u32 * id)540 static int plic_parse_nr_irqs_and_contexts(struct fwnode_handle *fwnode,
541 u32 *nr_irqs, u32 *nr_contexts,
542 u32 *gsi_base, u32 *id)
543 {
544 int rc;
545
546 if (!is_of_node(fwnode)) {
547 rc = riscv_acpi_get_gsi_info(fwnode, gsi_base, id, nr_irqs, NULL);
548 if (rc) {
549 pr_err("%pfwP: failed to find GSI mapping\n", fwnode);
550 return rc;
551 }
552
553 *nr_contexts = acpi_rintc_get_plic_nr_contexts(*id);
554 if (WARN_ON(!*nr_contexts)) {
555 pr_err("%pfwP: no PLIC context available\n", fwnode);
556 return -EINVAL;
557 }
558
559 return 0;
560 }
561
562 rc = of_property_read_u32(to_of_node(fwnode), "riscv,ndev", nr_irqs);
563 if (rc) {
564 pr_err("%pfwP: riscv,ndev property not available\n", fwnode);
565 return rc;
566 }
567
568 *nr_contexts = of_irq_count(to_of_node(fwnode));
569 if (WARN_ON(!(*nr_contexts))) {
570 pr_err("%pfwP: no PLIC context available\n", fwnode);
571 return -EINVAL;
572 }
573
574 *gsi_base = 0;
575 *id = 0;
576
577 return 0;
578 }
579
plic_parse_context_parent(struct fwnode_handle * fwnode,u32 context,u32 * parent_hwirq,int * parent_cpu,u32 id)580 static int plic_parse_context_parent(struct fwnode_handle *fwnode, u32 context,
581 u32 *parent_hwirq, int *parent_cpu, u32 id)
582 {
583 struct of_phandle_args parent;
584 unsigned long hartid;
585 int rc;
586
587 if (!is_of_node(fwnode)) {
588 hartid = acpi_rintc_ext_parent_to_hartid(id, context);
589 if (hartid == INVALID_HARTID)
590 return -EINVAL;
591
592 *parent_cpu = riscv_hartid_to_cpuid(hartid);
593 *parent_hwirq = RV_IRQ_EXT;
594 return 0;
595 }
596
597 rc = of_irq_parse_one(to_of_node(fwnode), context, &parent);
598 if (rc)
599 return rc;
600
601 rc = riscv_of_parent_hartid(parent.np, &hartid);
602 if (rc)
603 return rc;
604
605 *parent_hwirq = parent.args[0];
606 *parent_cpu = riscv_hartid_to_cpuid(hartid);
607 return 0;
608 }
609
plic_probe(struct fwnode_handle * fwnode)610 static int plic_probe(struct fwnode_handle *fwnode)
611 {
612 int error = 0, nr_contexts, nr_handlers = 0, cpu, i;
613 unsigned long plic_quirks = 0;
614 struct plic_handler *handler;
615 u32 nr_irqs, parent_hwirq;
616 struct plic_priv *priv;
617 irq_hw_number_t hwirq;
618 void __iomem *regs;
619 int id, context_id;
620 u32 gsi_base;
621
622 if (is_of_node(fwnode)) {
623 const struct of_device_id *id;
624
625 id = of_match_node(plic_match, to_of_node(fwnode));
626 if (id)
627 plic_quirks = (unsigned long)id->data;
628
629 regs = of_iomap(to_of_node(fwnode), 0);
630 if (!regs)
631 return -ENOMEM;
632 } else {
633 regs = devm_platform_ioremap_resource(to_platform_device(fwnode->dev), 0);
634 if (IS_ERR(regs))
635 return PTR_ERR(regs);
636 }
637
638 error = plic_parse_nr_irqs_and_contexts(fwnode, &nr_irqs, &nr_contexts, &gsi_base, &id);
639 if (error)
640 goto fail_free_regs;
641
642 priv = kzalloc(sizeof(*priv), GFP_KERNEL);
643 if (!priv) {
644 error = -ENOMEM;
645 goto fail_free_regs;
646 }
647
648 priv->fwnode = fwnode;
649 priv->plic_quirks = plic_quirks;
650 priv->nr_irqs = nr_irqs;
651 priv->regs = regs;
652 priv->gsi_base = gsi_base;
653 priv->acpi_plic_id = id;
654
655 priv->prio_save = bitmap_zalloc(nr_irqs, GFP_KERNEL);
656 if (!priv->prio_save) {
657 error = -ENOMEM;
658 goto fail_free_priv;
659 }
660
661 for (i = 0; i < nr_contexts; i++) {
662 error = plic_parse_context_parent(fwnode, i, &parent_hwirq, &cpu,
663 priv->acpi_plic_id);
664 if (error) {
665 pr_warn("%pfwP: hwirq for context%d not found\n", fwnode, i);
666 continue;
667 }
668
669 if (is_of_node(fwnode)) {
670 context_id = i;
671 } else {
672 context_id = acpi_rintc_get_plic_context(priv->acpi_plic_id, i);
673 if (context_id == INVALID_CONTEXT) {
674 pr_warn("%pfwP: invalid context id for context%d\n", fwnode, i);
675 continue;
676 }
677 }
678
679 /*
680 * Skip contexts other than external interrupts for our
681 * privilege level.
682 */
683 if (parent_hwirq != RV_IRQ_EXT) {
684 /* Disable S-mode enable bits if running in M-mode. */
685 if (IS_ENABLED(CONFIG_RISCV_M_MODE)) {
686 u32 __iomem *enable_base = priv->regs + CONTEXT_ENABLE_BASE +
687 i * CONTEXT_ENABLE_SIZE;
688
689 for (int j = 0; j <= nr_irqs / 32; j++)
690 writel(0, enable_base + j);
691 }
692 continue;
693 }
694
695 if (cpu < 0) {
696 pr_warn("%pfwP: Invalid cpuid for context %d\n", fwnode, i);
697 continue;
698 }
699
700 /*
701 * When running in M-mode we need to ignore the S-mode handler.
702 * Here we assume it always comes later, but that might be a
703 * little fragile.
704 */
705 handler = per_cpu_ptr(&plic_handlers, cpu);
706 if (handler->present) {
707 pr_warn("%pfwP: handler already present for context %d.\n", fwnode, i);
708 plic_set_threshold(handler, PLIC_DISABLE_THRESHOLD);
709 goto done;
710 }
711
712 cpumask_set_cpu(cpu, &priv->lmask);
713 handler->present = true;
714 handler->hart_base = priv->regs + CONTEXT_BASE +
715 context_id * CONTEXT_SIZE;
716 raw_spin_lock_init(&handler->enable_lock);
717 handler->enable_base = priv->regs + CONTEXT_ENABLE_BASE +
718 context_id * CONTEXT_ENABLE_SIZE;
719 handler->priv = priv;
720
721 handler->enable_save = kcalloc(DIV_ROUND_UP(nr_irqs, 32),
722 sizeof(*handler->enable_save), GFP_KERNEL);
723 if (!handler->enable_save) {
724 error = -ENOMEM;
725 goto fail_cleanup_contexts;
726 }
727 done:
728 for (hwirq = 1; hwirq <= nr_irqs; hwirq++) {
729 plic_toggle(handler, hwirq, 0);
730 writel(1, priv->regs + PRIORITY_BASE +
731 hwirq * PRIORITY_PER_ID);
732 }
733 nr_handlers++;
734 }
735
736 priv->irqdomain = irq_domain_create_linear(fwnode, nr_irqs + 1,
737 &plic_irqdomain_ops, priv);
738 if (WARN_ON(!priv->irqdomain)) {
739 error = -ENOMEM;
740 goto fail_cleanup_contexts;
741 }
742
743 /*
744 * We can have multiple PLIC instances so setup global state
745 * and register syscore operations only once after context
746 * handlers of all online CPUs are initialized.
747 */
748 if (!plic_global_setup_done) {
749 struct irq_domain *domain;
750 bool global_setup = true;
751
752 for_each_online_cpu(cpu) {
753 handler = per_cpu_ptr(&plic_handlers, cpu);
754 if (!handler->present) {
755 global_setup = false;
756 break;
757 }
758 }
759
760 if (global_setup) {
761 void (*handler_fn)(struct irq_desc *) = plic_handle_irq;
762
763 if (test_bit(PLIC_QUIRK_CP100_CLAIM_REGISTER_ERRATUM, &handler->priv->plic_quirks))
764 handler_fn = plic_handle_irq_cp100;
765
766 /* Find parent domain and register chained handler */
767 domain = irq_find_matching_fwnode(riscv_get_intc_hwnode(), DOMAIN_BUS_ANY);
768 if (domain)
769 plic_parent_irq = irq_create_mapping(domain, RV_IRQ_EXT);
770 if (plic_parent_irq)
771 irq_set_chained_handler(plic_parent_irq, handler_fn);
772
773 cpuhp_setup_state(CPUHP_AP_IRQ_SIFIVE_PLIC_STARTING,
774 "irqchip/sifive/plic:starting",
775 plic_starting_cpu, plic_dying_cpu);
776 register_syscore(&plic_irq_syscore);
777 plic_global_setup_done = true;
778 }
779 }
780
781 #ifdef CONFIG_ACPI
782 if (!acpi_disabled)
783 acpi_dev_clear_dependencies(ACPI_COMPANION(fwnode->dev));
784 #endif
785
786 pr_info("%pfwP: mapped %d interrupts with %d handlers for %d contexts.\n",
787 fwnode, nr_irqs, nr_handlers, nr_contexts);
788 return 0;
789
790 fail_cleanup_contexts:
791 for (i = 0; i < nr_contexts; i++) {
792 if (plic_parse_context_parent(fwnode, i, &parent_hwirq, &cpu, priv->acpi_plic_id))
793 continue;
794 if (parent_hwirq != RV_IRQ_EXT || cpu < 0)
795 continue;
796
797 handler = per_cpu_ptr(&plic_handlers, cpu);
798 handler->present = false;
799 handler->hart_base = NULL;
800 handler->enable_base = NULL;
801 kfree(handler->enable_save);
802 handler->enable_save = NULL;
803 handler->priv = NULL;
804 }
805 bitmap_free(priv->prio_save);
806 fail_free_priv:
807 kfree(priv);
808 fail_free_regs:
809 iounmap(regs);
810 return error;
811 }
812
plic_platform_probe(struct platform_device * pdev)813 static int plic_platform_probe(struct platform_device *pdev)
814 {
815 return plic_probe(pdev->dev.fwnode);
816 }
817
818 static struct platform_driver plic_driver = {
819 .driver = {
820 .name = "riscv-plic",
821 .of_match_table = plic_match,
822 .suppress_bind_attrs = true,
823 .acpi_match_table = ACPI_PTR(plic_acpi_match),
824 },
825 .probe = plic_platform_probe,
826 };
827 builtin_platform_driver(plic_driver);
828
plic_early_probe(struct device_node * node,struct device_node * parent)829 static int __init plic_early_probe(struct device_node *node,
830 struct device_node *parent)
831 {
832 return plic_probe(&node->fwnode);
833 }
834
835 IRQCHIP_DECLARE(riscv, "allwinner,sun20i-d1-plic", plic_early_probe);
836