xref: /linux/drivers/irqchip/irq-gic-v3.c (revision a7f7f6248d9740d710fd6bd190293fe5e16410ac)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved.
4  * Author: Marc Zyngier <marc.zyngier@arm.com>
5  */
6 
7 #define pr_fmt(fmt)	"GICv3: " fmt
8 
9 #include <linux/acpi.h>
10 #include <linux/cpu.h>
11 #include <linux/cpu_pm.h>
12 #include <linux/delay.h>
13 #include <linux/interrupt.h>
14 #include <linux/irqdomain.h>
15 #include <linux/of.h>
16 #include <linux/of_address.h>
17 #include <linux/of_irq.h>
18 #include <linux/percpu.h>
19 #include <linux/refcount.h>
20 #include <linux/slab.h>
21 
22 #include <linux/irqchip.h>
23 #include <linux/irqchip/arm-gic-common.h>
24 #include <linux/irqchip/arm-gic-v3.h>
25 #include <linux/irqchip/irq-partition-percpu.h>
26 
27 #include <asm/cputype.h>
28 #include <asm/exception.h>
29 #include <asm/smp_plat.h>
30 #include <asm/virt.h>
31 
32 #include "irq-gic-common.h"
33 
34 #define GICD_INT_NMI_PRI	(GICD_INT_DEF_PRI & ~0x80)
35 
36 #define FLAGS_WORKAROUND_GICR_WAKER_MSM8996	(1ULL << 0)
37 #define FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539	(1ULL << 1)
38 
39 struct redist_region {
40 	void __iomem		*redist_base;
41 	phys_addr_t		phys_base;
42 	bool			single_redist;
43 };
44 
45 struct gic_chip_data {
46 	struct fwnode_handle	*fwnode;
47 	void __iomem		*dist_base;
48 	struct redist_region	*redist_regions;
49 	struct rdists		rdists;
50 	struct irq_domain	*domain;
51 	u64			redist_stride;
52 	u32			nr_redist_regions;
53 	u64			flags;
54 	bool			has_rss;
55 	unsigned int		ppi_nr;
56 	struct partition_desc	**ppi_descs;
57 };
58 
59 static struct gic_chip_data gic_data __read_mostly;
60 static DEFINE_STATIC_KEY_TRUE(supports_deactivate_key);
61 
62 #define GIC_ID_NR	(1U << GICD_TYPER_ID_BITS(gic_data.rdists.gicd_typer))
63 #define GIC_LINE_NR	min(GICD_TYPER_SPIS(gic_data.rdists.gicd_typer), 1020U)
64 #define GIC_ESPI_NR	GICD_TYPER_ESPIS(gic_data.rdists.gicd_typer)
65 
66 /*
67  * The behaviours of RPR and PMR registers differ depending on the value of
68  * SCR_EL3.FIQ, and the behaviour of non-secure priority registers of the
69  * distributor and redistributors depends on whether security is enabled in the
70  * GIC.
71  *
72  * When security is enabled, non-secure priority values from the (re)distributor
73  * are presented to the GIC CPUIF as follow:
74  *     (GIC_(R)DIST_PRI[irq] >> 1) | 0x80;
75  *
76  * If SCR_EL3.FIQ == 1, the values writen to/read from PMR and RPR at non-secure
77  * EL1 are subject to a similar operation thus matching the priorities presented
78  * from the (re)distributor when security is enabled.
79  *
80  * see GICv3/GICv4 Architecture Specification (IHI0069D):
81  * - section 4.8.1 Non-secure accesses to register fields for Secure interrupt
82  *   priorities.
83  * - Figure 4-7 Secure read of the priority field for a Non-secure Group 1
84  *   interrupt.
85  *
86  * For now, we only support pseudo-NMIs if we have non-secure view of
87  * priorities.
88  */
89 static DEFINE_STATIC_KEY_FALSE(supports_pseudo_nmis);
90 
91 /*
92  * Global static key controlling whether an update to PMR allowing more
93  * interrupts requires to be propagated to the redistributor (DSB SY).
94  * And this needs to be exported for modules to be able to enable
95  * interrupts...
96  */
97 DEFINE_STATIC_KEY_FALSE(gic_pmr_sync);
98 EXPORT_SYMBOL(gic_pmr_sync);
99 
100 /* ppi_nmi_refs[n] == number of cpus having ppi[n + 16] set as NMI */
101 static refcount_t *ppi_nmi_refs;
102 
103 static struct gic_kvm_info gic_v3_kvm_info;
104 static DEFINE_PER_CPU(bool, has_rss);
105 
106 #define MPIDR_RS(mpidr)			(((mpidr) & 0xF0UL) >> 4)
107 #define gic_data_rdist()		(this_cpu_ptr(gic_data.rdists.rdist))
108 #define gic_data_rdist_rd_base()	(gic_data_rdist()->rd_base)
109 #define gic_data_rdist_sgi_base()	(gic_data_rdist_rd_base() + SZ_64K)
110 
111 /* Our default, arbitrary priority value. Linux only uses one anyway. */
112 #define DEFAULT_PMR_VALUE	0xf0
113 
114 enum gic_intid_range {
115 	PPI_RANGE,
116 	SPI_RANGE,
117 	EPPI_RANGE,
118 	ESPI_RANGE,
119 	LPI_RANGE,
120 	__INVALID_RANGE__
121 };
122 
123 static enum gic_intid_range __get_intid_range(irq_hw_number_t hwirq)
124 {
125 	switch (hwirq) {
126 	case 16 ... 31:
127 		return PPI_RANGE;
128 	case 32 ... 1019:
129 		return SPI_RANGE;
130 	case EPPI_BASE_INTID ... (EPPI_BASE_INTID + 63):
131 		return EPPI_RANGE;
132 	case ESPI_BASE_INTID ... (ESPI_BASE_INTID + 1023):
133 		return ESPI_RANGE;
134 	case 8192 ... GENMASK(23, 0):
135 		return LPI_RANGE;
136 	default:
137 		return __INVALID_RANGE__;
138 	}
139 }
140 
141 static enum gic_intid_range get_intid_range(struct irq_data *d)
142 {
143 	return __get_intid_range(d->hwirq);
144 }
145 
146 static inline unsigned int gic_irq(struct irq_data *d)
147 {
148 	return d->hwirq;
149 }
150 
151 static inline int gic_irq_in_rdist(struct irq_data *d)
152 {
153 	enum gic_intid_range range = get_intid_range(d);
154 	return range == PPI_RANGE || range == EPPI_RANGE;
155 }
156 
157 static inline void __iomem *gic_dist_base(struct irq_data *d)
158 {
159 	switch (get_intid_range(d)) {
160 	case PPI_RANGE:
161 	case EPPI_RANGE:
162 		/* SGI+PPI -> SGI_base for this CPU */
163 		return gic_data_rdist_sgi_base();
164 
165 	case SPI_RANGE:
166 	case ESPI_RANGE:
167 		/* SPI -> dist_base */
168 		return gic_data.dist_base;
169 
170 	default:
171 		return NULL;
172 	}
173 }
174 
175 static void gic_do_wait_for_rwp(void __iomem *base)
176 {
177 	u32 count = 1000000;	/* 1s! */
178 
179 	while (readl_relaxed(base + GICD_CTLR) & GICD_CTLR_RWP) {
180 		count--;
181 		if (!count) {
182 			pr_err_ratelimited("RWP timeout, gone fishing\n");
183 			return;
184 		}
185 		cpu_relax();
186 		udelay(1);
187 	}
188 }
189 
190 /* Wait for completion of a distributor change */
191 static void gic_dist_wait_for_rwp(void)
192 {
193 	gic_do_wait_for_rwp(gic_data.dist_base);
194 }
195 
196 /* Wait for completion of a redistributor change */
197 static void gic_redist_wait_for_rwp(void)
198 {
199 	gic_do_wait_for_rwp(gic_data_rdist_rd_base());
200 }
201 
202 #ifdef CONFIG_ARM64
203 
204 static u64 __maybe_unused gic_read_iar(void)
205 {
206 	if (cpus_have_const_cap(ARM64_WORKAROUND_CAVIUM_23154))
207 		return gic_read_iar_cavium_thunderx();
208 	else
209 		return gic_read_iar_common();
210 }
211 #endif
212 
213 static void gic_enable_redist(bool enable)
214 {
215 	void __iomem *rbase;
216 	u32 count = 1000000;	/* 1s! */
217 	u32 val;
218 
219 	if (gic_data.flags & FLAGS_WORKAROUND_GICR_WAKER_MSM8996)
220 		return;
221 
222 	rbase = gic_data_rdist_rd_base();
223 
224 	val = readl_relaxed(rbase + GICR_WAKER);
225 	if (enable)
226 		/* Wake up this CPU redistributor */
227 		val &= ~GICR_WAKER_ProcessorSleep;
228 	else
229 		val |= GICR_WAKER_ProcessorSleep;
230 	writel_relaxed(val, rbase + GICR_WAKER);
231 
232 	if (!enable) {		/* Check that GICR_WAKER is writeable */
233 		val = readl_relaxed(rbase + GICR_WAKER);
234 		if (!(val & GICR_WAKER_ProcessorSleep))
235 			return;	/* No PM support in this redistributor */
236 	}
237 
238 	while (--count) {
239 		val = readl_relaxed(rbase + GICR_WAKER);
240 		if (enable ^ (bool)(val & GICR_WAKER_ChildrenAsleep))
241 			break;
242 		cpu_relax();
243 		udelay(1);
244 	}
245 	if (!count)
246 		pr_err_ratelimited("redistributor failed to %s...\n",
247 				   enable ? "wakeup" : "sleep");
248 }
249 
250 /*
251  * Routines to disable, enable, EOI and route interrupts
252  */
253 static u32 convert_offset_index(struct irq_data *d, u32 offset, u32 *index)
254 {
255 	switch (get_intid_range(d)) {
256 	case PPI_RANGE:
257 	case SPI_RANGE:
258 		*index = d->hwirq;
259 		return offset;
260 	case EPPI_RANGE:
261 		/*
262 		 * Contrary to the ESPI range, the EPPI range is contiguous
263 		 * to the PPI range in the registers, so let's adjust the
264 		 * displacement accordingly. Consistency is overrated.
265 		 */
266 		*index = d->hwirq - EPPI_BASE_INTID + 32;
267 		return offset;
268 	case ESPI_RANGE:
269 		*index = d->hwirq - ESPI_BASE_INTID;
270 		switch (offset) {
271 		case GICD_ISENABLER:
272 			return GICD_ISENABLERnE;
273 		case GICD_ICENABLER:
274 			return GICD_ICENABLERnE;
275 		case GICD_ISPENDR:
276 			return GICD_ISPENDRnE;
277 		case GICD_ICPENDR:
278 			return GICD_ICPENDRnE;
279 		case GICD_ISACTIVER:
280 			return GICD_ISACTIVERnE;
281 		case GICD_ICACTIVER:
282 			return GICD_ICACTIVERnE;
283 		case GICD_IPRIORITYR:
284 			return GICD_IPRIORITYRnE;
285 		case GICD_ICFGR:
286 			return GICD_ICFGRnE;
287 		case GICD_IROUTER:
288 			return GICD_IROUTERnE;
289 		default:
290 			break;
291 		}
292 		break;
293 	default:
294 		break;
295 	}
296 
297 	WARN_ON(1);
298 	*index = d->hwirq;
299 	return offset;
300 }
301 
302 static int gic_peek_irq(struct irq_data *d, u32 offset)
303 {
304 	void __iomem *base;
305 	u32 index, mask;
306 
307 	offset = convert_offset_index(d, offset, &index);
308 	mask = 1 << (index % 32);
309 
310 	if (gic_irq_in_rdist(d))
311 		base = gic_data_rdist_sgi_base();
312 	else
313 		base = gic_data.dist_base;
314 
315 	return !!(readl_relaxed(base + offset + (index / 32) * 4) & mask);
316 }
317 
318 static void gic_poke_irq(struct irq_data *d, u32 offset)
319 {
320 	void (*rwp_wait)(void);
321 	void __iomem *base;
322 	u32 index, mask;
323 
324 	offset = convert_offset_index(d, offset, &index);
325 	mask = 1 << (index % 32);
326 
327 	if (gic_irq_in_rdist(d)) {
328 		base = gic_data_rdist_sgi_base();
329 		rwp_wait = gic_redist_wait_for_rwp;
330 	} else {
331 		base = gic_data.dist_base;
332 		rwp_wait = gic_dist_wait_for_rwp;
333 	}
334 
335 	writel_relaxed(mask, base + offset + (index / 32) * 4);
336 	rwp_wait();
337 }
338 
339 static void gic_mask_irq(struct irq_data *d)
340 {
341 	gic_poke_irq(d, GICD_ICENABLER);
342 }
343 
344 static void gic_eoimode1_mask_irq(struct irq_data *d)
345 {
346 	gic_mask_irq(d);
347 	/*
348 	 * When masking a forwarded interrupt, make sure it is
349 	 * deactivated as well.
350 	 *
351 	 * This ensures that an interrupt that is getting
352 	 * disabled/masked will not get "stuck", because there is
353 	 * noone to deactivate it (guest is being terminated).
354 	 */
355 	if (irqd_is_forwarded_to_vcpu(d))
356 		gic_poke_irq(d, GICD_ICACTIVER);
357 }
358 
359 static void gic_unmask_irq(struct irq_data *d)
360 {
361 	gic_poke_irq(d, GICD_ISENABLER);
362 }
363 
364 static inline bool gic_supports_nmi(void)
365 {
366 	return IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) &&
367 	       static_branch_likely(&supports_pseudo_nmis);
368 }
369 
370 static int gic_irq_set_irqchip_state(struct irq_data *d,
371 				     enum irqchip_irq_state which, bool val)
372 {
373 	u32 reg;
374 
375 	if (d->hwirq >= 8192) /* PPI/SPI only */
376 		return -EINVAL;
377 
378 	switch (which) {
379 	case IRQCHIP_STATE_PENDING:
380 		reg = val ? GICD_ISPENDR : GICD_ICPENDR;
381 		break;
382 
383 	case IRQCHIP_STATE_ACTIVE:
384 		reg = val ? GICD_ISACTIVER : GICD_ICACTIVER;
385 		break;
386 
387 	case IRQCHIP_STATE_MASKED:
388 		reg = val ? GICD_ICENABLER : GICD_ISENABLER;
389 		break;
390 
391 	default:
392 		return -EINVAL;
393 	}
394 
395 	gic_poke_irq(d, reg);
396 	return 0;
397 }
398 
399 static int gic_irq_get_irqchip_state(struct irq_data *d,
400 				     enum irqchip_irq_state which, bool *val)
401 {
402 	if (d->hwirq >= 8192) /* PPI/SPI only */
403 		return -EINVAL;
404 
405 	switch (which) {
406 	case IRQCHIP_STATE_PENDING:
407 		*val = gic_peek_irq(d, GICD_ISPENDR);
408 		break;
409 
410 	case IRQCHIP_STATE_ACTIVE:
411 		*val = gic_peek_irq(d, GICD_ISACTIVER);
412 		break;
413 
414 	case IRQCHIP_STATE_MASKED:
415 		*val = !gic_peek_irq(d, GICD_ISENABLER);
416 		break;
417 
418 	default:
419 		return -EINVAL;
420 	}
421 
422 	return 0;
423 }
424 
425 static void gic_irq_set_prio(struct irq_data *d, u8 prio)
426 {
427 	void __iomem *base = gic_dist_base(d);
428 	u32 offset, index;
429 
430 	offset = convert_offset_index(d, GICD_IPRIORITYR, &index);
431 
432 	writeb_relaxed(prio, base + offset + index);
433 }
434 
435 static u32 gic_get_ppi_index(struct irq_data *d)
436 {
437 	switch (get_intid_range(d)) {
438 	case PPI_RANGE:
439 		return d->hwirq - 16;
440 	case EPPI_RANGE:
441 		return d->hwirq - EPPI_BASE_INTID + 16;
442 	default:
443 		unreachable();
444 	}
445 }
446 
447 static int gic_irq_nmi_setup(struct irq_data *d)
448 {
449 	struct irq_desc *desc = irq_to_desc(d->irq);
450 
451 	if (!gic_supports_nmi())
452 		return -EINVAL;
453 
454 	if (gic_peek_irq(d, GICD_ISENABLER)) {
455 		pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq);
456 		return -EINVAL;
457 	}
458 
459 	/*
460 	 * A secondary irq_chip should be in charge of LPI request,
461 	 * it should not be possible to get there
462 	 */
463 	if (WARN_ON(gic_irq(d) >= 8192))
464 		return -EINVAL;
465 
466 	/* desc lock should already be held */
467 	if (gic_irq_in_rdist(d)) {
468 		u32 idx = gic_get_ppi_index(d);
469 
470 		/* Setting up PPI as NMI, only switch handler for first NMI */
471 		if (!refcount_inc_not_zero(&ppi_nmi_refs[idx])) {
472 			refcount_set(&ppi_nmi_refs[idx], 1);
473 			desc->handle_irq = handle_percpu_devid_fasteoi_nmi;
474 		}
475 	} else {
476 		desc->handle_irq = handle_fasteoi_nmi;
477 	}
478 
479 	gic_irq_set_prio(d, GICD_INT_NMI_PRI);
480 
481 	return 0;
482 }
483 
484 static void gic_irq_nmi_teardown(struct irq_data *d)
485 {
486 	struct irq_desc *desc = irq_to_desc(d->irq);
487 
488 	if (WARN_ON(!gic_supports_nmi()))
489 		return;
490 
491 	if (gic_peek_irq(d, GICD_ISENABLER)) {
492 		pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq);
493 		return;
494 	}
495 
496 	/*
497 	 * A secondary irq_chip should be in charge of LPI request,
498 	 * it should not be possible to get there
499 	 */
500 	if (WARN_ON(gic_irq(d) >= 8192))
501 		return;
502 
503 	/* desc lock should already be held */
504 	if (gic_irq_in_rdist(d)) {
505 		u32 idx = gic_get_ppi_index(d);
506 
507 		/* Tearing down NMI, only switch handler for last NMI */
508 		if (refcount_dec_and_test(&ppi_nmi_refs[idx]))
509 			desc->handle_irq = handle_percpu_devid_irq;
510 	} else {
511 		desc->handle_irq = handle_fasteoi_irq;
512 	}
513 
514 	gic_irq_set_prio(d, GICD_INT_DEF_PRI);
515 }
516 
517 static void gic_eoi_irq(struct irq_data *d)
518 {
519 	gic_write_eoir(gic_irq(d));
520 }
521 
522 static void gic_eoimode1_eoi_irq(struct irq_data *d)
523 {
524 	/*
525 	 * No need to deactivate an LPI, or an interrupt that
526 	 * is is getting forwarded to a vcpu.
527 	 */
528 	if (gic_irq(d) >= 8192 || irqd_is_forwarded_to_vcpu(d))
529 		return;
530 	gic_write_dir(gic_irq(d));
531 }
532 
533 static int gic_set_type(struct irq_data *d, unsigned int type)
534 {
535 	enum gic_intid_range range;
536 	unsigned int irq = gic_irq(d);
537 	void (*rwp_wait)(void);
538 	void __iomem *base;
539 	u32 offset, index;
540 	int ret;
541 
542 	/* Interrupt configuration for SGIs can't be changed */
543 	if (irq < 16)
544 		return -EINVAL;
545 
546 	range = get_intid_range(d);
547 
548 	/* SPIs have restrictions on the supported types */
549 	if ((range == SPI_RANGE || range == ESPI_RANGE) &&
550 	    type != IRQ_TYPE_LEVEL_HIGH && type != IRQ_TYPE_EDGE_RISING)
551 		return -EINVAL;
552 
553 	if (gic_irq_in_rdist(d)) {
554 		base = gic_data_rdist_sgi_base();
555 		rwp_wait = gic_redist_wait_for_rwp;
556 	} else {
557 		base = gic_data.dist_base;
558 		rwp_wait = gic_dist_wait_for_rwp;
559 	}
560 
561 	offset = convert_offset_index(d, GICD_ICFGR, &index);
562 
563 	ret = gic_configure_irq(index, type, base + offset, rwp_wait);
564 	if (ret && (range == PPI_RANGE || range == EPPI_RANGE)) {
565 		/* Misconfigured PPIs are usually not fatal */
566 		pr_warn("GIC: PPI INTID%d is secure or misconfigured\n", irq);
567 		ret = 0;
568 	}
569 
570 	return ret;
571 }
572 
573 static int gic_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu)
574 {
575 	if (vcpu)
576 		irqd_set_forwarded_to_vcpu(d);
577 	else
578 		irqd_clr_forwarded_to_vcpu(d);
579 	return 0;
580 }
581 
582 static u64 gic_mpidr_to_affinity(unsigned long mpidr)
583 {
584 	u64 aff;
585 
586 	aff = ((u64)MPIDR_AFFINITY_LEVEL(mpidr, 3) << 32 |
587 	       MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
588 	       MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8  |
589 	       MPIDR_AFFINITY_LEVEL(mpidr, 0));
590 
591 	return aff;
592 }
593 
594 static void gic_deactivate_unhandled(u32 irqnr)
595 {
596 	if (static_branch_likely(&supports_deactivate_key)) {
597 		if (irqnr < 8192)
598 			gic_write_dir(irqnr);
599 	} else {
600 		gic_write_eoir(irqnr);
601 	}
602 }
603 
604 static inline void gic_handle_nmi(u32 irqnr, struct pt_regs *regs)
605 {
606 	bool irqs_enabled = interrupts_enabled(regs);
607 	int err;
608 
609 	if (irqs_enabled)
610 		nmi_enter();
611 
612 	if (static_branch_likely(&supports_deactivate_key))
613 		gic_write_eoir(irqnr);
614 	/*
615 	 * Leave the PSR.I bit set to prevent other NMIs to be
616 	 * received while handling this one.
617 	 * PSR.I will be restored when we ERET to the
618 	 * interrupted context.
619 	 */
620 	err = handle_domain_nmi(gic_data.domain, irqnr, regs);
621 	if (err)
622 		gic_deactivate_unhandled(irqnr);
623 
624 	if (irqs_enabled)
625 		nmi_exit();
626 }
627 
628 static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs)
629 {
630 	u32 irqnr;
631 
632 	irqnr = gic_read_iar();
633 
634 	if (gic_supports_nmi() &&
635 	    unlikely(gic_read_rpr() == GICD_INT_NMI_PRI)) {
636 		gic_handle_nmi(irqnr, regs);
637 		return;
638 	}
639 
640 	if (gic_prio_masking_enabled()) {
641 		gic_pmr_mask_irqs();
642 		gic_arch_enable_irqs();
643 	}
644 
645 	/* Check for special IDs first */
646 	if ((irqnr >= 1020 && irqnr <= 1023))
647 		return;
648 
649 	/* Treat anything but SGIs in a uniform way */
650 	if (likely(irqnr > 15)) {
651 		int err;
652 
653 		if (static_branch_likely(&supports_deactivate_key))
654 			gic_write_eoir(irqnr);
655 		else
656 			isb();
657 
658 		err = handle_domain_irq(gic_data.domain, irqnr, regs);
659 		if (err) {
660 			WARN_ONCE(true, "Unexpected interrupt received!\n");
661 			gic_deactivate_unhandled(irqnr);
662 		}
663 		return;
664 	}
665 	if (irqnr < 16) {
666 		gic_write_eoir(irqnr);
667 		if (static_branch_likely(&supports_deactivate_key))
668 			gic_write_dir(irqnr);
669 #ifdef CONFIG_SMP
670 		/*
671 		 * Unlike GICv2, we don't need an smp_rmb() here.
672 		 * The control dependency from gic_read_iar to
673 		 * the ISB in gic_write_eoir is enough to ensure
674 		 * that any shared data read by handle_IPI will
675 		 * be read after the ACK.
676 		 */
677 		handle_IPI(irqnr, regs);
678 #else
679 		WARN_ONCE(true, "Unexpected SGI received!\n");
680 #endif
681 	}
682 }
683 
684 static u32 gic_get_pribits(void)
685 {
686 	u32 pribits;
687 
688 	pribits = gic_read_ctlr();
689 	pribits &= ICC_CTLR_EL1_PRI_BITS_MASK;
690 	pribits >>= ICC_CTLR_EL1_PRI_BITS_SHIFT;
691 	pribits++;
692 
693 	return pribits;
694 }
695 
696 static bool gic_has_group0(void)
697 {
698 	u32 val;
699 	u32 old_pmr;
700 
701 	old_pmr = gic_read_pmr();
702 
703 	/*
704 	 * Let's find out if Group0 is under control of EL3 or not by
705 	 * setting the highest possible, non-zero priority in PMR.
706 	 *
707 	 * If SCR_EL3.FIQ is set, the priority gets shifted down in
708 	 * order for the CPU interface to set bit 7, and keep the
709 	 * actual priority in the non-secure range. In the process, it
710 	 * looses the least significant bit and the actual priority
711 	 * becomes 0x80. Reading it back returns 0, indicating that
712 	 * we're don't have access to Group0.
713 	 */
714 	gic_write_pmr(BIT(8 - gic_get_pribits()));
715 	val = gic_read_pmr();
716 
717 	gic_write_pmr(old_pmr);
718 
719 	return val != 0;
720 }
721 
722 static void __init gic_dist_init(void)
723 {
724 	unsigned int i;
725 	u64 affinity;
726 	void __iomem *base = gic_data.dist_base;
727 	u32 val;
728 
729 	/* Disable the distributor */
730 	writel_relaxed(0, base + GICD_CTLR);
731 	gic_dist_wait_for_rwp();
732 
733 	/*
734 	 * Configure SPIs as non-secure Group-1. This will only matter
735 	 * if the GIC only has a single security state. This will not
736 	 * do the right thing if the kernel is running in secure mode,
737 	 * but that's not the intended use case anyway.
738 	 */
739 	for (i = 32; i < GIC_LINE_NR; i += 32)
740 		writel_relaxed(~0, base + GICD_IGROUPR + i / 8);
741 
742 	/* Extended SPI range, not handled by the GICv2/GICv3 common code */
743 	for (i = 0; i < GIC_ESPI_NR; i += 32) {
744 		writel_relaxed(~0U, base + GICD_ICENABLERnE + i / 8);
745 		writel_relaxed(~0U, base + GICD_ICACTIVERnE + i / 8);
746 	}
747 
748 	for (i = 0; i < GIC_ESPI_NR; i += 32)
749 		writel_relaxed(~0U, base + GICD_IGROUPRnE + i / 8);
750 
751 	for (i = 0; i < GIC_ESPI_NR; i += 16)
752 		writel_relaxed(0, base + GICD_ICFGRnE + i / 4);
753 
754 	for (i = 0; i < GIC_ESPI_NR; i += 4)
755 		writel_relaxed(GICD_INT_DEF_PRI_X4, base + GICD_IPRIORITYRnE + i);
756 
757 	/* Now do the common stuff, and wait for the distributor to drain */
758 	gic_dist_config(base, GIC_LINE_NR, gic_dist_wait_for_rwp);
759 
760 	val = GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1;
761 	if (gic_data.rdists.gicd_typer2 & GICD_TYPER2_nASSGIcap) {
762 		pr_info("Enabling SGIs without active state\n");
763 		val |= GICD_CTLR_nASSGIreq;
764 	}
765 
766 	/* Enable distributor with ARE, Group1 */
767 	writel_relaxed(val, base + GICD_CTLR);
768 
769 	/*
770 	 * Set all global interrupts to the boot CPU only. ARE must be
771 	 * enabled.
772 	 */
773 	affinity = gic_mpidr_to_affinity(cpu_logical_map(smp_processor_id()));
774 	for (i = 32; i < GIC_LINE_NR; i++)
775 		gic_write_irouter(affinity, base + GICD_IROUTER + i * 8);
776 
777 	for (i = 0; i < GIC_ESPI_NR; i++)
778 		gic_write_irouter(affinity, base + GICD_IROUTERnE + i * 8);
779 }
780 
781 static int gic_iterate_rdists(int (*fn)(struct redist_region *, void __iomem *))
782 {
783 	int ret = -ENODEV;
784 	int i;
785 
786 	for (i = 0; i < gic_data.nr_redist_regions; i++) {
787 		void __iomem *ptr = gic_data.redist_regions[i].redist_base;
788 		u64 typer;
789 		u32 reg;
790 
791 		reg = readl_relaxed(ptr + GICR_PIDR2) & GIC_PIDR2_ARCH_MASK;
792 		if (reg != GIC_PIDR2_ARCH_GICv3 &&
793 		    reg != GIC_PIDR2_ARCH_GICv4) { /* We're in trouble... */
794 			pr_warn("No redistributor present @%p\n", ptr);
795 			break;
796 		}
797 
798 		do {
799 			typer = gic_read_typer(ptr + GICR_TYPER);
800 			ret = fn(gic_data.redist_regions + i, ptr);
801 			if (!ret)
802 				return 0;
803 
804 			if (gic_data.redist_regions[i].single_redist)
805 				break;
806 
807 			if (gic_data.redist_stride) {
808 				ptr += gic_data.redist_stride;
809 			} else {
810 				ptr += SZ_64K * 2; /* Skip RD_base + SGI_base */
811 				if (typer & GICR_TYPER_VLPIS)
812 					ptr += SZ_64K * 2; /* Skip VLPI_base + reserved page */
813 			}
814 		} while (!(typer & GICR_TYPER_LAST));
815 	}
816 
817 	return ret ? -ENODEV : 0;
818 }
819 
820 static int __gic_populate_rdist(struct redist_region *region, void __iomem *ptr)
821 {
822 	unsigned long mpidr = cpu_logical_map(smp_processor_id());
823 	u64 typer;
824 	u32 aff;
825 
826 	/*
827 	 * Convert affinity to a 32bit value that can be matched to
828 	 * GICR_TYPER bits [63:32].
829 	 */
830 	aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 |
831 	       MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
832 	       MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 |
833 	       MPIDR_AFFINITY_LEVEL(mpidr, 0));
834 
835 	typer = gic_read_typer(ptr + GICR_TYPER);
836 	if ((typer >> 32) == aff) {
837 		u64 offset = ptr - region->redist_base;
838 		raw_spin_lock_init(&gic_data_rdist()->rd_lock);
839 		gic_data_rdist_rd_base() = ptr;
840 		gic_data_rdist()->phys_base = region->phys_base + offset;
841 
842 		pr_info("CPU%d: found redistributor %lx region %d:%pa\n",
843 			smp_processor_id(), mpidr,
844 			(int)(region - gic_data.redist_regions),
845 			&gic_data_rdist()->phys_base);
846 		return 0;
847 	}
848 
849 	/* Try next one */
850 	return 1;
851 }
852 
853 static int gic_populate_rdist(void)
854 {
855 	if (gic_iterate_rdists(__gic_populate_rdist) == 0)
856 		return 0;
857 
858 	/* We couldn't even deal with ourselves... */
859 	WARN(true, "CPU%d: mpidr %lx has no re-distributor!\n",
860 	     smp_processor_id(),
861 	     (unsigned long)cpu_logical_map(smp_processor_id()));
862 	return -ENODEV;
863 }
864 
865 static int __gic_update_rdist_properties(struct redist_region *region,
866 					 void __iomem *ptr)
867 {
868 	u64 typer = gic_read_typer(ptr + GICR_TYPER);
869 
870 	gic_data.rdists.has_vlpis &= !!(typer & GICR_TYPER_VLPIS);
871 
872 	/* RVPEID implies some form of DirectLPI, no matter what the doc says... :-/ */
873 	gic_data.rdists.has_rvpeid &= !!(typer & GICR_TYPER_RVPEID);
874 	gic_data.rdists.has_direct_lpi &= (!!(typer & GICR_TYPER_DirectLPIS) |
875 					   gic_data.rdists.has_rvpeid);
876 	gic_data.rdists.has_vpend_valid_dirty &= !!(typer & GICR_TYPER_DIRTY);
877 
878 	/* Detect non-sensical configurations */
879 	if (WARN_ON_ONCE(gic_data.rdists.has_rvpeid && !gic_data.rdists.has_vlpis)) {
880 		gic_data.rdists.has_direct_lpi = false;
881 		gic_data.rdists.has_vlpis = false;
882 		gic_data.rdists.has_rvpeid = false;
883 	}
884 
885 	gic_data.ppi_nr = min(GICR_TYPER_NR_PPIS(typer), gic_data.ppi_nr);
886 
887 	return 1;
888 }
889 
890 static void gic_update_rdist_properties(void)
891 {
892 	gic_data.ppi_nr = UINT_MAX;
893 	gic_iterate_rdists(__gic_update_rdist_properties);
894 	if (WARN_ON(gic_data.ppi_nr == UINT_MAX))
895 		gic_data.ppi_nr = 0;
896 	pr_info("%d PPIs implemented\n", gic_data.ppi_nr);
897 	if (gic_data.rdists.has_vlpis)
898 		pr_info("GICv4 features: %s%s%s\n",
899 			gic_data.rdists.has_direct_lpi ? "DirectLPI " : "",
900 			gic_data.rdists.has_rvpeid ? "RVPEID " : "",
901 			gic_data.rdists.has_vpend_valid_dirty ? "Valid+Dirty " : "");
902 }
903 
904 /* Check whether it's single security state view */
905 static inline bool gic_dist_security_disabled(void)
906 {
907 	return readl_relaxed(gic_data.dist_base + GICD_CTLR) & GICD_CTLR_DS;
908 }
909 
910 static void gic_cpu_sys_reg_init(void)
911 {
912 	int i, cpu = smp_processor_id();
913 	u64 mpidr = cpu_logical_map(cpu);
914 	u64 need_rss = MPIDR_RS(mpidr);
915 	bool group0;
916 	u32 pribits;
917 
918 	/*
919 	 * Need to check that the SRE bit has actually been set. If
920 	 * not, it means that SRE is disabled at EL2. We're going to
921 	 * die painfully, and there is nothing we can do about it.
922 	 *
923 	 * Kindly inform the luser.
924 	 */
925 	if (!gic_enable_sre())
926 		pr_err("GIC: unable to set SRE (disabled at EL2), panic ahead\n");
927 
928 	pribits = gic_get_pribits();
929 
930 	group0 = gic_has_group0();
931 
932 	/* Set priority mask register */
933 	if (!gic_prio_masking_enabled()) {
934 		write_gicreg(DEFAULT_PMR_VALUE, ICC_PMR_EL1);
935 	} else {
936 		/*
937 		 * Mismatch configuration with boot CPU, the system is likely
938 		 * to die as interrupt masking will not work properly on all
939 		 * CPUs
940 		 */
941 		WARN_ON(gic_supports_nmi() && group0 &&
942 			!gic_dist_security_disabled());
943 	}
944 
945 	/*
946 	 * Some firmwares hand over to the kernel with the BPR changed from
947 	 * its reset value (and with a value large enough to prevent
948 	 * any pre-emptive interrupts from working at all). Writing a zero
949 	 * to BPR restores is reset value.
950 	 */
951 	gic_write_bpr1(0);
952 
953 	if (static_branch_likely(&supports_deactivate_key)) {
954 		/* EOI drops priority only (mode 1) */
955 		gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop);
956 	} else {
957 		/* EOI deactivates interrupt too (mode 0) */
958 		gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop_dir);
959 	}
960 
961 	/* Always whack Group0 before Group1 */
962 	if (group0) {
963 		switch(pribits) {
964 		case 8:
965 		case 7:
966 			write_gicreg(0, ICC_AP0R3_EL1);
967 			write_gicreg(0, ICC_AP0R2_EL1);
968 		/* Fall through */
969 		case 6:
970 			write_gicreg(0, ICC_AP0R1_EL1);
971 		/* Fall through */
972 		case 5:
973 		case 4:
974 			write_gicreg(0, ICC_AP0R0_EL1);
975 		}
976 
977 		isb();
978 	}
979 
980 	switch(pribits) {
981 	case 8:
982 	case 7:
983 		write_gicreg(0, ICC_AP1R3_EL1);
984 		write_gicreg(0, ICC_AP1R2_EL1);
985 		/* Fall through */
986 	case 6:
987 		write_gicreg(0, ICC_AP1R1_EL1);
988 		/* Fall through */
989 	case 5:
990 	case 4:
991 		write_gicreg(0, ICC_AP1R0_EL1);
992 	}
993 
994 	isb();
995 
996 	/* ... and let's hit the road... */
997 	gic_write_grpen1(1);
998 
999 	/* Keep the RSS capability status in per_cpu variable */
1000 	per_cpu(has_rss, cpu) = !!(gic_read_ctlr() & ICC_CTLR_EL1_RSS);
1001 
1002 	/* Check all the CPUs have capable of sending SGIs to other CPUs */
1003 	for_each_online_cpu(i) {
1004 		bool have_rss = per_cpu(has_rss, i) && per_cpu(has_rss, cpu);
1005 
1006 		need_rss |= MPIDR_RS(cpu_logical_map(i));
1007 		if (need_rss && (!have_rss))
1008 			pr_crit("CPU%d (%lx) can't SGI CPU%d (%lx), no RSS\n",
1009 				cpu, (unsigned long)mpidr,
1010 				i, (unsigned long)cpu_logical_map(i));
1011 	}
1012 
1013 	/**
1014 	 * GIC spec says, when ICC_CTLR_EL1.RSS==1 and GICD_TYPER.RSS==0,
1015 	 * writing ICC_ASGI1R_EL1 register with RS != 0 is a CONSTRAINED
1016 	 * UNPREDICTABLE choice of :
1017 	 *   - The write is ignored.
1018 	 *   - The RS field is treated as 0.
1019 	 */
1020 	if (need_rss && (!gic_data.has_rss))
1021 		pr_crit_once("RSS is required but GICD doesn't support it\n");
1022 }
1023 
1024 static bool gicv3_nolpi;
1025 
1026 static int __init gicv3_nolpi_cfg(char *buf)
1027 {
1028 	return strtobool(buf, &gicv3_nolpi);
1029 }
1030 early_param("irqchip.gicv3_nolpi", gicv3_nolpi_cfg);
1031 
1032 static int gic_dist_supports_lpis(void)
1033 {
1034 	return (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) &&
1035 		!!(readl_relaxed(gic_data.dist_base + GICD_TYPER) & GICD_TYPER_LPIS) &&
1036 		!gicv3_nolpi);
1037 }
1038 
1039 static void gic_cpu_init(void)
1040 {
1041 	void __iomem *rbase;
1042 	int i;
1043 
1044 	/* Register ourselves with the rest of the world */
1045 	if (gic_populate_rdist())
1046 		return;
1047 
1048 	gic_enable_redist(true);
1049 
1050 	WARN((gic_data.ppi_nr > 16 || GIC_ESPI_NR != 0) &&
1051 	     !(gic_read_ctlr() & ICC_CTLR_EL1_ExtRange),
1052 	     "Distributor has extended ranges, but CPU%d doesn't\n",
1053 	     smp_processor_id());
1054 
1055 	rbase = gic_data_rdist_sgi_base();
1056 
1057 	/* Configure SGIs/PPIs as non-secure Group-1 */
1058 	for (i = 0; i < gic_data.ppi_nr + 16; i += 32)
1059 		writel_relaxed(~0, rbase + GICR_IGROUPR0 + i / 8);
1060 
1061 	gic_cpu_config(rbase, gic_data.ppi_nr + 16, gic_redist_wait_for_rwp);
1062 
1063 	/* initialise system registers */
1064 	gic_cpu_sys_reg_init();
1065 }
1066 
1067 #ifdef CONFIG_SMP
1068 
1069 #define MPIDR_TO_SGI_RS(mpidr)	(MPIDR_RS(mpidr) << ICC_SGI1R_RS_SHIFT)
1070 #define MPIDR_TO_SGI_CLUSTER_ID(mpidr)	((mpidr) & ~0xFUL)
1071 
1072 static int gic_starting_cpu(unsigned int cpu)
1073 {
1074 	gic_cpu_init();
1075 
1076 	if (gic_dist_supports_lpis())
1077 		its_cpu_init();
1078 
1079 	return 0;
1080 }
1081 
1082 static u16 gic_compute_target_list(int *base_cpu, const struct cpumask *mask,
1083 				   unsigned long cluster_id)
1084 {
1085 	int next_cpu, cpu = *base_cpu;
1086 	unsigned long mpidr = cpu_logical_map(cpu);
1087 	u16 tlist = 0;
1088 
1089 	while (cpu < nr_cpu_ids) {
1090 		tlist |= 1 << (mpidr & 0xf);
1091 
1092 		next_cpu = cpumask_next(cpu, mask);
1093 		if (next_cpu >= nr_cpu_ids)
1094 			goto out;
1095 		cpu = next_cpu;
1096 
1097 		mpidr = cpu_logical_map(cpu);
1098 
1099 		if (cluster_id != MPIDR_TO_SGI_CLUSTER_ID(mpidr)) {
1100 			cpu--;
1101 			goto out;
1102 		}
1103 	}
1104 out:
1105 	*base_cpu = cpu;
1106 	return tlist;
1107 }
1108 
1109 #define MPIDR_TO_SGI_AFFINITY(cluster_id, level) \
1110 	(MPIDR_AFFINITY_LEVEL(cluster_id, level) \
1111 		<< ICC_SGI1R_AFFINITY_## level ##_SHIFT)
1112 
1113 static void gic_send_sgi(u64 cluster_id, u16 tlist, unsigned int irq)
1114 {
1115 	u64 val;
1116 
1117 	val = (MPIDR_TO_SGI_AFFINITY(cluster_id, 3)	|
1118 	       MPIDR_TO_SGI_AFFINITY(cluster_id, 2)	|
1119 	       irq << ICC_SGI1R_SGI_ID_SHIFT		|
1120 	       MPIDR_TO_SGI_AFFINITY(cluster_id, 1)	|
1121 	       MPIDR_TO_SGI_RS(cluster_id)		|
1122 	       tlist << ICC_SGI1R_TARGET_LIST_SHIFT);
1123 
1124 	pr_devel("CPU%d: ICC_SGI1R_EL1 %llx\n", smp_processor_id(), val);
1125 	gic_write_sgi1r(val);
1126 }
1127 
1128 static void gic_raise_softirq(const struct cpumask *mask, unsigned int irq)
1129 {
1130 	int cpu;
1131 
1132 	if (WARN_ON(irq >= 16))
1133 		return;
1134 
1135 	/*
1136 	 * Ensure that stores to Normal memory are visible to the
1137 	 * other CPUs before issuing the IPI.
1138 	 */
1139 	wmb();
1140 
1141 	for_each_cpu(cpu, mask) {
1142 		u64 cluster_id = MPIDR_TO_SGI_CLUSTER_ID(cpu_logical_map(cpu));
1143 		u16 tlist;
1144 
1145 		tlist = gic_compute_target_list(&cpu, mask, cluster_id);
1146 		gic_send_sgi(cluster_id, tlist, irq);
1147 	}
1148 
1149 	/* Force the above writes to ICC_SGI1R_EL1 to be executed */
1150 	isb();
1151 }
1152 
1153 static void __init gic_smp_init(void)
1154 {
1155 	set_smp_cross_call(gic_raise_softirq);
1156 	cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_GIC_STARTING,
1157 				  "irqchip/arm/gicv3:starting",
1158 				  gic_starting_cpu, NULL);
1159 }
1160 
1161 static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
1162 			    bool force)
1163 {
1164 	unsigned int cpu;
1165 	u32 offset, index;
1166 	void __iomem *reg;
1167 	int enabled;
1168 	u64 val;
1169 
1170 	if (force)
1171 		cpu = cpumask_first(mask_val);
1172 	else
1173 		cpu = cpumask_any_and(mask_val, cpu_online_mask);
1174 
1175 	if (cpu >= nr_cpu_ids)
1176 		return -EINVAL;
1177 
1178 	if (gic_irq_in_rdist(d))
1179 		return -EINVAL;
1180 
1181 	/* If interrupt was enabled, disable it first */
1182 	enabled = gic_peek_irq(d, GICD_ISENABLER);
1183 	if (enabled)
1184 		gic_mask_irq(d);
1185 
1186 	offset = convert_offset_index(d, GICD_IROUTER, &index);
1187 	reg = gic_dist_base(d) + offset + (index * 8);
1188 	val = gic_mpidr_to_affinity(cpu_logical_map(cpu));
1189 
1190 	gic_write_irouter(val, reg);
1191 
1192 	/*
1193 	 * If the interrupt was enabled, enabled it again. Otherwise,
1194 	 * just wait for the distributor to have digested our changes.
1195 	 */
1196 	if (enabled)
1197 		gic_unmask_irq(d);
1198 	else
1199 		gic_dist_wait_for_rwp();
1200 
1201 	irq_data_update_effective_affinity(d, cpumask_of(cpu));
1202 
1203 	return IRQ_SET_MASK_OK_DONE;
1204 }
1205 #else
1206 #define gic_set_affinity	NULL
1207 #define gic_smp_init()		do { } while(0)
1208 #endif
1209 
1210 #ifdef CONFIG_CPU_PM
1211 static int gic_cpu_pm_notifier(struct notifier_block *self,
1212 			       unsigned long cmd, void *v)
1213 {
1214 	if (cmd == CPU_PM_EXIT) {
1215 		if (gic_dist_security_disabled())
1216 			gic_enable_redist(true);
1217 		gic_cpu_sys_reg_init();
1218 	} else if (cmd == CPU_PM_ENTER && gic_dist_security_disabled()) {
1219 		gic_write_grpen1(0);
1220 		gic_enable_redist(false);
1221 	}
1222 	return NOTIFY_OK;
1223 }
1224 
1225 static struct notifier_block gic_cpu_pm_notifier_block = {
1226 	.notifier_call = gic_cpu_pm_notifier,
1227 };
1228 
1229 static void gic_cpu_pm_init(void)
1230 {
1231 	cpu_pm_register_notifier(&gic_cpu_pm_notifier_block);
1232 }
1233 
1234 #else
1235 static inline void gic_cpu_pm_init(void) { }
1236 #endif /* CONFIG_CPU_PM */
1237 
1238 static struct irq_chip gic_chip = {
1239 	.name			= "GICv3",
1240 	.irq_mask		= gic_mask_irq,
1241 	.irq_unmask		= gic_unmask_irq,
1242 	.irq_eoi		= gic_eoi_irq,
1243 	.irq_set_type		= gic_set_type,
1244 	.irq_set_affinity	= gic_set_affinity,
1245 	.irq_get_irqchip_state	= gic_irq_get_irqchip_state,
1246 	.irq_set_irqchip_state	= gic_irq_set_irqchip_state,
1247 	.irq_nmi_setup		= gic_irq_nmi_setup,
1248 	.irq_nmi_teardown	= gic_irq_nmi_teardown,
1249 	.flags			= IRQCHIP_SET_TYPE_MASKED |
1250 				  IRQCHIP_SKIP_SET_WAKE |
1251 				  IRQCHIP_MASK_ON_SUSPEND,
1252 };
1253 
1254 static struct irq_chip gic_eoimode1_chip = {
1255 	.name			= "GICv3",
1256 	.irq_mask		= gic_eoimode1_mask_irq,
1257 	.irq_unmask		= gic_unmask_irq,
1258 	.irq_eoi		= gic_eoimode1_eoi_irq,
1259 	.irq_set_type		= gic_set_type,
1260 	.irq_set_affinity	= gic_set_affinity,
1261 	.irq_get_irqchip_state	= gic_irq_get_irqchip_state,
1262 	.irq_set_irqchip_state	= gic_irq_set_irqchip_state,
1263 	.irq_set_vcpu_affinity	= gic_irq_set_vcpu_affinity,
1264 	.irq_nmi_setup		= gic_irq_nmi_setup,
1265 	.irq_nmi_teardown	= gic_irq_nmi_teardown,
1266 	.flags			= IRQCHIP_SET_TYPE_MASKED |
1267 				  IRQCHIP_SKIP_SET_WAKE |
1268 				  IRQCHIP_MASK_ON_SUSPEND,
1269 };
1270 
1271 static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq,
1272 			      irq_hw_number_t hw)
1273 {
1274 	struct irq_chip *chip = &gic_chip;
1275 
1276 	if (static_branch_likely(&supports_deactivate_key))
1277 		chip = &gic_eoimode1_chip;
1278 
1279 	switch (__get_intid_range(hw)) {
1280 	case PPI_RANGE:
1281 	case EPPI_RANGE:
1282 		irq_set_percpu_devid(irq);
1283 		irq_domain_set_info(d, irq, hw, chip, d->host_data,
1284 				    handle_percpu_devid_irq, NULL, NULL);
1285 		break;
1286 
1287 	case SPI_RANGE:
1288 	case ESPI_RANGE:
1289 		irq_domain_set_info(d, irq, hw, chip, d->host_data,
1290 				    handle_fasteoi_irq, NULL, NULL);
1291 		irq_set_probe(irq);
1292 		irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(irq)));
1293 		break;
1294 
1295 	case LPI_RANGE:
1296 		if (!gic_dist_supports_lpis())
1297 			return -EPERM;
1298 		irq_domain_set_info(d, irq, hw, chip, d->host_data,
1299 				    handle_fasteoi_irq, NULL, NULL);
1300 		break;
1301 
1302 	default:
1303 		return -EPERM;
1304 	}
1305 
1306 	return 0;
1307 }
1308 
1309 #define GIC_IRQ_TYPE_PARTITION	(GIC_IRQ_TYPE_LPI + 1)
1310 
1311 static int gic_irq_domain_translate(struct irq_domain *d,
1312 				    struct irq_fwspec *fwspec,
1313 				    unsigned long *hwirq,
1314 				    unsigned int *type)
1315 {
1316 	if (is_of_node(fwspec->fwnode)) {
1317 		if (fwspec->param_count < 3)
1318 			return -EINVAL;
1319 
1320 		switch (fwspec->param[0]) {
1321 		case 0:			/* SPI */
1322 			*hwirq = fwspec->param[1] + 32;
1323 			break;
1324 		case 1:			/* PPI */
1325 			*hwirq = fwspec->param[1] + 16;
1326 			break;
1327 		case 2:			/* ESPI */
1328 			*hwirq = fwspec->param[1] + ESPI_BASE_INTID;
1329 			break;
1330 		case 3:			/* EPPI */
1331 			*hwirq = fwspec->param[1] + EPPI_BASE_INTID;
1332 			break;
1333 		case GIC_IRQ_TYPE_LPI:	/* LPI */
1334 			*hwirq = fwspec->param[1];
1335 			break;
1336 		case GIC_IRQ_TYPE_PARTITION:
1337 			*hwirq = fwspec->param[1];
1338 			if (fwspec->param[1] >= 16)
1339 				*hwirq += EPPI_BASE_INTID - 16;
1340 			else
1341 				*hwirq += 16;
1342 			break;
1343 		default:
1344 			return -EINVAL;
1345 		}
1346 
1347 		*type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK;
1348 
1349 		/*
1350 		 * Make it clear that broken DTs are... broken.
1351 		 * Partitionned PPIs are an unfortunate exception.
1352 		 */
1353 		WARN_ON(*type == IRQ_TYPE_NONE &&
1354 			fwspec->param[0] != GIC_IRQ_TYPE_PARTITION);
1355 		return 0;
1356 	}
1357 
1358 	if (is_fwnode_irqchip(fwspec->fwnode)) {
1359 		if(fwspec->param_count != 2)
1360 			return -EINVAL;
1361 
1362 		*hwirq = fwspec->param[0];
1363 		*type = fwspec->param[1];
1364 
1365 		WARN_ON(*type == IRQ_TYPE_NONE);
1366 		return 0;
1367 	}
1368 
1369 	return -EINVAL;
1370 }
1371 
1372 static int gic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
1373 				unsigned int nr_irqs, void *arg)
1374 {
1375 	int i, ret;
1376 	irq_hw_number_t hwirq;
1377 	unsigned int type = IRQ_TYPE_NONE;
1378 	struct irq_fwspec *fwspec = arg;
1379 
1380 	ret = gic_irq_domain_translate(domain, fwspec, &hwirq, &type);
1381 	if (ret)
1382 		return ret;
1383 
1384 	for (i = 0; i < nr_irqs; i++) {
1385 		ret = gic_irq_domain_map(domain, virq + i, hwirq + i);
1386 		if (ret)
1387 			return ret;
1388 	}
1389 
1390 	return 0;
1391 }
1392 
1393 static void gic_irq_domain_free(struct irq_domain *domain, unsigned int virq,
1394 				unsigned int nr_irqs)
1395 {
1396 	int i;
1397 
1398 	for (i = 0; i < nr_irqs; i++) {
1399 		struct irq_data *d = irq_domain_get_irq_data(domain, virq + i);
1400 		irq_set_handler(virq + i, NULL);
1401 		irq_domain_reset_irq_data(d);
1402 	}
1403 }
1404 
1405 static int gic_irq_domain_select(struct irq_domain *d,
1406 				 struct irq_fwspec *fwspec,
1407 				 enum irq_domain_bus_token bus_token)
1408 {
1409 	/* Not for us */
1410         if (fwspec->fwnode != d->fwnode)
1411 		return 0;
1412 
1413 	/* If this is not DT, then we have a single domain */
1414 	if (!is_of_node(fwspec->fwnode))
1415 		return 1;
1416 
1417 	/*
1418 	 * If this is a PPI and we have a 4th (non-null) parameter,
1419 	 * then we need to match the partition domain.
1420 	 */
1421 	if (fwspec->param_count >= 4 &&
1422 	    fwspec->param[0] == 1 && fwspec->param[3] != 0 &&
1423 	    gic_data.ppi_descs)
1424 		return d == partition_get_domain(gic_data.ppi_descs[fwspec->param[1]]);
1425 
1426 	return d == gic_data.domain;
1427 }
1428 
1429 static const struct irq_domain_ops gic_irq_domain_ops = {
1430 	.translate = gic_irq_domain_translate,
1431 	.alloc = gic_irq_domain_alloc,
1432 	.free = gic_irq_domain_free,
1433 	.select = gic_irq_domain_select,
1434 };
1435 
1436 static int partition_domain_translate(struct irq_domain *d,
1437 				      struct irq_fwspec *fwspec,
1438 				      unsigned long *hwirq,
1439 				      unsigned int *type)
1440 {
1441 	struct device_node *np;
1442 	int ret;
1443 
1444 	if (!gic_data.ppi_descs)
1445 		return -ENOMEM;
1446 
1447 	np = of_find_node_by_phandle(fwspec->param[3]);
1448 	if (WARN_ON(!np))
1449 		return -EINVAL;
1450 
1451 	ret = partition_translate_id(gic_data.ppi_descs[fwspec->param[1]],
1452 				     of_node_to_fwnode(np));
1453 	if (ret < 0)
1454 		return ret;
1455 
1456 	*hwirq = ret;
1457 	*type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK;
1458 
1459 	return 0;
1460 }
1461 
1462 static const struct irq_domain_ops partition_domain_ops = {
1463 	.translate = partition_domain_translate,
1464 	.select = gic_irq_domain_select,
1465 };
1466 
1467 static bool gic_enable_quirk_msm8996(void *data)
1468 {
1469 	struct gic_chip_data *d = data;
1470 
1471 	d->flags |= FLAGS_WORKAROUND_GICR_WAKER_MSM8996;
1472 
1473 	return true;
1474 }
1475 
1476 static bool gic_enable_quirk_cavium_38539(void *data)
1477 {
1478 	struct gic_chip_data *d = data;
1479 
1480 	d->flags |= FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539;
1481 
1482 	return true;
1483 }
1484 
1485 static bool gic_enable_quirk_hip06_07(void *data)
1486 {
1487 	struct gic_chip_data *d = data;
1488 
1489 	/*
1490 	 * HIP06 GICD_IIDR clashes with GIC-600 product number (despite
1491 	 * not being an actual ARM implementation). The saving grace is
1492 	 * that GIC-600 doesn't have ESPI, so nothing to do in that case.
1493 	 * HIP07 doesn't even have a proper IIDR, and still pretends to
1494 	 * have ESPI. In both cases, put them right.
1495 	 */
1496 	if (d->rdists.gicd_typer & GICD_TYPER_ESPI) {
1497 		/* Zero both ESPI and the RES0 field next to it... */
1498 		d->rdists.gicd_typer &= ~GENMASK(9, 8);
1499 		return true;
1500 	}
1501 
1502 	return false;
1503 }
1504 
1505 static const struct gic_quirk gic_quirks[] = {
1506 	{
1507 		.desc	= "GICv3: Qualcomm MSM8996 broken firmware",
1508 		.compatible = "qcom,msm8996-gic-v3",
1509 		.init	= gic_enable_quirk_msm8996,
1510 	},
1511 	{
1512 		.desc	= "GICv3: HIP06 erratum 161010803",
1513 		.iidr	= 0x0204043b,
1514 		.mask	= 0xffffffff,
1515 		.init	= gic_enable_quirk_hip06_07,
1516 	},
1517 	{
1518 		.desc	= "GICv3: HIP07 erratum 161010803",
1519 		.iidr	= 0x00000000,
1520 		.mask	= 0xffffffff,
1521 		.init	= gic_enable_quirk_hip06_07,
1522 	},
1523 	{
1524 		/*
1525 		 * Reserved register accesses generate a Synchronous
1526 		 * External Abort. This erratum applies to:
1527 		 * - ThunderX: CN88xx
1528 		 * - OCTEON TX: CN83xx, CN81xx
1529 		 * - OCTEON TX2: CN93xx, CN96xx, CN98xx, CNF95xx*
1530 		 */
1531 		.desc	= "GICv3: Cavium erratum 38539",
1532 		.iidr	= 0xa000034c,
1533 		.mask	= 0xe8f00fff,
1534 		.init	= gic_enable_quirk_cavium_38539,
1535 	},
1536 	{
1537 	}
1538 };
1539 
1540 static void gic_enable_nmi_support(void)
1541 {
1542 	int i;
1543 
1544 	if (!gic_prio_masking_enabled())
1545 		return;
1546 
1547 	if (gic_has_group0() && !gic_dist_security_disabled()) {
1548 		pr_warn("SCR_EL3.FIQ is cleared, cannot enable use of pseudo-NMIs\n");
1549 		return;
1550 	}
1551 
1552 	ppi_nmi_refs = kcalloc(gic_data.ppi_nr, sizeof(*ppi_nmi_refs), GFP_KERNEL);
1553 	if (!ppi_nmi_refs)
1554 		return;
1555 
1556 	for (i = 0; i < gic_data.ppi_nr; i++)
1557 		refcount_set(&ppi_nmi_refs[i], 0);
1558 
1559 	/*
1560 	 * Linux itself doesn't use 1:N distribution, so has no need to
1561 	 * set PMHE. The only reason to have it set is if EL3 requires it
1562 	 * (and we can't change it).
1563 	 */
1564 	if (gic_read_ctlr() & ICC_CTLR_EL1_PMHE_MASK)
1565 		static_branch_enable(&gic_pmr_sync);
1566 
1567 	pr_info("%s ICC_PMR_EL1 synchronisation\n",
1568 		static_branch_unlikely(&gic_pmr_sync) ? "Forcing" : "Relaxing");
1569 
1570 	static_branch_enable(&supports_pseudo_nmis);
1571 
1572 	if (static_branch_likely(&supports_deactivate_key))
1573 		gic_eoimode1_chip.flags |= IRQCHIP_SUPPORTS_NMI;
1574 	else
1575 		gic_chip.flags |= IRQCHIP_SUPPORTS_NMI;
1576 }
1577 
1578 static int __init gic_init_bases(void __iomem *dist_base,
1579 				 struct redist_region *rdist_regs,
1580 				 u32 nr_redist_regions,
1581 				 u64 redist_stride,
1582 				 struct fwnode_handle *handle)
1583 {
1584 	u32 typer;
1585 	int err;
1586 
1587 	if (!is_hyp_mode_available())
1588 		static_branch_disable(&supports_deactivate_key);
1589 
1590 	if (static_branch_likely(&supports_deactivate_key))
1591 		pr_info("GIC: Using split EOI/Deactivate mode\n");
1592 
1593 	gic_data.fwnode = handle;
1594 	gic_data.dist_base = dist_base;
1595 	gic_data.redist_regions = rdist_regs;
1596 	gic_data.nr_redist_regions = nr_redist_regions;
1597 	gic_data.redist_stride = redist_stride;
1598 
1599 	/*
1600 	 * Find out how many interrupts are supported.
1601 	 */
1602 	typer = readl_relaxed(gic_data.dist_base + GICD_TYPER);
1603 	gic_data.rdists.gicd_typer = typer;
1604 
1605 	gic_enable_quirks(readl_relaxed(gic_data.dist_base + GICD_IIDR),
1606 			  gic_quirks, &gic_data);
1607 
1608 	pr_info("%d SPIs implemented\n", GIC_LINE_NR - 32);
1609 	pr_info("%d Extended SPIs implemented\n", GIC_ESPI_NR);
1610 
1611 	/*
1612 	 * ThunderX1 explodes on reading GICD_TYPER2, in violation of the
1613 	 * architecture spec (which says that reserved registers are RES0).
1614 	 */
1615 	if (!(gic_data.flags & FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539))
1616 		gic_data.rdists.gicd_typer2 = readl_relaxed(gic_data.dist_base + GICD_TYPER2);
1617 
1618 	gic_data.domain = irq_domain_create_tree(handle, &gic_irq_domain_ops,
1619 						 &gic_data);
1620 	gic_data.rdists.rdist = alloc_percpu(typeof(*gic_data.rdists.rdist));
1621 	gic_data.rdists.has_rvpeid = true;
1622 	gic_data.rdists.has_vlpis = true;
1623 	gic_data.rdists.has_direct_lpi = true;
1624 	gic_data.rdists.has_vpend_valid_dirty = true;
1625 
1626 	if (WARN_ON(!gic_data.domain) || WARN_ON(!gic_data.rdists.rdist)) {
1627 		err = -ENOMEM;
1628 		goto out_free;
1629 	}
1630 
1631 	irq_domain_update_bus_token(gic_data.domain, DOMAIN_BUS_WIRED);
1632 
1633 	gic_data.has_rss = !!(typer & GICD_TYPER_RSS);
1634 	pr_info("Distributor has %sRange Selector support\n",
1635 		gic_data.has_rss ? "" : "no ");
1636 
1637 	if (typer & GICD_TYPER_MBIS) {
1638 		err = mbi_init(handle, gic_data.domain);
1639 		if (err)
1640 			pr_err("Failed to initialize MBIs\n");
1641 	}
1642 
1643 	set_handle_irq(gic_handle_irq);
1644 
1645 	gic_update_rdist_properties();
1646 
1647 	gic_smp_init();
1648 	gic_dist_init();
1649 	gic_cpu_init();
1650 	gic_cpu_pm_init();
1651 
1652 	if (gic_dist_supports_lpis()) {
1653 		its_init(handle, &gic_data.rdists, gic_data.domain);
1654 		its_cpu_init();
1655 	} else {
1656 		if (IS_ENABLED(CONFIG_ARM_GIC_V2M))
1657 			gicv2m_init(handle, gic_data.domain);
1658 	}
1659 
1660 	gic_enable_nmi_support();
1661 
1662 	return 0;
1663 
1664 out_free:
1665 	if (gic_data.domain)
1666 		irq_domain_remove(gic_data.domain);
1667 	free_percpu(gic_data.rdists.rdist);
1668 	return err;
1669 }
1670 
1671 static int __init gic_validate_dist_version(void __iomem *dist_base)
1672 {
1673 	u32 reg = readl_relaxed(dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK;
1674 
1675 	if (reg != GIC_PIDR2_ARCH_GICv3 && reg != GIC_PIDR2_ARCH_GICv4)
1676 		return -ENODEV;
1677 
1678 	return 0;
1679 }
1680 
1681 /* Create all possible partitions at boot time */
1682 static void __init gic_populate_ppi_partitions(struct device_node *gic_node)
1683 {
1684 	struct device_node *parts_node, *child_part;
1685 	int part_idx = 0, i;
1686 	int nr_parts;
1687 	struct partition_affinity *parts;
1688 
1689 	parts_node = of_get_child_by_name(gic_node, "ppi-partitions");
1690 	if (!parts_node)
1691 		return;
1692 
1693 	gic_data.ppi_descs = kcalloc(gic_data.ppi_nr, sizeof(*gic_data.ppi_descs), GFP_KERNEL);
1694 	if (!gic_data.ppi_descs)
1695 		return;
1696 
1697 	nr_parts = of_get_child_count(parts_node);
1698 
1699 	if (!nr_parts)
1700 		goto out_put_node;
1701 
1702 	parts = kcalloc(nr_parts, sizeof(*parts), GFP_KERNEL);
1703 	if (WARN_ON(!parts))
1704 		goto out_put_node;
1705 
1706 	for_each_child_of_node(parts_node, child_part) {
1707 		struct partition_affinity *part;
1708 		int n;
1709 
1710 		part = &parts[part_idx];
1711 
1712 		part->partition_id = of_node_to_fwnode(child_part);
1713 
1714 		pr_info("GIC: PPI partition %pOFn[%d] { ",
1715 			child_part, part_idx);
1716 
1717 		n = of_property_count_elems_of_size(child_part, "affinity",
1718 						    sizeof(u32));
1719 		WARN_ON(n <= 0);
1720 
1721 		for (i = 0; i < n; i++) {
1722 			int err, cpu;
1723 			u32 cpu_phandle;
1724 			struct device_node *cpu_node;
1725 
1726 			err = of_property_read_u32_index(child_part, "affinity",
1727 							 i, &cpu_phandle);
1728 			if (WARN_ON(err))
1729 				continue;
1730 
1731 			cpu_node = of_find_node_by_phandle(cpu_phandle);
1732 			if (WARN_ON(!cpu_node))
1733 				continue;
1734 
1735 			cpu = of_cpu_node_to_id(cpu_node);
1736 			if (WARN_ON(cpu < 0))
1737 				continue;
1738 
1739 			pr_cont("%pOF[%d] ", cpu_node, cpu);
1740 
1741 			cpumask_set_cpu(cpu, &part->mask);
1742 		}
1743 
1744 		pr_cont("}\n");
1745 		part_idx++;
1746 	}
1747 
1748 	for (i = 0; i < gic_data.ppi_nr; i++) {
1749 		unsigned int irq;
1750 		struct partition_desc *desc;
1751 		struct irq_fwspec ppi_fwspec = {
1752 			.fwnode		= gic_data.fwnode,
1753 			.param_count	= 3,
1754 			.param		= {
1755 				[0]	= GIC_IRQ_TYPE_PARTITION,
1756 				[1]	= i,
1757 				[2]	= IRQ_TYPE_NONE,
1758 			},
1759 		};
1760 
1761 		irq = irq_create_fwspec_mapping(&ppi_fwspec);
1762 		if (WARN_ON(!irq))
1763 			continue;
1764 		desc = partition_create_desc(gic_data.fwnode, parts, nr_parts,
1765 					     irq, &partition_domain_ops);
1766 		if (WARN_ON(!desc))
1767 			continue;
1768 
1769 		gic_data.ppi_descs[i] = desc;
1770 	}
1771 
1772 out_put_node:
1773 	of_node_put(parts_node);
1774 }
1775 
1776 static void __init gic_of_setup_kvm_info(struct device_node *node)
1777 {
1778 	int ret;
1779 	struct resource r;
1780 	u32 gicv_idx;
1781 
1782 	gic_v3_kvm_info.type = GIC_V3;
1783 
1784 	gic_v3_kvm_info.maint_irq = irq_of_parse_and_map(node, 0);
1785 	if (!gic_v3_kvm_info.maint_irq)
1786 		return;
1787 
1788 	if (of_property_read_u32(node, "#redistributor-regions",
1789 				 &gicv_idx))
1790 		gicv_idx = 1;
1791 
1792 	gicv_idx += 3;	/* Also skip GICD, GICC, GICH */
1793 	ret = of_address_to_resource(node, gicv_idx, &r);
1794 	if (!ret)
1795 		gic_v3_kvm_info.vcpu = r;
1796 
1797 	gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis;
1798 	gic_v3_kvm_info.has_v4_1 = gic_data.rdists.has_rvpeid;
1799 	gic_set_kvm_info(&gic_v3_kvm_info);
1800 }
1801 
1802 static int __init gic_of_init(struct device_node *node, struct device_node *parent)
1803 {
1804 	void __iomem *dist_base;
1805 	struct redist_region *rdist_regs;
1806 	u64 redist_stride;
1807 	u32 nr_redist_regions;
1808 	int err, i;
1809 
1810 	dist_base = of_iomap(node, 0);
1811 	if (!dist_base) {
1812 		pr_err("%pOF: unable to map gic dist registers\n", node);
1813 		return -ENXIO;
1814 	}
1815 
1816 	err = gic_validate_dist_version(dist_base);
1817 	if (err) {
1818 		pr_err("%pOF: no distributor detected, giving up\n", node);
1819 		goto out_unmap_dist;
1820 	}
1821 
1822 	if (of_property_read_u32(node, "#redistributor-regions", &nr_redist_regions))
1823 		nr_redist_regions = 1;
1824 
1825 	rdist_regs = kcalloc(nr_redist_regions, sizeof(*rdist_regs),
1826 			     GFP_KERNEL);
1827 	if (!rdist_regs) {
1828 		err = -ENOMEM;
1829 		goto out_unmap_dist;
1830 	}
1831 
1832 	for (i = 0; i < nr_redist_regions; i++) {
1833 		struct resource res;
1834 		int ret;
1835 
1836 		ret = of_address_to_resource(node, 1 + i, &res);
1837 		rdist_regs[i].redist_base = of_iomap(node, 1 + i);
1838 		if (ret || !rdist_regs[i].redist_base) {
1839 			pr_err("%pOF: couldn't map region %d\n", node, i);
1840 			err = -ENODEV;
1841 			goto out_unmap_rdist;
1842 		}
1843 		rdist_regs[i].phys_base = res.start;
1844 	}
1845 
1846 	if (of_property_read_u64(node, "redistributor-stride", &redist_stride))
1847 		redist_stride = 0;
1848 
1849 	gic_enable_of_quirks(node, gic_quirks, &gic_data);
1850 
1851 	err = gic_init_bases(dist_base, rdist_regs, nr_redist_regions,
1852 			     redist_stride, &node->fwnode);
1853 	if (err)
1854 		goto out_unmap_rdist;
1855 
1856 	gic_populate_ppi_partitions(node);
1857 
1858 	if (static_branch_likely(&supports_deactivate_key))
1859 		gic_of_setup_kvm_info(node);
1860 	return 0;
1861 
1862 out_unmap_rdist:
1863 	for (i = 0; i < nr_redist_regions; i++)
1864 		if (rdist_regs[i].redist_base)
1865 			iounmap(rdist_regs[i].redist_base);
1866 	kfree(rdist_regs);
1867 out_unmap_dist:
1868 	iounmap(dist_base);
1869 	return err;
1870 }
1871 
1872 IRQCHIP_DECLARE(gic_v3, "arm,gic-v3", gic_of_init);
1873 
1874 #ifdef CONFIG_ACPI
1875 static struct
1876 {
1877 	void __iomem *dist_base;
1878 	struct redist_region *redist_regs;
1879 	u32 nr_redist_regions;
1880 	bool single_redist;
1881 	int enabled_rdists;
1882 	u32 maint_irq;
1883 	int maint_irq_mode;
1884 	phys_addr_t vcpu_base;
1885 } acpi_data __initdata;
1886 
1887 static void __init
1888 gic_acpi_register_redist(phys_addr_t phys_base, void __iomem *redist_base)
1889 {
1890 	static int count = 0;
1891 
1892 	acpi_data.redist_regs[count].phys_base = phys_base;
1893 	acpi_data.redist_regs[count].redist_base = redist_base;
1894 	acpi_data.redist_regs[count].single_redist = acpi_data.single_redist;
1895 	count++;
1896 }
1897 
1898 static int __init
1899 gic_acpi_parse_madt_redist(union acpi_subtable_headers *header,
1900 			   const unsigned long end)
1901 {
1902 	struct acpi_madt_generic_redistributor *redist =
1903 			(struct acpi_madt_generic_redistributor *)header;
1904 	void __iomem *redist_base;
1905 
1906 	redist_base = ioremap(redist->base_address, redist->length);
1907 	if (!redist_base) {
1908 		pr_err("Couldn't map GICR region @%llx\n", redist->base_address);
1909 		return -ENOMEM;
1910 	}
1911 
1912 	gic_acpi_register_redist(redist->base_address, redist_base);
1913 	return 0;
1914 }
1915 
1916 static int __init
1917 gic_acpi_parse_madt_gicc(union acpi_subtable_headers *header,
1918 			 const unsigned long end)
1919 {
1920 	struct acpi_madt_generic_interrupt *gicc =
1921 				(struct acpi_madt_generic_interrupt *)header;
1922 	u32 reg = readl_relaxed(acpi_data.dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK;
1923 	u32 size = reg == GIC_PIDR2_ARCH_GICv4 ? SZ_64K * 4 : SZ_64K * 2;
1924 	void __iomem *redist_base;
1925 
1926 	/* GICC entry which has !ACPI_MADT_ENABLED is not unusable so skip */
1927 	if (!(gicc->flags & ACPI_MADT_ENABLED))
1928 		return 0;
1929 
1930 	redist_base = ioremap(gicc->gicr_base_address, size);
1931 	if (!redist_base)
1932 		return -ENOMEM;
1933 
1934 	gic_acpi_register_redist(gicc->gicr_base_address, redist_base);
1935 	return 0;
1936 }
1937 
1938 static int __init gic_acpi_collect_gicr_base(void)
1939 {
1940 	acpi_tbl_entry_handler redist_parser;
1941 	enum acpi_madt_type type;
1942 
1943 	if (acpi_data.single_redist) {
1944 		type = ACPI_MADT_TYPE_GENERIC_INTERRUPT;
1945 		redist_parser = gic_acpi_parse_madt_gicc;
1946 	} else {
1947 		type = ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR;
1948 		redist_parser = gic_acpi_parse_madt_redist;
1949 	}
1950 
1951 	/* Collect redistributor base addresses in GICR entries */
1952 	if (acpi_table_parse_madt(type, redist_parser, 0) > 0)
1953 		return 0;
1954 
1955 	pr_info("No valid GICR entries exist\n");
1956 	return -ENODEV;
1957 }
1958 
1959 static int __init gic_acpi_match_gicr(union acpi_subtable_headers *header,
1960 				  const unsigned long end)
1961 {
1962 	/* Subtable presence means that redist exists, that's it */
1963 	return 0;
1964 }
1965 
1966 static int __init gic_acpi_match_gicc(union acpi_subtable_headers *header,
1967 				      const unsigned long end)
1968 {
1969 	struct acpi_madt_generic_interrupt *gicc =
1970 				(struct acpi_madt_generic_interrupt *)header;
1971 
1972 	/*
1973 	 * If GICC is enabled and has valid gicr base address, then it means
1974 	 * GICR base is presented via GICC
1975 	 */
1976 	if ((gicc->flags & ACPI_MADT_ENABLED) && gicc->gicr_base_address) {
1977 		acpi_data.enabled_rdists++;
1978 		return 0;
1979 	}
1980 
1981 	/*
1982 	 * It's perfectly valid firmware can pass disabled GICC entry, driver
1983 	 * should not treat as errors, skip the entry instead of probe fail.
1984 	 */
1985 	if (!(gicc->flags & ACPI_MADT_ENABLED))
1986 		return 0;
1987 
1988 	return -ENODEV;
1989 }
1990 
1991 static int __init gic_acpi_count_gicr_regions(void)
1992 {
1993 	int count;
1994 
1995 	/*
1996 	 * Count how many redistributor regions we have. It is not allowed
1997 	 * to mix redistributor description, GICR and GICC subtables have to be
1998 	 * mutually exclusive.
1999 	 */
2000 	count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR,
2001 				      gic_acpi_match_gicr, 0);
2002 	if (count > 0) {
2003 		acpi_data.single_redist = false;
2004 		return count;
2005 	}
2006 
2007 	count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT,
2008 				      gic_acpi_match_gicc, 0);
2009 	if (count > 0) {
2010 		acpi_data.single_redist = true;
2011 		count = acpi_data.enabled_rdists;
2012 	}
2013 
2014 	return count;
2015 }
2016 
2017 static bool __init acpi_validate_gic_table(struct acpi_subtable_header *header,
2018 					   struct acpi_probe_entry *ape)
2019 {
2020 	struct acpi_madt_generic_distributor *dist;
2021 	int count;
2022 
2023 	dist = (struct acpi_madt_generic_distributor *)header;
2024 	if (dist->version != ape->driver_data)
2025 		return false;
2026 
2027 	/* We need to do that exercise anyway, the sooner the better */
2028 	count = gic_acpi_count_gicr_regions();
2029 	if (count <= 0)
2030 		return false;
2031 
2032 	acpi_data.nr_redist_regions = count;
2033 	return true;
2034 }
2035 
2036 static int __init gic_acpi_parse_virt_madt_gicc(union acpi_subtable_headers *header,
2037 						const unsigned long end)
2038 {
2039 	struct acpi_madt_generic_interrupt *gicc =
2040 		(struct acpi_madt_generic_interrupt *)header;
2041 	int maint_irq_mode;
2042 	static int first_madt = true;
2043 
2044 	/* Skip unusable CPUs */
2045 	if (!(gicc->flags & ACPI_MADT_ENABLED))
2046 		return 0;
2047 
2048 	maint_irq_mode = (gicc->flags & ACPI_MADT_VGIC_IRQ_MODE) ?
2049 		ACPI_EDGE_SENSITIVE : ACPI_LEVEL_SENSITIVE;
2050 
2051 	if (first_madt) {
2052 		first_madt = false;
2053 
2054 		acpi_data.maint_irq = gicc->vgic_interrupt;
2055 		acpi_data.maint_irq_mode = maint_irq_mode;
2056 		acpi_data.vcpu_base = gicc->gicv_base_address;
2057 
2058 		return 0;
2059 	}
2060 
2061 	/*
2062 	 * The maintenance interrupt and GICV should be the same for every CPU
2063 	 */
2064 	if ((acpi_data.maint_irq != gicc->vgic_interrupt) ||
2065 	    (acpi_data.maint_irq_mode != maint_irq_mode) ||
2066 	    (acpi_data.vcpu_base != gicc->gicv_base_address))
2067 		return -EINVAL;
2068 
2069 	return 0;
2070 }
2071 
2072 static bool __init gic_acpi_collect_virt_info(void)
2073 {
2074 	int count;
2075 
2076 	count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT,
2077 				      gic_acpi_parse_virt_madt_gicc, 0);
2078 
2079 	return (count > 0);
2080 }
2081 
2082 #define ACPI_GICV3_DIST_MEM_SIZE (SZ_64K)
2083 #define ACPI_GICV2_VCTRL_MEM_SIZE	(SZ_4K)
2084 #define ACPI_GICV2_VCPU_MEM_SIZE	(SZ_8K)
2085 
2086 static void __init gic_acpi_setup_kvm_info(void)
2087 {
2088 	int irq;
2089 
2090 	if (!gic_acpi_collect_virt_info()) {
2091 		pr_warn("Unable to get hardware information used for virtualization\n");
2092 		return;
2093 	}
2094 
2095 	gic_v3_kvm_info.type = GIC_V3;
2096 
2097 	irq = acpi_register_gsi(NULL, acpi_data.maint_irq,
2098 				acpi_data.maint_irq_mode,
2099 				ACPI_ACTIVE_HIGH);
2100 	if (irq <= 0)
2101 		return;
2102 
2103 	gic_v3_kvm_info.maint_irq = irq;
2104 
2105 	if (acpi_data.vcpu_base) {
2106 		struct resource *vcpu = &gic_v3_kvm_info.vcpu;
2107 
2108 		vcpu->flags = IORESOURCE_MEM;
2109 		vcpu->start = acpi_data.vcpu_base;
2110 		vcpu->end = vcpu->start + ACPI_GICV2_VCPU_MEM_SIZE - 1;
2111 	}
2112 
2113 	gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis;
2114 	gic_v3_kvm_info.has_v4_1 = gic_data.rdists.has_rvpeid;
2115 	gic_set_kvm_info(&gic_v3_kvm_info);
2116 }
2117 
2118 static int __init
2119 gic_acpi_init(struct acpi_subtable_header *header, const unsigned long end)
2120 {
2121 	struct acpi_madt_generic_distributor *dist;
2122 	struct fwnode_handle *domain_handle;
2123 	size_t size;
2124 	int i, err;
2125 
2126 	/* Get distributor base address */
2127 	dist = (struct acpi_madt_generic_distributor *)header;
2128 	acpi_data.dist_base = ioremap(dist->base_address,
2129 				      ACPI_GICV3_DIST_MEM_SIZE);
2130 	if (!acpi_data.dist_base) {
2131 		pr_err("Unable to map GICD registers\n");
2132 		return -ENOMEM;
2133 	}
2134 
2135 	err = gic_validate_dist_version(acpi_data.dist_base);
2136 	if (err) {
2137 		pr_err("No distributor detected at @%p, giving up\n",
2138 		       acpi_data.dist_base);
2139 		goto out_dist_unmap;
2140 	}
2141 
2142 	size = sizeof(*acpi_data.redist_regs) * acpi_data.nr_redist_regions;
2143 	acpi_data.redist_regs = kzalloc(size, GFP_KERNEL);
2144 	if (!acpi_data.redist_regs) {
2145 		err = -ENOMEM;
2146 		goto out_dist_unmap;
2147 	}
2148 
2149 	err = gic_acpi_collect_gicr_base();
2150 	if (err)
2151 		goto out_redist_unmap;
2152 
2153 	domain_handle = irq_domain_alloc_fwnode(&dist->base_address);
2154 	if (!domain_handle) {
2155 		err = -ENOMEM;
2156 		goto out_redist_unmap;
2157 	}
2158 
2159 	err = gic_init_bases(acpi_data.dist_base, acpi_data.redist_regs,
2160 			     acpi_data.nr_redist_regions, 0, domain_handle);
2161 	if (err)
2162 		goto out_fwhandle_free;
2163 
2164 	acpi_set_irq_model(ACPI_IRQ_MODEL_GIC, domain_handle);
2165 
2166 	if (static_branch_likely(&supports_deactivate_key))
2167 		gic_acpi_setup_kvm_info();
2168 
2169 	return 0;
2170 
2171 out_fwhandle_free:
2172 	irq_domain_free_fwnode(domain_handle);
2173 out_redist_unmap:
2174 	for (i = 0; i < acpi_data.nr_redist_regions; i++)
2175 		if (acpi_data.redist_regs[i].redist_base)
2176 			iounmap(acpi_data.redist_regs[i].redist_base);
2177 	kfree(acpi_data.redist_regs);
2178 out_dist_unmap:
2179 	iounmap(acpi_data.dist_base);
2180 	return err;
2181 }
2182 IRQCHIP_ACPI_DECLARE(gic_v3, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR,
2183 		     acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V3,
2184 		     gic_acpi_init);
2185 IRQCHIP_ACPI_DECLARE(gic_v4, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR,
2186 		     acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V4,
2187 		     gic_acpi_init);
2188 IRQCHIP_ACPI_DECLARE(gic_v3_or_v4, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR,
2189 		     acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_NONE,
2190 		     gic_acpi_init);
2191 #endif
2192