1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2024-2025 ARM Limited, All Rights Reserved.
4 */
5
6 #define pr_fmt(fmt) "GICv5: " fmt
7
8 #include <linux/acpi_iort.h>
9 #include <linux/cpuhotplug.h>
10 #include <linux/idr.h>
11 #include <linux/irqdomain.h>
12 #include <linux/slab.h>
13 #include <linux/wordpart.h>
14
15 #include <linux/irqchip.h>
16 #include <linux/irqchip/arm-gic-v5.h>
17 #include <linux/irqchip/arm-vgic-info.h>
18
19 #include <asm/cpufeature.h>
20 #include <asm/exception.h>
21
22 static u8 pri_bits __ro_after_init = 5;
23
24 #define GICV5_IRQ_PRI_MASK 0x1f
25 #define GICV5_IRQ_PRI_MI (GICV5_IRQ_PRI_MASK & GENMASK(4, 5 - pri_bits))
26
27 #define PPI_NR 128
28
gicv5_cpuif_has_gcie(void)29 static bool gicv5_cpuif_has_gcie(void)
30 {
31 return this_cpu_has_cap(ARM64_HAS_GICV5_CPUIF);
32 }
33
34 struct gicv5_chip_data gicv5_global_data __read_mostly;
35
36 static DEFINE_IDA(lpi_ida);
37 static u32 num_lpis __ro_after_init;
38
gicv5_init_lpis(u32 lpis)39 void __init gicv5_init_lpis(u32 lpis)
40 {
41 num_lpis = lpis;
42 }
43
gicv5_deinit_lpis(void)44 void __init gicv5_deinit_lpis(void)
45 {
46 num_lpis = 0;
47 }
48
alloc_lpi(void)49 static int alloc_lpi(void)
50 {
51 if (!num_lpis)
52 return -ENOSPC;
53
54 return ida_alloc_max(&lpi_ida, num_lpis - 1, GFP_KERNEL);
55 }
56
release_lpi(u32 lpi)57 static void release_lpi(u32 lpi)
58 {
59 ida_free(&lpi_ida, lpi);
60 }
61
gicv5_ppi_priority_init(void)62 static void gicv5_ppi_priority_init(void)
63 {
64 write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR0_EL1);
65 write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR1_EL1);
66 write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR2_EL1);
67 write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR3_EL1);
68 write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR4_EL1);
69 write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR5_EL1);
70 write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR6_EL1);
71 write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR7_EL1);
72 write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR8_EL1);
73 write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR9_EL1);
74 write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR10_EL1);
75 write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR11_EL1);
76 write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR12_EL1);
77 write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR13_EL1);
78 write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR14_EL1);
79 write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR15_EL1);
80
81 /*
82 * Context syncronization required to make sure system register writes
83 * effects are synchronised.
84 */
85 isb();
86 }
87
gicv5_hwirq_init(irq_hw_number_t hwirq,u8 priority,u8 hwirq_type)88 static void gicv5_hwirq_init(irq_hw_number_t hwirq, u8 priority, u8 hwirq_type)
89 {
90 u64 cdpri, cdaff;
91 u16 iaffid;
92 int ret;
93
94 if (hwirq_type == GICV5_HWIRQ_TYPE_LPI || hwirq_type == GICV5_HWIRQ_TYPE_SPI) {
95 cdpri = FIELD_PREP(GICV5_GIC_CDPRI_PRIORITY_MASK, priority) |
96 FIELD_PREP(GICV5_GIC_CDPRI_TYPE_MASK, hwirq_type) |
97 FIELD_PREP(GICV5_GIC_CDPRI_ID_MASK, hwirq);
98 gic_insn(cdpri, CDPRI);
99
100 ret = gicv5_irs_cpu_to_iaffid(smp_processor_id(), &iaffid);
101
102 if (WARN_ON_ONCE(ret))
103 return;
104
105 cdaff = FIELD_PREP(GICV5_GIC_CDAFF_IAFFID_MASK, iaffid) |
106 FIELD_PREP(GICV5_GIC_CDAFF_TYPE_MASK, hwirq_type) |
107 FIELD_PREP(GICV5_GIC_CDAFF_ID_MASK, hwirq);
108 gic_insn(cdaff, CDAFF);
109 }
110 }
111
gicv5_ppi_irq_mask(struct irq_data * d)112 static void gicv5_ppi_irq_mask(struct irq_data *d)
113 {
114 u64 hwirq_id_bit = BIT_ULL(d->hwirq % 64);
115
116 if (d->hwirq < 64)
117 sysreg_clear_set_s(SYS_ICC_PPI_ENABLER0_EL1, hwirq_id_bit, 0);
118 else
119 sysreg_clear_set_s(SYS_ICC_PPI_ENABLER1_EL1, hwirq_id_bit, 0);
120
121 /*
122 * We must ensure that the disable takes effect immediately to
123 * guarantee that the lazy-disabled IRQ mechanism works.
124 * A context synchronization event is required to guarantee it.
125 * Reference: I_ZLTKB/R_YRGMH GICv5 specification - section 2.9.1.
126 */
127 isb();
128 }
129
gicv5_iri_irq_mask(struct irq_data * d,u8 hwirq_type)130 static void gicv5_iri_irq_mask(struct irq_data *d, u8 hwirq_type)
131 {
132 u64 cddis;
133
134 cddis = FIELD_PREP(GICV5_GIC_CDDIS_ID_MASK, d->hwirq) |
135 FIELD_PREP(GICV5_GIC_CDDIS_TYPE_MASK, hwirq_type);
136
137 gic_insn(cddis, CDDIS);
138 /*
139 * We must make sure that GIC CDDIS write effects are propagated
140 * immediately to make sure the disable takes effect to guarantee
141 * that the lazy-disabled IRQ mechanism works.
142 * Rule R_XCLJC states that the effects of a GIC system instruction
143 * complete in finite time.
144 * The GSB ensures completion of the GIC instruction and prevents
145 * loads, stores and GIC instructions from executing part of their
146 * functionality before the GSB SYS.
147 */
148 gsb_sys();
149 }
150
gicv5_spi_irq_mask(struct irq_data * d)151 static void gicv5_spi_irq_mask(struct irq_data *d)
152 {
153 gicv5_iri_irq_mask(d, GICV5_HWIRQ_TYPE_SPI);
154 }
155
gicv5_lpi_irq_mask(struct irq_data * d)156 static void gicv5_lpi_irq_mask(struct irq_data *d)
157 {
158 gicv5_iri_irq_mask(d, GICV5_HWIRQ_TYPE_LPI);
159 }
160
gicv5_ppi_irq_unmask(struct irq_data * d)161 static void gicv5_ppi_irq_unmask(struct irq_data *d)
162 {
163 u64 hwirq_id_bit = BIT_ULL(d->hwirq % 64);
164
165 if (d->hwirq < 64)
166 sysreg_clear_set_s(SYS_ICC_PPI_ENABLER0_EL1, 0, hwirq_id_bit);
167 else
168 sysreg_clear_set_s(SYS_ICC_PPI_ENABLER1_EL1, 0, hwirq_id_bit);
169 /*
170 * We must ensure that the enable takes effect in finite time - a
171 * context synchronization event is required to guarantee it, we
172 * can not take for granted that would happen (eg a core going straight
173 * into idle after enabling a PPI).
174 * Reference: I_ZLTKB/R_YRGMH GICv5 specification - section 2.9.1.
175 */
176 isb();
177 }
178
gicv5_iri_irq_unmask(struct irq_data * d,u8 hwirq_type)179 static void gicv5_iri_irq_unmask(struct irq_data *d, u8 hwirq_type)
180 {
181 u64 cden;
182
183 cden = FIELD_PREP(GICV5_GIC_CDEN_ID_MASK, d->hwirq) |
184 FIELD_PREP(GICV5_GIC_CDEN_TYPE_MASK, hwirq_type);
185 /*
186 * Rule R_XCLJC states that the effects of a GIC system instruction
187 * complete in finite time and that's the only requirement when
188 * unmasking an SPI/LPI IRQ.
189 */
190 gic_insn(cden, CDEN);
191 }
192
gicv5_spi_irq_unmask(struct irq_data * d)193 static void gicv5_spi_irq_unmask(struct irq_data *d)
194 {
195 gicv5_iri_irq_unmask(d, GICV5_HWIRQ_TYPE_SPI);
196 }
197
gicv5_lpi_irq_unmask(struct irq_data * d)198 static void gicv5_lpi_irq_unmask(struct irq_data *d)
199 {
200 gicv5_iri_irq_unmask(d, GICV5_HWIRQ_TYPE_LPI);
201 }
202
gicv5_hwirq_eoi(u32 hwirq_id,u8 hwirq_type)203 static void gicv5_hwirq_eoi(u32 hwirq_id, u8 hwirq_type)
204 {
205 u64 cddi;
206
207 cddi = FIELD_PREP(GICV5_GIC_CDDI_ID_MASK, hwirq_id) |
208 FIELD_PREP(GICV5_GIC_CDDI_TYPE_MASK, hwirq_type);
209
210 gic_insn(cddi, CDDI);
211
212 gic_insn(0, CDEOI);
213 }
214
gicv5_ppi_irq_eoi(struct irq_data * d)215 static void gicv5_ppi_irq_eoi(struct irq_data *d)
216 {
217 /* Skip deactivate for forwarded PPI interrupts */
218 if (irqd_is_forwarded_to_vcpu(d)) {
219 gic_insn(0, CDEOI);
220 return;
221 }
222
223 gicv5_hwirq_eoi(d->hwirq, GICV5_HWIRQ_TYPE_PPI);
224 }
225
gicv5_spi_irq_eoi(struct irq_data * d)226 static void gicv5_spi_irq_eoi(struct irq_data *d)
227 {
228 gicv5_hwirq_eoi(d->hwirq, GICV5_HWIRQ_TYPE_SPI);
229 }
230
gicv5_lpi_irq_eoi(struct irq_data * d)231 static void gicv5_lpi_irq_eoi(struct irq_data *d)
232 {
233 gicv5_hwirq_eoi(d->hwirq, GICV5_HWIRQ_TYPE_LPI);
234 }
235
gicv5_iri_irq_set_affinity(struct irq_data * d,const struct cpumask * mask_val,bool force,u8 hwirq_type)236 static int gicv5_iri_irq_set_affinity(struct irq_data *d,
237 const struct cpumask *mask_val,
238 bool force, u8 hwirq_type)
239 {
240 int ret, cpuid;
241 u16 iaffid;
242 u64 cdaff;
243
244 if (force)
245 cpuid = cpumask_first(mask_val);
246 else
247 cpuid = cpumask_any_and(mask_val, cpu_online_mask);
248
249 ret = gicv5_irs_cpu_to_iaffid(cpuid, &iaffid);
250 if (ret)
251 return ret;
252
253 cdaff = FIELD_PREP(GICV5_GIC_CDAFF_IAFFID_MASK, iaffid) |
254 FIELD_PREP(GICV5_GIC_CDAFF_TYPE_MASK, hwirq_type) |
255 FIELD_PREP(GICV5_GIC_CDAFF_ID_MASK, d->hwirq);
256 gic_insn(cdaff, CDAFF);
257
258 irq_data_update_effective_affinity(d, cpumask_of(cpuid));
259
260 return IRQ_SET_MASK_OK_DONE;
261 }
262
gicv5_spi_irq_set_affinity(struct irq_data * d,const struct cpumask * mask_val,bool force)263 static int gicv5_spi_irq_set_affinity(struct irq_data *d,
264 const struct cpumask *mask_val,
265 bool force)
266 {
267 return gicv5_iri_irq_set_affinity(d, mask_val, force,
268 GICV5_HWIRQ_TYPE_SPI);
269 }
270
gicv5_lpi_irq_set_affinity(struct irq_data * d,const struct cpumask * mask_val,bool force)271 static int gicv5_lpi_irq_set_affinity(struct irq_data *d,
272 const struct cpumask *mask_val,
273 bool force)
274 {
275 return gicv5_iri_irq_set_affinity(d, mask_val, force,
276 GICV5_HWIRQ_TYPE_LPI);
277 }
278
279 enum ppi_reg {
280 PPI_PENDING,
281 PPI_ACTIVE,
282 PPI_HM
283 };
284
read_ppi_sysreg_s(unsigned int irq,const enum ppi_reg which)285 static __always_inline u64 read_ppi_sysreg_s(unsigned int irq,
286 const enum ppi_reg which)
287 {
288 switch (which) {
289 case PPI_PENDING:
290 return irq < 64 ? read_sysreg_s(SYS_ICC_PPI_SPENDR0_EL1) :
291 read_sysreg_s(SYS_ICC_PPI_SPENDR1_EL1);
292 case PPI_ACTIVE:
293 return irq < 64 ? read_sysreg_s(SYS_ICC_PPI_SACTIVER0_EL1) :
294 read_sysreg_s(SYS_ICC_PPI_SACTIVER1_EL1);
295 case PPI_HM:
296 return irq < 64 ? read_sysreg_s(SYS_ICC_PPI_HMR0_EL1) :
297 read_sysreg_s(SYS_ICC_PPI_HMR1_EL1);
298 default:
299 BUILD_BUG_ON(1);
300 }
301 }
302
write_ppi_sysreg_s(unsigned int irq,bool set,const enum ppi_reg which)303 static __always_inline void write_ppi_sysreg_s(unsigned int irq, bool set,
304 const enum ppi_reg which)
305 {
306 u64 bit = BIT_ULL(irq % 64);
307
308 switch (which) {
309 case PPI_PENDING:
310 if (set) {
311 if (irq < 64)
312 write_sysreg_s(bit, SYS_ICC_PPI_SPENDR0_EL1);
313 else
314 write_sysreg_s(bit, SYS_ICC_PPI_SPENDR1_EL1);
315 } else {
316 if (irq < 64)
317 write_sysreg_s(bit, SYS_ICC_PPI_CPENDR0_EL1);
318 else
319 write_sysreg_s(bit, SYS_ICC_PPI_CPENDR1_EL1);
320 }
321 return;
322 case PPI_ACTIVE:
323 if (set) {
324 if (irq < 64)
325 write_sysreg_s(bit, SYS_ICC_PPI_SACTIVER0_EL1);
326 else
327 write_sysreg_s(bit, SYS_ICC_PPI_SACTIVER1_EL1);
328 } else {
329 if (irq < 64)
330 write_sysreg_s(bit, SYS_ICC_PPI_CACTIVER0_EL1);
331 else
332 write_sysreg_s(bit, SYS_ICC_PPI_CACTIVER1_EL1);
333 }
334 return;
335 default:
336 BUILD_BUG_ON(1);
337 }
338 }
339
gicv5_ppi_irq_get_irqchip_state(struct irq_data * d,enum irqchip_irq_state which,bool * state)340 static int gicv5_ppi_irq_get_irqchip_state(struct irq_data *d,
341 enum irqchip_irq_state which,
342 bool *state)
343 {
344 u64 hwirq_id_bit = BIT_ULL(d->hwirq % 64);
345
346 switch (which) {
347 case IRQCHIP_STATE_PENDING:
348 *state = !!(read_ppi_sysreg_s(d->hwirq, PPI_PENDING) & hwirq_id_bit);
349 return 0;
350 case IRQCHIP_STATE_ACTIVE:
351 *state = !!(read_ppi_sysreg_s(d->hwirq, PPI_ACTIVE) & hwirq_id_bit);
352 return 0;
353 default:
354 pr_debug("Unexpected PPI irqchip state\n");
355 return -EINVAL;
356 }
357 }
358
gicv5_iri_irq_get_irqchip_state(struct irq_data * d,enum irqchip_irq_state which,bool * state,u8 hwirq_type)359 static int gicv5_iri_irq_get_irqchip_state(struct irq_data *d,
360 enum irqchip_irq_state which,
361 bool *state, u8 hwirq_type)
362 {
363 u64 icsr, cdrcfg;
364
365 cdrcfg = d->hwirq | FIELD_PREP(GICV5_GIC_CDRCFG_TYPE_MASK, hwirq_type);
366
367 gic_insn(cdrcfg, CDRCFG);
368 isb();
369 icsr = read_sysreg_s(SYS_ICC_ICSR_EL1);
370
371 if (FIELD_GET(ICC_ICSR_EL1_F, icsr)) {
372 pr_err("ICSR_EL1 is invalid\n");
373 return -EINVAL;
374 }
375
376 switch (which) {
377 case IRQCHIP_STATE_PENDING:
378 *state = !!(FIELD_GET(ICC_ICSR_EL1_Pending, icsr));
379 return 0;
380
381 case IRQCHIP_STATE_ACTIVE:
382 *state = !!(FIELD_GET(ICC_ICSR_EL1_Active, icsr));
383 return 0;
384
385 default:
386 pr_debug("Unexpected irqchip_irq_state\n");
387 return -EINVAL;
388 }
389 }
390
gicv5_spi_irq_get_irqchip_state(struct irq_data * d,enum irqchip_irq_state which,bool * state)391 static int gicv5_spi_irq_get_irqchip_state(struct irq_data *d,
392 enum irqchip_irq_state which,
393 bool *state)
394 {
395 return gicv5_iri_irq_get_irqchip_state(d, which, state,
396 GICV5_HWIRQ_TYPE_SPI);
397 }
398
gicv5_lpi_irq_get_irqchip_state(struct irq_data * d,enum irqchip_irq_state which,bool * state)399 static int gicv5_lpi_irq_get_irqchip_state(struct irq_data *d,
400 enum irqchip_irq_state which,
401 bool *state)
402 {
403 return gicv5_iri_irq_get_irqchip_state(d, which, state,
404 GICV5_HWIRQ_TYPE_LPI);
405 }
406
gicv5_ppi_irq_set_irqchip_state(struct irq_data * d,enum irqchip_irq_state which,bool state)407 static int gicv5_ppi_irq_set_irqchip_state(struct irq_data *d,
408 enum irqchip_irq_state which,
409 bool state)
410 {
411 switch (which) {
412 case IRQCHIP_STATE_PENDING:
413 write_ppi_sysreg_s(d->hwirq, state, PPI_PENDING);
414 return 0;
415 case IRQCHIP_STATE_ACTIVE:
416 write_ppi_sysreg_s(d->hwirq, state, PPI_ACTIVE);
417 return 0;
418 default:
419 pr_debug("Unexpected PPI irqchip state\n");
420 return -EINVAL;
421 }
422 }
423
gicv5_iri_irq_write_pending_state(struct irq_data * d,bool state,u8 hwirq_type)424 static void gicv5_iri_irq_write_pending_state(struct irq_data *d, bool state,
425 u8 hwirq_type)
426 {
427 u64 cdpend;
428
429 cdpend = FIELD_PREP(GICV5_GIC_CDPEND_TYPE_MASK, hwirq_type) |
430 FIELD_PREP(GICV5_GIC_CDPEND_ID_MASK, d->hwirq) |
431 FIELD_PREP(GICV5_GIC_CDPEND_PENDING_MASK, state);
432
433 gic_insn(cdpend, CDPEND);
434 }
435
gicv5_spi_irq_write_pending_state(struct irq_data * d,bool state)436 static void gicv5_spi_irq_write_pending_state(struct irq_data *d, bool state)
437 {
438 gicv5_iri_irq_write_pending_state(d, state, GICV5_HWIRQ_TYPE_SPI);
439 }
440
gicv5_lpi_irq_write_pending_state(struct irq_data * d,bool state)441 static void gicv5_lpi_irq_write_pending_state(struct irq_data *d, bool state)
442 {
443 gicv5_iri_irq_write_pending_state(d, state, GICV5_HWIRQ_TYPE_LPI);
444 }
445
gicv5_spi_irq_set_irqchip_state(struct irq_data * d,enum irqchip_irq_state which,bool state)446 static int gicv5_spi_irq_set_irqchip_state(struct irq_data *d,
447 enum irqchip_irq_state which,
448 bool state)
449 {
450 switch (which) {
451 case IRQCHIP_STATE_PENDING:
452 gicv5_spi_irq_write_pending_state(d, state);
453 break;
454 default:
455 pr_debug("Unexpected irqchip_irq_state\n");
456 return -EINVAL;
457 }
458
459 return 0;
460 }
461
gicv5_lpi_irq_set_irqchip_state(struct irq_data * d,enum irqchip_irq_state which,bool state)462 static int gicv5_lpi_irq_set_irqchip_state(struct irq_data *d,
463 enum irqchip_irq_state which,
464 bool state)
465 {
466 switch (which) {
467 case IRQCHIP_STATE_PENDING:
468 gicv5_lpi_irq_write_pending_state(d, state);
469 break;
470
471 default:
472 pr_debug("Unexpected irqchip_irq_state\n");
473 return -EINVAL;
474 }
475
476 return 0;
477 }
478
gicv5_spi_irq_retrigger(struct irq_data * data)479 static int gicv5_spi_irq_retrigger(struct irq_data *data)
480 {
481 return !gicv5_spi_irq_set_irqchip_state(data, IRQCHIP_STATE_PENDING,
482 true);
483 }
484
gicv5_lpi_irq_retrigger(struct irq_data * data)485 static int gicv5_lpi_irq_retrigger(struct irq_data *data)
486 {
487 return !gicv5_lpi_irq_set_irqchip_state(data, IRQCHIP_STATE_PENDING,
488 true);
489 }
490
gicv5_ipi_send_single(struct irq_data * d,unsigned int cpu)491 static void gicv5_ipi_send_single(struct irq_data *d, unsigned int cpu)
492 {
493 /* Mark the LPI pending */
494 irq_chip_retrigger_hierarchy(d);
495 }
496
gicv5_ppi_irq_is_level(irq_hw_number_t hwirq)497 static bool gicv5_ppi_irq_is_level(irq_hw_number_t hwirq)
498 {
499 u64 bit = BIT_ULL(hwirq % 64);
500
501 return !!(read_ppi_sysreg_s(hwirq, PPI_HM) & bit);
502 }
503
gicv5_ppi_irq_set_type(struct irq_data * d,unsigned int type)504 static int gicv5_ppi_irq_set_type(struct irq_data *d, unsigned int type)
505 {
506 /*
507 * GICv5's PPIs do not have a configurable trigger or handling
508 * mode. Check that the attempt to set a type matches what the
509 * hardware reports in the HMR, and error on a mismatch.
510 */
511
512 if (type & IRQ_TYPE_EDGE_BOTH && gicv5_ppi_irq_is_level(d->hwirq))
513 return -EINVAL;
514
515 if (type & IRQ_TYPE_LEVEL_MASK && !gicv5_ppi_irq_is_level(d->hwirq))
516 return -EINVAL;
517
518 return 0;
519 }
520
gicv5_ppi_irq_set_vcpu_affinity(struct irq_data * d,void * vcpu)521 static int gicv5_ppi_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu)
522 {
523 if (vcpu)
524 irqd_set_forwarded_to_vcpu(d);
525 else
526 irqd_clr_forwarded_to_vcpu(d);
527
528 return 0;
529 }
530
531 static const struct irq_chip gicv5_ppi_irq_chip = {
532 .name = "GICv5-PPI",
533 .irq_mask = gicv5_ppi_irq_mask,
534 .irq_unmask = gicv5_ppi_irq_unmask,
535 .irq_eoi = gicv5_ppi_irq_eoi,
536 .irq_set_type = gicv5_ppi_irq_set_type,
537 .irq_get_irqchip_state = gicv5_ppi_irq_get_irqchip_state,
538 .irq_set_irqchip_state = gicv5_ppi_irq_set_irqchip_state,
539 .irq_set_vcpu_affinity = gicv5_ppi_irq_set_vcpu_affinity,
540 .flags = IRQCHIP_SKIP_SET_WAKE |
541 IRQCHIP_MASK_ON_SUSPEND,
542 };
543
544 static const struct irq_chip gicv5_spi_irq_chip = {
545 .name = "GICv5-SPI",
546 .irq_mask = gicv5_spi_irq_mask,
547 .irq_unmask = gicv5_spi_irq_unmask,
548 .irq_eoi = gicv5_spi_irq_eoi,
549 .irq_set_type = gicv5_spi_irq_set_type,
550 .irq_set_affinity = gicv5_spi_irq_set_affinity,
551 .irq_retrigger = gicv5_spi_irq_retrigger,
552 .irq_get_irqchip_state = gicv5_spi_irq_get_irqchip_state,
553 .irq_set_irqchip_state = gicv5_spi_irq_set_irqchip_state,
554 .flags = IRQCHIP_SET_TYPE_MASKED |
555 IRQCHIP_SKIP_SET_WAKE |
556 IRQCHIP_MASK_ON_SUSPEND,
557 };
558
559 static const struct irq_chip gicv5_lpi_irq_chip = {
560 .name = "GICv5-LPI",
561 .irq_mask = gicv5_lpi_irq_mask,
562 .irq_unmask = gicv5_lpi_irq_unmask,
563 .irq_eoi = gicv5_lpi_irq_eoi,
564 .irq_set_affinity = gicv5_lpi_irq_set_affinity,
565 .irq_retrigger = gicv5_lpi_irq_retrigger,
566 .irq_get_irqchip_state = gicv5_lpi_irq_get_irqchip_state,
567 .irq_set_irqchip_state = gicv5_lpi_irq_set_irqchip_state,
568 .flags = IRQCHIP_SKIP_SET_WAKE |
569 IRQCHIP_MASK_ON_SUSPEND,
570 };
571
572 static const struct irq_chip gicv5_ipi_irq_chip = {
573 .name = "GICv5-IPI",
574 .irq_mask = irq_chip_mask_parent,
575 .irq_unmask = irq_chip_unmask_parent,
576 .irq_eoi = irq_chip_eoi_parent,
577 .irq_set_affinity = irq_chip_set_affinity_parent,
578 .irq_get_irqchip_state = irq_chip_get_parent_state,
579 .irq_set_irqchip_state = irq_chip_set_parent_state,
580 .ipi_send_single = gicv5_ipi_send_single,
581 .flags = IRQCHIP_SKIP_SET_WAKE |
582 IRQCHIP_MASK_ON_SUSPEND,
583 };
584
gicv5_irq_domain_translate(struct irq_domain * d,struct irq_fwspec * fwspec,irq_hw_number_t * hwirq,unsigned int * type,const u8 hwirq_type)585 static __always_inline int gicv5_irq_domain_translate(struct irq_domain *d,
586 struct irq_fwspec *fwspec,
587 irq_hw_number_t *hwirq,
588 unsigned int *type,
589 const u8 hwirq_type)
590 {
591 unsigned int hwirq_trigger;
592 u8 fwspec_irq_type;
593
594 if (is_of_node(fwspec->fwnode)) {
595
596 if (fwspec->param_count < 3)
597 return -EINVAL;
598
599 fwspec_irq_type = fwspec->param[0];
600
601 if (fwspec->param[0] != hwirq_type)
602 return -EINVAL;
603
604 *hwirq = fwspec->param[1];
605 hwirq_trigger = fwspec->param[2];
606 }
607
608 if (is_fwnode_irqchip(fwspec->fwnode)) {
609
610 if (fwspec->param_count != 2)
611 return -EINVAL;
612
613 fwspec_irq_type = FIELD_GET(GICV5_HWIRQ_TYPE, fwspec->param[0]);
614
615 if (fwspec_irq_type != hwirq_type)
616 return -EINVAL;
617
618 *hwirq = FIELD_GET(GICV5_HWIRQ_ID, fwspec->param[0]);
619 hwirq_trigger = fwspec->param[1];
620 }
621
622 switch (hwirq_type) {
623 case GICV5_HWIRQ_TYPE_PPI:
624 /*
625 * Handling mode is hardcoded for PPIs, set the type using
626 * HW reported value.
627 */
628 *type = gicv5_ppi_irq_is_level(*hwirq) ? IRQ_TYPE_LEVEL_LOW :
629 IRQ_TYPE_EDGE_RISING;
630 break;
631 case GICV5_HWIRQ_TYPE_SPI:
632 *type = hwirq_trigger & IRQ_TYPE_SENSE_MASK;
633 break;
634 default:
635 BUILD_BUG_ON(1);
636 }
637
638 return 0;
639 }
640
gicv5_irq_ppi_domain_translate(struct irq_domain * d,struct irq_fwspec * fwspec,irq_hw_number_t * hwirq,unsigned int * type)641 static int gicv5_irq_ppi_domain_translate(struct irq_domain *d,
642 struct irq_fwspec *fwspec,
643 irq_hw_number_t *hwirq,
644 unsigned int *type)
645 {
646 return gicv5_irq_domain_translate(d, fwspec, hwirq, type,
647 GICV5_HWIRQ_TYPE_PPI);
648 }
649
gicv5_irq_ppi_domain_alloc(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs,void * arg)650 static int gicv5_irq_ppi_domain_alloc(struct irq_domain *domain, unsigned int virq,
651 unsigned int nr_irqs, void *arg)
652 {
653 unsigned int type = IRQ_TYPE_NONE;
654 struct irq_fwspec *fwspec = arg;
655 irq_hw_number_t hwirq;
656 int ret;
657
658 if (WARN_ON_ONCE(nr_irqs != 1))
659 return -EINVAL;
660
661 ret = gicv5_irq_ppi_domain_translate(domain, fwspec, &hwirq, &type);
662 if (ret)
663 return ret;
664
665 if (type & IRQ_TYPE_LEVEL_MASK)
666 irq_set_status_flags(virq, IRQ_LEVEL);
667
668 irq_set_percpu_devid(virq);
669 irq_domain_set_info(domain, virq, hwirq, &gicv5_ppi_irq_chip, NULL,
670 handle_percpu_devid_irq, NULL, NULL);
671
672 return 0;
673 }
674
gicv5_irq_domain_free(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs)675 static void gicv5_irq_domain_free(struct irq_domain *domain, unsigned int virq,
676 unsigned int nr_irqs)
677 {
678 struct irq_data *d;
679
680 if (WARN_ON_ONCE(nr_irqs != 1))
681 return;
682
683 d = irq_domain_get_irq_data(domain, virq);
684
685 irq_set_handler(virq, NULL);
686 irq_domain_reset_irq_data(d);
687 }
688
gicv5_irq_ppi_domain_select(struct irq_domain * d,struct irq_fwspec * fwspec,enum irq_domain_bus_token bus_token)689 static int gicv5_irq_ppi_domain_select(struct irq_domain *d, struct irq_fwspec *fwspec,
690 enum irq_domain_bus_token bus_token)
691 {
692 u32 hwirq_type;
693
694 if (fwspec->fwnode != d->fwnode)
695 return 0;
696
697 if (is_of_node(fwspec->fwnode))
698 hwirq_type = fwspec->param[0];
699
700 if (is_fwnode_irqchip(fwspec->fwnode))
701 hwirq_type = FIELD_GET(GICV5_HWIRQ_TYPE, fwspec->param[0]);
702
703 if (hwirq_type != GICV5_HWIRQ_TYPE_PPI)
704 return 0;
705
706 return (d == gicv5_global_data.ppi_domain);
707 }
708
709 static const struct irq_domain_ops gicv5_irq_ppi_domain_ops = {
710 .translate = gicv5_irq_ppi_domain_translate,
711 .alloc = gicv5_irq_ppi_domain_alloc,
712 .free = gicv5_irq_domain_free,
713 .select = gicv5_irq_ppi_domain_select
714 };
715
gicv5_irq_spi_domain_translate(struct irq_domain * d,struct irq_fwspec * fwspec,irq_hw_number_t * hwirq,unsigned int * type)716 static int gicv5_irq_spi_domain_translate(struct irq_domain *d,
717 struct irq_fwspec *fwspec,
718 irq_hw_number_t *hwirq,
719 unsigned int *type)
720 {
721 return gicv5_irq_domain_translate(d, fwspec, hwirq, type,
722 GICV5_HWIRQ_TYPE_SPI);
723 }
724
gicv5_irq_spi_domain_alloc(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs,void * arg)725 static int gicv5_irq_spi_domain_alloc(struct irq_domain *domain, unsigned int virq,
726 unsigned int nr_irqs, void *arg)
727 {
728 struct gicv5_irs_chip_data *chip_data;
729 unsigned int type = IRQ_TYPE_NONE;
730 struct irq_fwspec *fwspec = arg;
731 struct irq_data *irqd;
732 irq_hw_number_t hwirq;
733 int ret;
734
735 if (WARN_ON_ONCE(nr_irqs != 1))
736 return -EINVAL;
737
738 ret = gicv5_irq_spi_domain_translate(domain, fwspec, &hwirq, &type);
739 if (ret)
740 return ret;
741
742 irqd = irq_desc_get_irq_data(irq_to_desc(virq));
743 chip_data = gicv5_irs_lookup_by_spi_id(hwirq);
744
745 irq_domain_set_info(domain, virq, hwirq, &gicv5_spi_irq_chip, chip_data,
746 handle_fasteoi_irq, NULL, NULL);
747 irq_set_probe(virq);
748 irqd_set_single_target(irqd);
749
750 gicv5_hwirq_init(hwirq, GICV5_IRQ_PRI_MI, GICV5_HWIRQ_TYPE_SPI);
751
752 return 0;
753 }
754
gicv5_irq_spi_domain_select(struct irq_domain * d,struct irq_fwspec * fwspec,enum irq_domain_bus_token bus_token)755 static int gicv5_irq_spi_domain_select(struct irq_domain *d, struct irq_fwspec *fwspec,
756 enum irq_domain_bus_token bus_token)
757 {
758 u32 hwirq_type;
759
760 if (fwspec->fwnode != d->fwnode)
761 return 0;
762
763 if (is_of_node(fwspec->fwnode))
764 hwirq_type = fwspec->param[0];
765
766 if (is_fwnode_irqchip(fwspec->fwnode))
767 hwirq_type = FIELD_GET(GICV5_HWIRQ_TYPE, fwspec->param[0]);
768
769 if (hwirq_type != GICV5_HWIRQ_TYPE_SPI)
770 return 0;
771
772 return (d == gicv5_global_data.spi_domain);
773 }
774
775 static const struct irq_domain_ops gicv5_irq_spi_domain_ops = {
776 .translate = gicv5_irq_spi_domain_translate,
777 .alloc = gicv5_irq_spi_domain_alloc,
778 .free = gicv5_irq_domain_free,
779 .select = gicv5_irq_spi_domain_select
780 };
781
gicv5_lpi_config_reset(struct irq_data * d)782 static void gicv5_lpi_config_reset(struct irq_data *d)
783 {
784 u64 cdhm;
785
786 /*
787 * Reset LPIs handling mode to edge by default and clear pending
788 * state to make sure we start the LPI with a clean state from
789 * previous incarnations.
790 */
791 cdhm = FIELD_PREP(GICV5_GIC_CDHM_HM_MASK, 0) |
792 FIELD_PREP(GICV5_GIC_CDHM_TYPE_MASK, GICV5_HWIRQ_TYPE_LPI) |
793 FIELD_PREP(GICV5_GIC_CDHM_ID_MASK, d->hwirq);
794 gic_insn(cdhm, CDHM);
795
796 gicv5_lpi_irq_write_pending_state(d, false);
797 }
798
gicv5_irq_lpi_domain_free(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs)799 static void gicv5_irq_lpi_domain_free(struct irq_domain *domain, unsigned int virq,
800 unsigned int nr_irqs)
801 {
802 struct irq_data *d;
803
804 for (unsigned int i = 0; i < nr_irqs; i++, virq++) {
805 d = irq_domain_get_irq_data(domain, virq);
806
807 release_lpi(d->hwirq);
808
809 irq_set_handler(virq, NULL);
810 irq_domain_reset_irq_data(d);
811 }
812 }
813
gicv5_irq_lpi_domain_alloc(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs,void * arg)814 static int gicv5_irq_lpi_domain_alloc(struct irq_domain *domain, unsigned int virq,
815 unsigned int nr_irqs, void *arg)
816 {
817 irq_hw_number_t hwirq;
818 struct irq_data *irqd;
819 unsigned int i;
820 int ret;
821
822 for (i = 0; i < nr_irqs; i++) {
823 ret = alloc_lpi();
824 if (ret < 0)
825 goto out_free_lpis;
826 hwirq = ret;
827
828 ret = gicv5_irs_iste_alloc(hwirq);
829 if (ret < 0) {
830 /* Undo partial state first, then clean up the rest */
831 release_lpi(hwirq);
832 goto out_free_lpis;
833 }
834
835 irqd = irq_domain_get_irq_data(domain, virq + i);
836
837 irq_domain_set_info(domain, virq + i, hwirq, &gicv5_lpi_irq_chip,
838 NULL, handle_fasteoi_irq, NULL, NULL);
839 irqd_set_single_target(irqd);
840
841 gicv5_hwirq_init(hwirq, GICV5_IRQ_PRI_MI, GICV5_HWIRQ_TYPE_LPI);
842 gicv5_lpi_config_reset(irqd);
843 }
844
845 return 0;
846
847 out_free_lpis:
848 if (i)
849 gicv5_irq_lpi_domain_free(domain, virq, i);
850
851 return ret;
852 }
853
854 static const struct irq_domain_ops gicv5_irq_lpi_domain_ops = {
855 .alloc = gicv5_irq_lpi_domain_alloc,
856 .free = gicv5_irq_lpi_domain_free,
857 };
858
gicv5_init_lpi_domain(void)859 void __init gicv5_init_lpi_domain(void)
860 {
861 struct irq_domain *d;
862
863 d = irq_domain_create_tree(NULL, &gicv5_irq_lpi_domain_ops, NULL);
864 gicv5_global_data.lpi_domain = d;
865 }
866
gicv5_free_lpi_domain(void)867 void __init gicv5_free_lpi_domain(void)
868 {
869 irq_domain_remove(gicv5_global_data.lpi_domain);
870 gicv5_global_data.lpi_domain = NULL;
871 }
872
gicv5_irq_ipi_domain_alloc(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs,void * arg)873 static int gicv5_irq_ipi_domain_alloc(struct irq_domain *domain, unsigned int virq,
874 unsigned int nr_irqs, void *arg)
875 {
876 struct irq_data *irqd;
877 int ret;
878
879 ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg);
880 if (ret)
881 return ret;
882
883 for (unsigned int i = 0; i < nr_irqs; i++, virq++) {
884 irqd = irq_domain_get_irq_data(domain, virq);
885
886 irq_domain_set_hwirq_and_chip(domain, virq, i,
887 &gicv5_ipi_irq_chip, NULL);
888
889 irqd_set_single_target(irqd);
890
891 irq_set_handler(virq, handle_percpu_irq);
892 }
893
894 return 0;
895 }
896
gicv5_irq_ipi_domain_free(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs)897 static void gicv5_irq_ipi_domain_free(struct irq_domain *domain, unsigned int virq,
898 unsigned int nr_irqs)
899 {
900 struct irq_data *d;
901 unsigned int i;
902
903 for (i = 0; i < nr_irqs; i++) {
904 d = irq_domain_get_irq_data(domain, virq + i);
905
906 if (!d)
907 return;
908
909 irq_set_handler(virq + i, NULL);
910 irq_domain_reset_irq_data(d);
911 }
912
913 irq_domain_free_irqs_parent(domain, virq, nr_irqs);
914 }
915
916 static const struct irq_domain_ops gicv5_irq_ipi_domain_ops = {
917 .alloc = gicv5_irq_ipi_domain_alloc,
918 .free = gicv5_irq_ipi_domain_free,
919 };
920
handle_irq_per_domain(u32 hwirq)921 static void handle_irq_per_domain(u32 hwirq)
922 {
923 u8 hwirq_type = FIELD_GET(GICV5_HWIRQ_TYPE, hwirq);
924 u32 hwirq_id = FIELD_GET(GICV5_HWIRQ_ID, hwirq);
925 struct irq_domain *domain;
926
927 switch (hwirq_type) {
928 case GICV5_HWIRQ_TYPE_PPI:
929 domain = gicv5_global_data.ppi_domain;
930 break;
931 case GICV5_HWIRQ_TYPE_SPI:
932 domain = gicv5_global_data.spi_domain;
933 break;
934 case GICV5_HWIRQ_TYPE_LPI:
935 domain = gicv5_global_data.lpi_domain;
936 break;
937 default:
938 pr_err_once("Unknown IRQ type, bail out\n");
939 return;
940 }
941
942 if (generic_handle_domain_irq(domain, hwirq_id)) {
943 pr_err_once("Could not handle, hwirq = 0x%x", hwirq_id);
944 gicv5_hwirq_eoi(hwirq_id, hwirq_type);
945 }
946 }
947
gicv5_handle_irq(struct pt_regs * regs)948 static void __exception_irq_entry gicv5_handle_irq(struct pt_regs *regs)
949 {
950 bool valid;
951 u32 hwirq;
952 u64 ia;
953
954 ia = gicr_insn(CDIA);
955 valid = GICV5_GICR_CDIA_VALID(ia);
956
957 if (!valid)
958 return;
959
960 /*
961 * Ensure that the CDIA instruction effects (ie IRQ activation) are
962 * completed before handling the interrupt.
963 */
964 gsb_ack();
965
966 /*
967 * Ensure instruction ordering between an acknowledgment and subsequent
968 * instructions in the IRQ handler using an ISB.
969 */
970 isb();
971
972 hwirq = FIELD_GET(GICV5_HWIRQ_INTID, ia);
973
974 handle_irq_per_domain(hwirq);
975 }
976
gicv5_cpu_disable_interrupts(void)977 static void gicv5_cpu_disable_interrupts(void)
978 {
979 u64 cr0;
980
981 cr0 = FIELD_PREP(ICC_CR0_EL1_EN, 0);
982 write_sysreg_s(cr0, SYS_ICC_CR0_EL1);
983 }
984
gicv5_cpu_enable_interrupts(void)985 static void gicv5_cpu_enable_interrupts(void)
986 {
987 u64 cr0, pcr;
988
989 write_sysreg_s(0, SYS_ICC_PPI_ENABLER0_EL1);
990 write_sysreg_s(0, SYS_ICC_PPI_ENABLER1_EL1);
991
992 gicv5_ppi_priority_init();
993
994 pcr = FIELD_PREP(ICC_PCR_EL1_PRIORITY, GICV5_IRQ_PRI_MI);
995 write_sysreg_s(pcr, SYS_ICC_PCR_EL1);
996
997 cr0 = FIELD_PREP(ICC_CR0_EL1_EN, 1);
998 write_sysreg_s(cr0, SYS_ICC_CR0_EL1);
999 }
1000
1001 static int base_ipi_virq;
1002
gicv5_starting_cpu(unsigned int cpu)1003 static int gicv5_starting_cpu(unsigned int cpu)
1004 {
1005 if (WARN(!gicv5_cpuif_has_gcie(),
1006 "GICv5 system components present but CPU does not have FEAT_GCIE"))
1007 return -ENODEV;
1008
1009 gicv5_cpu_enable_interrupts();
1010
1011 return gicv5_irs_register_cpu(cpu);
1012 }
1013
gicv5_smp_init(void)1014 static void __init gicv5_smp_init(void)
1015 {
1016 unsigned int num_ipis = GICV5_IPIS_PER_CPU * nr_cpu_ids;
1017
1018 cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_GIC_STARTING,
1019 "irqchip/arm/gicv5:starting",
1020 gicv5_starting_cpu, NULL);
1021
1022 base_ipi_virq = irq_domain_alloc_irqs(gicv5_global_data.ipi_domain,
1023 num_ipis, NUMA_NO_NODE, NULL);
1024 if (WARN(base_ipi_virq <= 0, "IPI IRQ allocation was not successful"))
1025 return;
1026
1027 set_smp_ipi_range_percpu(base_ipi_virq, GICV5_IPIS_PER_CPU, nr_cpu_ids);
1028 }
1029
gicv5_free_domains(void)1030 static void __init gicv5_free_domains(void)
1031 {
1032 if (gicv5_global_data.ppi_domain)
1033 irq_domain_remove(gicv5_global_data.ppi_domain);
1034 if (gicv5_global_data.spi_domain)
1035 irq_domain_remove(gicv5_global_data.spi_domain);
1036 if (gicv5_global_data.ipi_domain)
1037 irq_domain_remove(gicv5_global_data.ipi_domain);
1038
1039 gicv5_global_data.ppi_domain = NULL;
1040 gicv5_global_data.spi_domain = NULL;
1041 gicv5_global_data.ipi_domain = NULL;
1042 }
1043
gicv5_init_domains(struct fwnode_handle * handle)1044 static int __init gicv5_init_domains(struct fwnode_handle *handle)
1045 {
1046 u32 spi_count = gicv5_global_data.global_spi_count;
1047 struct irq_domain *d;
1048
1049 d = irq_domain_create_linear(handle, PPI_NR, &gicv5_irq_ppi_domain_ops, NULL);
1050 if (!d)
1051 return -ENOMEM;
1052
1053 irq_domain_update_bus_token(d, DOMAIN_BUS_WIRED);
1054 gicv5_global_data.ppi_domain = d;
1055
1056 if (spi_count) {
1057 d = irq_domain_create_linear(handle, spi_count,
1058 &gicv5_irq_spi_domain_ops, NULL);
1059
1060 if (!d) {
1061 gicv5_free_domains();
1062 return -ENOMEM;
1063 }
1064
1065 gicv5_global_data.spi_domain = d;
1066 irq_domain_update_bus_token(d, DOMAIN_BUS_WIRED);
1067 }
1068
1069 if (!WARN(!gicv5_global_data.lpi_domain,
1070 "LPI domain uninitialized, can't set up IPIs")) {
1071 d = irq_domain_create_hierarchy(gicv5_global_data.lpi_domain,
1072 0, GICV5_IPIS_PER_CPU * nr_cpu_ids,
1073 NULL, &gicv5_irq_ipi_domain_ops,
1074 NULL);
1075
1076 if (!d) {
1077 gicv5_free_domains();
1078 return -ENOMEM;
1079 }
1080 gicv5_global_data.ipi_domain = d;
1081 }
1082 gicv5_global_data.fwnode = handle;
1083
1084 return 0;
1085 }
1086
gicv5_set_cpuif_pribits(void)1087 static void gicv5_set_cpuif_pribits(void)
1088 {
1089 u64 icc_idr0 = read_sysreg_s(SYS_ICC_IDR0_EL1);
1090
1091 switch (FIELD_GET(ICC_IDR0_EL1_PRI_BITS, icc_idr0)) {
1092 case ICC_IDR0_EL1_PRI_BITS_4BITS:
1093 gicv5_global_data.cpuif_pri_bits = 4;
1094 break;
1095 case ICC_IDR0_EL1_PRI_BITS_5BITS:
1096 gicv5_global_data.cpuif_pri_bits = 5;
1097 break;
1098 default:
1099 pr_err("Unexpected ICC_IDR0_EL1_PRI_BITS value, default to 4");
1100 gicv5_global_data.cpuif_pri_bits = 4;
1101 break;
1102 }
1103 }
1104
gicv5_set_cpuif_idbits(void)1105 static void gicv5_set_cpuif_idbits(void)
1106 {
1107 u32 icc_idr0 = read_sysreg_s(SYS_ICC_IDR0_EL1);
1108
1109 switch (FIELD_GET(ICC_IDR0_EL1_ID_BITS, icc_idr0)) {
1110 case ICC_IDR0_EL1_ID_BITS_16BITS:
1111 gicv5_global_data.cpuif_id_bits = 16;
1112 break;
1113 case ICC_IDR0_EL1_ID_BITS_24BITS:
1114 gicv5_global_data.cpuif_id_bits = 24;
1115 break;
1116 default:
1117 pr_err("Unexpected ICC_IDR0_EL1_ID_BITS value, default to 16");
1118 gicv5_global_data.cpuif_id_bits = 16;
1119 break;
1120 }
1121 }
1122
1123 #ifdef CONFIG_KVM
1124 static struct gic_kvm_info gic_v5_kvm_info __initdata;
1125
gic_of_setup_kvm_info(struct device_node * node)1126 static void __init gic_of_setup_kvm_info(struct device_node *node)
1127 {
1128 /*
1129 * If we don't have native GICv5 virtualisation support, then
1130 * we also don't have FEAT_GCIE_LEGACY - the architecture
1131 * forbids this combination.
1132 */
1133 if (!gicv5_global_data.virt_capable) {
1134 pr_info("GIC implementation is not virtualization capable\n");
1135 return;
1136 }
1137
1138 gic_v5_kvm_info.type = GIC_V5;
1139
1140 /* GIC Virtual CPU interface maintenance interrupt */
1141 gic_v5_kvm_info.no_maint_irq_mask = false;
1142 gic_v5_kvm_info.maint_irq = irq_of_parse_and_map(node, 0);
1143 if (!gic_v5_kvm_info.maint_irq) {
1144 pr_warn("cannot find GICv5 virtual CPU interface maintenance interrupt\n");
1145 return;
1146 }
1147
1148 vgic_set_kvm_info(&gic_v5_kvm_info);
1149 }
1150 #else
gic_of_setup_kvm_info(struct device_node * node)1151 static inline void __init gic_of_setup_kvm_info(struct device_node *node)
1152 {
1153 }
1154 #endif // CONFIG_KVM
1155
gicv5_init_common(struct fwnode_handle * parent_domain)1156 static int __init gicv5_init_common(struct fwnode_handle *parent_domain)
1157 {
1158 int ret = gicv5_init_domains(parent_domain);
1159 if (ret)
1160 return ret;
1161
1162 gicv5_set_cpuif_pribits();
1163 gicv5_set_cpuif_idbits();
1164
1165 pri_bits = min_not_zero(gicv5_global_data.cpuif_pri_bits,
1166 gicv5_global_data.irs_pri_bits);
1167
1168 ret = gicv5_starting_cpu(smp_processor_id());
1169 if (ret)
1170 goto out_dom;
1171
1172 ret = set_handle_irq(gicv5_handle_irq);
1173 if (ret)
1174 goto out_int;
1175
1176 ret = gicv5_irs_enable();
1177 if (ret)
1178 goto out_int;
1179
1180 gicv5_smp_init();
1181
1182 gicv5_irs_its_probe();
1183 return 0;
1184
1185 out_int:
1186 gicv5_cpu_disable_interrupts();
1187 out_dom:
1188 gicv5_free_domains();
1189 return ret;
1190 }
1191
gicv5_of_init(struct device_node * node,struct device_node * parent)1192 static int __init gicv5_of_init(struct device_node *node, struct device_node *parent)
1193 {
1194 int ret = gicv5_irs_of_probe(node);
1195 if (ret)
1196 return ret;
1197
1198 ret = gicv5_init_common(of_fwnode_handle(node));
1199 if (ret)
1200 goto out_irs;
1201
1202 gic_of_setup_kvm_info(node);
1203
1204 return 0;
1205 out_irs:
1206 gicv5_irs_remove();
1207
1208 return ret;
1209 }
1210 IRQCHIP_DECLARE(gic_v5, "arm,gic-v5", gicv5_of_init);
1211
1212 #ifdef CONFIG_ACPI
acpi_validate_gic_table(struct acpi_subtable_header * header,struct acpi_probe_entry * ape)1213 static bool __init acpi_validate_gic_table(struct acpi_subtable_header *header,
1214 struct acpi_probe_entry *ape)
1215 {
1216 struct acpi_madt_gicv5_irs *irs = (struct acpi_madt_gicv5_irs *)header;
1217
1218 return (irs->version == ape->driver_data);
1219 }
1220
1221 static struct fwnode_handle *gsi_domain_handle;
1222
gic_v5_get_gsi_domain_id(u32 gsi)1223 static struct fwnode_handle *gic_v5_get_gsi_domain_id(u32 gsi)
1224 {
1225 if (FIELD_GET(GICV5_GSI_IC_TYPE, gsi) == GICV5_GSI_IWB_TYPE)
1226 return iort_iwb_handle(FIELD_GET(GICV5_GSI_IWB_FRAME_ID, gsi));
1227
1228 return gsi_domain_handle;
1229 }
1230
gic_acpi_init(union acpi_subtable_headers * header,const unsigned long end)1231 static int __init gic_acpi_init(union acpi_subtable_headers *header, const unsigned long end)
1232 {
1233 struct acpi_madt_gicv5_irs *irs = (struct acpi_madt_gicv5_irs *)header;
1234 int ret;
1235
1236 if (gsi_domain_handle)
1237 return 0;
1238
1239 gsi_domain_handle = irq_domain_alloc_fwnode(&irs->config_base_address);
1240 if (!gsi_domain_handle)
1241 return -ENOMEM;
1242
1243 ret = gicv5_irs_acpi_probe();
1244 if (ret)
1245 goto out_fwnode;
1246
1247 ret = gicv5_init_common(gsi_domain_handle);
1248 if (ret)
1249 goto out_irs;
1250
1251 acpi_set_irq_model(ACPI_IRQ_MODEL_GIC_V5, gic_v5_get_gsi_domain_id);
1252
1253 return 0;
1254
1255 out_irs:
1256 gicv5_irs_remove();
1257 out_fwnode:
1258 irq_domain_free_fwnode(gsi_domain_handle);
1259 return ret;
1260 }
1261 IRQCHIP_ACPI_DECLARE(gic_v5, ACPI_MADT_TYPE_GICV5_IRS,
1262 acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V5,
1263 gic_acpi_init);
1264 #endif
1265