1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2024-2025 ARM Limited, All Rights Reserved.
4 */
5
6 #define pr_fmt(fmt) "GICv5 IRS: " fmt
7
8 #include <linux/acpi.h>
9 #include <linux/kmemleak.h>
10 #include <linux/log2.h>
11 #include <linux/of.h>
12 #include <linux/of_address.h>
13
14 #include <linux/irqchip.h>
15 #include <linux/irqchip/arm-gic-v5.h>
16
17 /*
18 * Hardcoded ID_BITS limit for systems supporting only a 1-level IST
19 * table. Systems supporting only a 1-level IST table aren't expected
20 * to require more than 2^12 LPIs. Tweak as required.
21 */
22 #define LPI_ID_BITS_LINEAR 12
23
24 #define IRS_FLAGS_NON_COHERENT BIT(0)
25
26 static DEFINE_PER_CPU_READ_MOSTLY(struct gicv5_irs_chip_data *, per_cpu_irs_data);
27 static LIST_HEAD(irs_nodes);
28
irs_readl_relaxed(struct gicv5_irs_chip_data * irs_data,const u32 reg_offset)29 static u32 irs_readl_relaxed(struct gicv5_irs_chip_data *irs_data,
30 const u32 reg_offset)
31 {
32 return readl_relaxed(irs_data->irs_base + reg_offset);
33 }
34
irs_writel_relaxed(struct gicv5_irs_chip_data * irs_data,const u32 val,const u32 reg_offset)35 static void irs_writel_relaxed(struct gicv5_irs_chip_data *irs_data,
36 const u32 val, const u32 reg_offset)
37 {
38 writel_relaxed(val, irs_data->irs_base + reg_offset);
39 }
40
irs_readq_relaxed(struct gicv5_irs_chip_data * irs_data,const u32 reg_offset)41 static u64 irs_readq_relaxed(struct gicv5_irs_chip_data *irs_data,
42 const u32 reg_offset)
43 {
44 return readq_relaxed(irs_data->irs_base + reg_offset);
45 }
46
irs_writeq_relaxed(struct gicv5_irs_chip_data * irs_data,const u64 val,const u32 reg_offset)47 static void irs_writeq_relaxed(struct gicv5_irs_chip_data *irs_data,
48 const u64 val, const u32 reg_offset)
49 {
50 writeq_relaxed(val, irs_data->irs_base + reg_offset);
51 }
52
53 /*
54 * The polling wait (in gicv5_wait_for_op_s_atomic()) on a GIC register
55 * provides the memory barriers (through MMIO accessors)
56 * required to synchronize CPU and GIC access to IST memory.
57 */
gicv5_irs_ist_synchronise(struct gicv5_irs_chip_data * irs_data)58 static int gicv5_irs_ist_synchronise(struct gicv5_irs_chip_data *irs_data)
59 {
60 return gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_IST_STATUSR,
61 GICV5_IRS_IST_STATUSR_IDLE, NULL);
62 }
63
gicv5_irs_init_ist_linear(struct gicv5_irs_chip_data * irs_data,unsigned int lpi_id_bits,unsigned int istsz)64 static int __init gicv5_irs_init_ist_linear(struct gicv5_irs_chip_data *irs_data,
65 unsigned int lpi_id_bits,
66 unsigned int istsz)
67 {
68 size_t l2istsz;
69 u32 n, cfgr;
70 void *ist;
71 u64 baser;
72 int ret;
73
74 /* Taken from GICv5 specifications 10.2.1.13 IRS_IST_BASER */
75 n = max(5, lpi_id_bits + 1 + istsz);
76
77 l2istsz = BIT(n + 1);
78 /*
79 * Check memory requirements. For a linear IST we cap the
80 * number of ID bits to a value that should never exceed
81 * kmalloc interface memory allocation limits, so this
82 * check is really belt and braces.
83 */
84 if (l2istsz > KMALLOC_MAX_SIZE) {
85 u8 lpi_id_cap = ilog2(KMALLOC_MAX_SIZE) - 2 + istsz;
86
87 pr_warn("Limiting LPI ID bits from %u to %u\n",
88 lpi_id_bits, lpi_id_cap);
89 lpi_id_bits = lpi_id_cap;
90 l2istsz = KMALLOC_MAX_SIZE;
91 }
92
93 ist = kzalloc(l2istsz, GFP_KERNEL);
94 if (!ist)
95 return -ENOMEM;
96
97 if (irs_data->flags & IRS_FLAGS_NON_COHERENT)
98 dcache_clean_inval_poc((unsigned long)ist,
99 (unsigned long)ist + l2istsz);
100 else
101 dsb(ishst);
102
103 cfgr = FIELD_PREP(GICV5_IRS_IST_CFGR_STRUCTURE,
104 GICV5_IRS_IST_CFGR_STRUCTURE_LINEAR) |
105 FIELD_PREP(GICV5_IRS_IST_CFGR_ISTSZ, istsz) |
106 FIELD_PREP(GICV5_IRS_IST_CFGR_L2SZ,
107 GICV5_IRS_IST_CFGR_L2SZ_4K) |
108 FIELD_PREP(GICV5_IRS_IST_CFGR_LPI_ID_BITS, lpi_id_bits);
109 irs_writel_relaxed(irs_data, cfgr, GICV5_IRS_IST_CFGR);
110
111 gicv5_global_data.ist.l2 = false;
112
113 baser = (virt_to_phys(ist) & GICV5_IRS_IST_BASER_ADDR_MASK) |
114 FIELD_PREP(GICV5_IRS_IST_BASER_VALID, 0x1);
115 irs_writeq_relaxed(irs_data, baser, GICV5_IRS_IST_BASER);
116
117 ret = gicv5_irs_ist_synchronise(irs_data);
118 if (ret) {
119 kfree(ist);
120 return ret;
121 }
122 kmemleak_ignore(ist);
123
124 return 0;
125 }
126
gicv5_irs_init_ist_two_level(struct gicv5_irs_chip_data * irs_data,unsigned int lpi_id_bits,unsigned int istsz,unsigned int l2sz)127 static int __init gicv5_irs_init_ist_two_level(struct gicv5_irs_chip_data *irs_data,
128 unsigned int lpi_id_bits,
129 unsigned int istsz,
130 unsigned int l2sz)
131 {
132 __le64 *l1ist;
133 u32 cfgr, n;
134 size_t l1sz;
135 u64 baser;
136 int ret;
137
138 /* Taken from GICv5 specifications 10.2.1.13 IRS_IST_BASER */
139 n = max(5, lpi_id_bits - ((10 - istsz) + (2 * l2sz)) + 2);
140
141 l1sz = BIT(n + 1);
142
143 l1ist = kzalloc(l1sz, GFP_KERNEL);
144 if (!l1ist)
145 return -ENOMEM;
146
147 if (irs_data->flags & IRS_FLAGS_NON_COHERENT)
148 dcache_clean_inval_poc((unsigned long)l1ist,
149 (unsigned long)l1ist + l1sz);
150 else
151 dsb(ishst);
152
153 cfgr = FIELD_PREP(GICV5_IRS_IST_CFGR_STRUCTURE,
154 GICV5_IRS_IST_CFGR_STRUCTURE_TWO_LEVEL) |
155 FIELD_PREP(GICV5_IRS_IST_CFGR_ISTSZ, istsz) |
156 FIELD_PREP(GICV5_IRS_IST_CFGR_L2SZ, l2sz) |
157 FIELD_PREP(GICV5_IRS_IST_CFGR_LPI_ID_BITS, lpi_id_bits);
158 irs_writel_relaxed(irs_data, cfgr, GICV5_IRS_IST_CFGR);
159
160 /*
161 * The L2SZ determine bits required at L2 level. Number of bytes
162 * required by metadata is reported through istsz - the number of bits
163 * covered by L2 entries scales accordingly.
164 */
165 gicv5_global_data.ist.l2_size = BIT(11 + (2 * l2sz) + 1);
166 gicv5_global_data.ist.l2_bits = (10 - istsz) + (2 * l2sz);
167 gicv5_global_data.ist.l1ist_addr = l1ist;
168 gicv5_global_data.ist.l2 = true;
169
170 baser = (virt_to_phys(l1ist) & GICV5_IRS_IST_BASER_ADDR_MASK) |
171 FIELD_PREP(GICV5_IRS_IST_BASER_VALID, 0x1);
172 irs_writeq_relaxed(irs_data, baser, GICV5_IRS_IST_BASER);
173
174 ret = gicv5_irs_ist_synchronise(irs_data);
175 if (ret) {
176 kfree(l1ist);
177 return ret;
178 }
179
180 return 0;
181 }
182
183 /*
184 * Alloc L2 IST entries on demand.
185 *
186 * Locking/serialization is guaranteed by irqdomain core code by
187 * taking the hierarchical domain struct irq_domain.root->mutex.
188 */
gicv5_irs_iste_alloc(const u32 lpi)189 int gicv5_irs_iste_alloc(const u32 lpi)
190 {
191 struct gicv5_irs_chip_data *irs_data;
192 unsigned int index;
193 u32 l2istr, l2bits;
194 __le64 *l1ist;
195 size_t l2size;
196 void *l2ist;
197 int ret;
198
199 if (!gicv5_global_data.ist.l2)
200 return 0;
201
202 irs_data = per_cpu(per_cpu_irs_data, 0);
203 if (!irs_data)
204 return -ENOENT;
205
206 l2size = gicv5_global_data.ist.l2_size;
207 l2bits = gicv5_global_data.ist.l2_bits;
208 l1ist = gicv5_global_data.ist.l1ist_addr;
209 index = lpi >> l2bits;
210
211 if (FIELD_GET(GICV5_ISTL1E_VALID, le64_to_cpu(l1ist[index])))
212 return 0;
213
214 l2ist = kzalloc(l2size, GFP_KERNEL);
215 if (!l2ist)
216 return -ENOMEM;
217
218 l1ist[index] = cpu_to_le64(virt_to_phys(l2ist) & GICV5_ISTL1E_L2_ADDR_MASK);
219
220 if (irs_data->flags & IRS_FLAGS_NON_COHERENT) {
221 dcache_clean_inval_poc((unsigned long)l2ist,
222 (unsigned long)l2ist + l2size);
223 dcache_clean_poc((unsigned long)(l1ist + index),
224 (unsigned long)(l1ist + index) + sizeof(*l1ist));
225 } else {
226 dsb(ishst);
227 }
228
229 l2istr = FIELD_PREP(GICV5_IRS_MAP_L2_ISTR_ID, lpi);
230 irs_writel_relaxed(irs_data, l2istr, GICV5_IRS_MAP_L2_ISTR);
231
232 ret = gicv5_irs_ist_synchronise(irs_data);
233 if (ret) {
234 l1ist[index] = 0;
235 kfree(l2ist);
236 return ret;
237 }
238 kmemleak_ignore(l2ist);
239
240 /*
241 * Make sure we invalidate the cache line pulled before the IRS
242 * had a chance to update the L1 entry and mark it valid.
243 */
244 if (irs_data->flags & IRS_FLAGS_NON_COHERENT) {
245 /*
246 * gicv5_irs_ist_synchronise() includes memory
247 * barriers (MMIO accessors) required to guarantee that the
248 * following dcache invalidation is not executed before the
249 * IST mapping operation has completed.
250 */
251 dcache_inval_poc((unsigned long)(l1ist + index),
252 (unsigned long)(l1ist + index) + sizeof(*l1ist));
253 }
254
255 return 0;
256 }
257
258 /*
259 * Try to match the L2 IST size to the pagesize, and if this is not possible
260 * pick the smallest supported L2 size in order to minimise the requirement for
261 * physically contiguous blocks of memory as page-sized allocations are
262 * guaranteed to be physically contiguous, and are by definition the easiest to
263 * find.
264 *
265 * Fall back to the smallest supported size (in the event that the pagesize
266 * itself is not supported) again serves to make it easier to find physically
267 * contiguous blocks of memory.
268 */
gicv5_irs_l2_sz(u32 idr2)269 static unsigned int gicv5_irs_l2_sz(u32 idr2)
270 {
271 switch (PAGE_SIZE) {
272 case SZ_64K:
273 if (GICV5_IRS_IST_L2SZ_SUPPORT_64KB(idr2))
274 return GICV5_IRS_IST_CFGR_L2SZ_64K;
275 fallthrough;
276 case SZ_4K:
277 if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(idr2))
278 return GICV5_IRS_IST_CFGR_L2SZ_4K;
279 fallthrough;
280 case SZ_16K:
281 if (GICV5_IRS_IST_L2SZ_SUPPORT_16KB(idr2))
282 return GICV5_IRS_IST_CFGR_L2SZ_16K;
283 break;
284 }
285
286 if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(idr2))
287 return GICV5_IRS_IST_CFGR_L2SZ_4K;
288
289 return GICV5_IRS_IST_CFGR_L2SZ_64K;
290 }
291
gicv5_irs_init_ist(struct gicv5_irs_chip_data * irs_data)292 static int __init gicv5_irs_init_ist(struct gicv5_irs_chip_data *irs_data)
293 {
294 u32 lpi_id_bits, idr2_id_bits, idr2_min_lpi_id_bits, l2_iste_sz, l2sz;
295 u32 l2_iste_sz_split, idr2;
296 bool two_levels, istmd;
297 u64 baser;
298 int ret;
299
300 baser = irs_readq_relaxed(irs_data, GICV5_IRS_IST_BASER);
301 if (FIELD_GET(GICV5_IRS_IST_BASER_VALID, baser)) {
302 pr_err("IST is marked as valid already; cannot allocate\n");
303 return -EPERM;
304 }
305
306 idr2 = irs_readl_relaxed(irs_data, GICV5_IRS_IDR2);
307 two_levels = !!FIELD_GET(GICV5_IRS_IDR2_IST_LEVELS, idr2);
308
309 idr2_id_bits = FIELD_GET(GICV5_IRS_IDR2_ID_BITS, idr2);
310 idr2_min_lpi_id_bits = FIELD_GET(GICV5_IRS_IDR2_MIN_LPI_ID_BITS, idr2);
311
312 /*
313 * For two level tables we are always supporting the maximum allowed
314 * number of IDs.
315 *
316 * For 1-level tables, we should support a number of bits that
317 * is >= min_lpi_id_bits but cap it to LPI_ID_BITS_LINEAR lest
318 * the level 1-table gets too large and its memory allocation
319 * may fail.
320 */
321 if (two_levels) {
322 lpi_id_bits = idr2_id_bits;
323 } else {
324 lpi_id_bits = max(LPI_ID_BITS_LINEAR, idr2_min_lpi_id_bits);
325 lpi_id_bits = min(lpi_id_bits, idr2_id_bits);
326 }
327
328 /*
329 * Cap the ID bits according to the CPUIF supported ID bits
330 */
331 lpi_id_bits = min(lpi_id_bits, gicv5_global_data.cpuif_id_bits);
332
333 if (two_levels)
334 l2sz = gicv5_irs_l2_sz(idr2);
335
336 istmd = !!FIELD_GET(GICV5_IRS_IDR2_ISTMD, idr2);
337
338 l2_iste_sz = GICV5_IRS_IST_CFGR_ISTSZ_4;
339
340 if (istmd) {
341 l2_iste_sz_split = FIELD_GET(GICV5_IRS_IDR2_ISTMD_SZ, idr2);
342
343 if (lpi_id_bits < l2_iste_sz_split)
344 l2_iste_sz = GICV5_IRS_IST_CFGR_ISTSZ_8;
345 else
346 l2_iste_sz = GICV5_IRS_IST_CFGR_ISTSZ_16;
347 }
348
349 /*
350 * Follow GICv5 specification recommendation to opt in for two
351 * level tables (ref: 10.2.1.14 IRS_IST_CFGR).
352 */
353 if (two_levels && (lpi_id_bits > ((10 - l2_iste_sz) + (2 * l2sz)))) {
354 ret = gicv5_irs_init_ist_two_level(irs_data, lpi_id_bits,
355 l2_iste_sz, l2sz);
356 } else {
357 ret = gicv5_irs_init_ist_linear(irs_data, lpi_id_bits,
358 l2_iste_sz);
359 }
360 if (ret)
361 return ret;
362
363 gicv5_init_lpis(BIT(lpi_id_bits));
364
365 return 0;
366 }
367
368 struct iaffid_entry {
369 u16 iaffid;
370 bool valid;
371 };
372
373 static DEFINE_PER_CPU(struct iaffid_entry, cpu_iaffid);
374
gicv5_irs_cpu_to_iaffid(int cpuid,u16 * iaffid)375 int gicv5_irs_cpu_to_iaffid(int cpuid, u16 *iaffid)
376 {
377 if (!per_cpu(cpu_iaffid, cpuid).valid) {
378 pr_err("IAFFID for CPU %d has not been initialised\n", cpuid);
379 return -ENODEV;
380 }
381
382 *iaffid = per_cpu(cpu_iaffid, cpuid).iaffid;
383
384 return 0;
385 }
386
gicv5_irs_lookup_by_spi_id(u32 spi_id)387 struct gicv5_irs_chip_data *gicv5_irs_lookup_by_spi_id(u32 spi_id)
388 {
389 struct gicv5_irs_chip_data *irs_data;
390 u32 min, max;
391
392 list_for_each_entry(irs_data, &irs_nodes, entry) {
393 if (!irs_data->spi_range)
394 continue;
395
396 min = irs_data->spi_min;
397 max = irs_data->spi_min + irs_data->spi_range - 1;
398 if (spi_id >= min && spi_id <= max)
399 return irs_data;
400 }
401
402 return NULL;
403 }
404
gicv5_irs_wait_for_spi_op(struct gicv5_irs_chip_data * irs_data)405 static int gicv5_irs_wait_for_spi_op(struct gicv5_irs_chip_data *irs_data)
406 {
407 u32 statusr;
408 int ret;
409
410 ret = gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_SPI_STATUSR,
411 GICV5_IRS_SPI_STATUSR_IDLE, &statusr);
412 if (ret)
413 return ret;
414
415 return !!FIELD_GET(GICV5_IRS_SPI_STATUSR_V, statusr) ? 0 : -EIO;
416 }
417
gicv5_irs_wait_for_irs_pe(struct gicv5_irs_chip_data * irs_data,bool selr)418 static int gicv5_irs_wait_for_irs_pe(struct gicv5_irs_chip_data *irs_data,
419 bool selr)
420 {
421 bool valid = true;
422 u32 statusr;
423 int ret;
424
425 ret = gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_PE_STATUSR,
426 GICV5_IRS_PE_STATUSR_IDLE, &statusr);
427 if (ret)
428 return ret;
429
430 if (selr)
431 valid = !!FIELD_GET(GICV5_IRS_PE_STATUSR_V, statusr);
432
433 return valid ? 0 : -EIO;
434 }
435
gicv5_irs_wait_for_pe_selr(struct gicv5_irs_chip_data * irs_data)436 static int gicv5_irs_wait_for_pe_selr(struct gicv5_irs_chip_data *irs_data)
437 {
438 return gicv5_irs_wait_for_irs_pe(irs_data, true);
439 }
440
gicv5_irs_wait_for_pe_cr0(struct gicv5_irs_chip_data * irs_data)441 static int gicv5_irs_wait_for_pe_cr0(struct gicv5_irs_chip_data *irs_data)
442 {
443 return gicv5_irs_wait_for_irs_pe(irs_data, false);
444 }
445
gicv5_spi_irq_set_type(struct irq_data * d,unsigned int type)446 int gicv5_spi_irq_set_type(struct irq_data *d, unsigned int type)
447 {
448 struct gicv5_irs_chip_data *irs_data = d->chip_data;
449 u32 selr, cfgr;
450 bool level;
451 int ret;
452
453 /*
454 * There is no distinction between HIGH/LOW for level IRQs
455 * and RISING/FALLING for edge IRQs in the architecture,
456 * hence consider them equivalent.
457 */
458 switch (type) {
459 case IRQ_TYPE_EDGE_RISING:
460 case IRQ_TYPE_EDGE_FALLING:
461 level = false;
462 break;
463 case IRQ_TYPE_LEVEL_HIGH:
464 case IRQ_TYPE_LEVEL_LOW:
465 level = true;
466 break;
467 default:
468 return -EINVAL;
469 }
470
471 guard(raw_spinlock)(&irs_data->spi_config_lock);
472
473 selr = FIELD_PREP(GICV5_IRS_SPI_SELR_ID, d->hwirq);
474 irs_writel_relaxed(irs_data, selr, GICV5_IRS_SPI_SELR);
475 ret = gicv5_irs_wait_for_spi_op(irs_data);
476 if (ret)
477 return ret;
478
479 cfgr = FIELD_PREP(GICV5_IRS_SPI_CFGR_TM, level);
480 irs_writel_relaxed(irs_data, cfgr, GICV5_IRS_SPI_CFGR);
481
482 return gicv5_irs_wait_for_spi_op(irs_data);
483 }
484
gicv5_irs_wait_for_idle(struct gicv5_irs_chip_data * irs_data)485 static int gicv5_irs_wait_for_idle(struct gicv5_irs_chip_data *irs_data)
486 {
487 return gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_CR0,
488 GICV5_IRS_CR0_IDLE, NULL);
489 }
490
gicv5_irs_disable(struct gicv5_irs_chip_data * irs_data)491 static void __init gicv5_irs_disable(struct gicv5_irs_chip_data *irs_data)
492 {
493 irs_writel_relaxed(irs_data, 0, GICV5_IRS_CR0);
494 gicv5_irs_wait_for_idle(irs_data);
495 }
496
gicv5_irs_syncr(void)497 void gicv5_irs_syncr(void)
498 {
499 struct gicv5_irs_chip_data *irs_data;
500 u32 syncr;
501
502 irs_data = list_first_entry_or_null(&irs_nodes, struct gicv5_irs_chip_data, entry);
503 if (WARN_ON_ONCE(!irs_data))
504 return;
505
506 syncr = FIELD_PREP(GICV5_IRS_SYNCR_SYNC, 1);
507 irs_writel_relaxed(irs_data, syncr, GICV5_IRS_SYNCR);
508
509 gicv5_wait_for_op(irs_data->irs_base, GICV5_IRS_SYNC_STATUSR,
510 GICV5_IRS_SYNC_STATUSR_IDLE);
511 }
512
gicv5_irs_register_cpu(int cpuid)513 int gicv5_irs_register_cpu(int cpuid)
514 {
515 struct gicv5_irs_chip_data *irs_data;
516 u32 selr, cr0;
517 u16 iaffid;
518 int ret;
519
520 ret = gicv5_irs_cpu_to_iaffid(cpuid, &iaffid);
521 if (ret) {
522 pr_err("IAFFID for CPU %d has not been initialised\n", cpuid);
523 return ret;
524 }
525
526 irs_data = per_cpu(per_cpu_irs_data, cpuid);
527 if (!irs_data) {
528 pr_err("No IRS associated with CPU %u\n", cpuid);
529 return -ENXIO;
530 }
531
532 selr = FIELD_PREP(GICV5_IRS_PE_SELR_IAFFID, iaffid);
533 irs_writel_relaxed(irs_data, selr, GICV5_IRS_PE_SELR);
534
535 ret = gicv5_irs_wait_for_pe_selr(irs_data);
536 if (ret) {
537 pr_err("IAFFID 0x%x used in IRS_PE_SELR is invalid\n", iaffid);
538 return -ENXIO;
539 }
540
541 cr0 = FIELD_PREP(GICV5_IRS_PE_CR0_DPS, 0x1);
542 irs_writel_relaxed(irs_data, cr0, GICV5_IRS_PE_CR0);
543
544 ret = gicv5_irs_wait_for_pe_cr0(irs_data);
545 if (ret)
546 return ret;
547
548 pr_debug("CPU %d enabled PE IAFFID 0x%x\n", cpuid, iaffid);
549
550 return 0;
551 }
552
gicv5_irs_init_bases(struct gicv5_irs_chip_data * irs_data,void __iomem * irs_base,bool noncoherent)553 static void __init gicv5_irs_init_bases(struct gicv5_irs_chip_data *irs_data,
554 void __iomem *irs_base,
555 bool noncoherent)
556 {
557 u32 cr0, cr1;
558
559 irs_data->irs_base = irs_base;
560
561 if (noncoherent) {
562 /*
563 * A non-coherent IRS implies that some cache levels cannot be
564 * used coherently by the cores and GIC. Our only option is to mark
565 * memory attributes for the GIC as non-cacheable; by default,
566 * non-cacheable memory attributes imply outer-shareable
567 * shareability, the value written into IRS_CR1_SH is ignored.
568 */
569 cr1 = FIELD_PREP(GICV5_IRS_CR1_VPED_WA, GICV5_NO_WRITE_ALLOC) |
570 FIELD_PREP(GICV5_IRS_CR1_VPED_RA, GICV5_NO_READ_ALLOC) |
571 FIELD_PREP(GICV5_IRS_CR1_VMD_WA, GICV5_NO_WRITE_ALLOC) |
572 FIELD_PREP(GICV5_IRS_CR1_VMD_RA, GICV5_NO_READ_ALLOC) |
573 FIELD_PREP(GICV5_IRS_CR1_VPET_RA, GICV5_NO_READ_ALLOC) |
574 FIELD_PREP(GICV5_IRS_CR1_VMT_RA, GICV5_NO_READ_ALLOC) |
575 FIELD_PREP(GICV5_IRS_CR1_IST_WA, GICV5_NO_WRITE_ALLOC) |
576 FIELD_PREP(GICV5_IRS_CR1_IST_RA, GICV5_NO_READ_ALLOC) |
577 FIELD_PREP(GICV5_IRS_CR1_IC, GICV5_NON_CACHE) |
578 FIELD_PREP(GICV5_IRS_CR1_OC, GICV5_NON_CACHE);
579 irs_data->flags |= IRS_FLAGS_NON_COHERENT;
580 } else {
581 cr1 = FIELD_PREP(GICV5_IRS_CR1_VPED_WA, GICV5_WRITE_ALLOC) |
582 FIELD_PREP(GICV5_IRS_CR1_VPED_RA, GICV5_READ_ALLOC) |
583 FIELD_PREP(GICV5_IRS_CR1_VMD_WA, GICV5_WRITE_ALLOC) |
584 FIELD_PREP(GICV5_IRS_CR1_VMD_RA, GICV5_READ_ALLOC) |
585 FIELD_PREP(GICV5_IRS_CR1_VPET_RA, GICV5_READ_ALLOC) |
586 FIELD_PREP(GICV5_IRS_CR1_VMT_RA, GICV5_READ_ALLOC) |
587 FIELD_PREP(GICV5_IRS_CR1_IST_WA, GICV5_WRITE_ALLOC) |
588 FIELD_PREP(GICV5_IRS_CR1_IST_RA, GICV5_READ_ALLOC) |
589 FIELD_PREP(GICV5_IRS_CR1_IC, GICV5_WB_CACHE) |
590 FIELD_PREP(GICV5_IRS_CR1_OC, GICV5_WB_CACHE) |
591 FIELD_PREP(GICV5_IRS_CR1_SH, GICV5_INNER_SHARE);
592 }
593
594 irs_writel_relaxed(irs_data, cr1, GICV5_IRS_CR1);
595
596 cr0 = FIELD_PREP(GICV5_IRS_CR0_IRSEN, 0x1);
597 irs_writel_relaxed(irs_data, cr0, GICV5_IRS_CR0);
598 gicv5_irs_wait_for_idle(irs_data);
599 }
600
gicv5_irs_of_init_affinity(struct device_node * node,struct gicv5_irs_chip_data * irs_data,u8 iaffid_bits)601 static int __init gicv5_irs_of_init_affinity(struct device_node *node,
602 struct gicv5_irs_chip_data *irs_data,
603 u8 iaffid_bits)
604 {
605 /*
606 * Detect IAFFID<->CPU mappings from the device tree and
607 * record IRS<->CPU topology information.
608 */
609 u16 iaffid_mask = GENMASK(iaffid_bits - 1, 0);
610 int ret, i, ncpus, niaffids;
611
612 ncpus = of_count_phandle_with_args(node, "cpus", NULL);
613 if (ncpus < 0)
614 return -EINVAL;
615
616 niaffids = of_property_count_elems_of_size(node, "arm,iaffids",
617 sizeof(u16));
618 if (niaffids != ncpus)
619 return -EINVAL;
620
621 u16 *iaffids __free(kfree) = kzalloc_objs(*iaffids, niaffids);
622 if (!iaffids)
623 return -ENOMEM;
624
625 ret = of_property_read_u16_array(node, "arm,iaffids", iaffids, niaffids);
626 if (ret)
627 return ret;
628
629 for (i = 0; i < ncpus; i++) {
630 struct device_node *cpu_node;
631 int cpu;
632
633 cpu_node = of_parse_phandle(node, "cpus", i);
634 if (!cpu_node) {
635 pr_warn(FW_BUG "Erroneous CPU node phandle\n");
636 continue;
637 }
638
639 cpu = of_cpu_node_to_id(cpu_node);
640 of_node_put(cpu_node);
641 if (cpu < 0)
642 continue;
643
644 if (iaffids[i] & ~iaffid_mask) {
645 pr_warn("CPU %d iaffid 0x%x exceeds IRS iaffid bits\n",
646 cpu, iaffids[i]);
647 continue;
648 }
649
650 per_cpu(cpu_iaffid, cpu).iaffid = iaffids[i];
651 per_cpu(cpu_iaffid, cpu).valid = true;
652
653 /* We also know that the CPU is connected to this IRS */
654 per_cpu(per_cpu_irs_data, cpu) = irs_data;
655 }
656
657 return ret;
658 }
659
gicv5_irs_clear_affinity(struct gicv5_irs_chip_data * irs_data)660 static void __init gicv5_irs_clear_affinity(struct gicv5_irs_chip_data *irs_data)
661 {
662 int cpu;
663
664 for_each_possible_cpu(cpu) {
665 if (per_cpu(per_cpu_irs_data, cpu) == irs_data) {
666 per_cpu(cpu_iaffid, cpu).valid = false;
667 per_cpu(per_cpu_irs_data, cpu) = NULL;
668 }
669 }
670 }
671
irs_setup_pri_bits(u32 idr1)672 static void irs_setup_pri_bits(u32 idr1)
673 {
674 switch (FIELD_GET(GICV5_IRS_IDR1_PRIORITY_BITS, idr1)) {
675 case GICV5_IRS_IDR1_PRIORITY_BITS_1BITS:
676 gicv5_global_data.irs_pri_bits = 1;
677 break;
678 case GICV5_IRS_IDR1_PRIORITY_BITS_2BITS:
679 gicv5_global_data.irs_pri_bits = 2;
680 break;
681 case GICV5_IRS_IDR1_PRIORITY_BITS_3BITS:
682 gicv5_global_data.irs_pri_bits = 3;
683 break;
684 case GICV5_IRS_IDR1_PRIORITY_BITS_4BITS:
685 gicv5_global_data.irs_pri_bits = 4;
686 break;
687 case GICV5_IRS_IDR1_PRIORITY_BITS_5BITS:
688 gicv5_global_data.irs_pri_bits = 5;
689 break;
690 default:
691 pr_warn("Detected wrong IDR priority bits value 0x%lx\n",
692 FIELD_GET(GICV5_IRS_IDR1_PRIORITY_BITS, idr1));
693 gicv5_global_data.irs_pri_bits = 1;
694 break;
695 }
696 }
697
gicv5_irs_init(struct gicv5_irs_chip_data * irs_data)698 static int __init gicv5_irs_init(struct gicv5_irs_chip_data *irs_data)
699 {
700 u32 spi_count, idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR2);
701
702 if (WARN(!FIELD_GET(GICV5_IRS_IDR2_LPI, idr),
703 "LPI support not available - no IPIs, can't proceed\n")) {
704 return -ENODEV;
705 }
706
707 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR7);
708 irs_data->spi_min = FIELD_GET(GICV5_IRS_IDR7_SPI_BASE, idr);
709
710 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR6);
711 irs_data->spi_range = FIELD_GET(GICV5_IRS_IDR6_SPI_IRS_RANGE, idr);
712
713 /*
714 * Do the global setting only on the first IRS.
715 * Global properties (iaffid_bits, global spi count) are guaranteed to
716 * be consistent across IRSes by the architecture.
717 */
718 if (list_empty(&irs_nodes)) {
719 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR0);
720 gicv5_global_data.virt_capable = !!FIELD_GET(GICV5_IRS_IDR0_VIRT, idr);
721
722 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR1);
723 irs_setup_pri_bits(idr);
724
725 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR5);
726
727 spi_count = FIELD_GET(GICV5_IRS_IDR5_SPI_RANGE, idr);
728 gicv5_global_data.global_spi_count = spi_count;
729
730 gicv5_init_lpi_domain();
731
732 pr_debug("Detected %u SPIs globally\n", spi_count);
733 }
734
735 list_add_tail(&irs_data->entry, &irs_nodes);
736
737 return 0;
738 }
739
gicv5_irs_of_init(struct device_node * node)740 static int __init gicv5_irs_of_init(struct device_node *node)
741 {
742 struct gicv5_irs_chip_data *irs_data;
743 struct resource res;
744 void __iomem *irs_base;
745 u8 iaffid_bits;
746 u32 idr;
747 int ret;
748
749 irs_data = kzalloc_obj(*irs_data);
750 if (!irs_data)
751 return -ENOMEM;
752
753 raw_spin_lock_init(&irs_data->spi_config_lock);
754
755 ret = of_property_match_string(node, "reg-names", "ns-config");
756 if (ret < 0) {
757 pr_err("%pOF: ns-config reg-name not present\n", node);
758 goto out_err;
759 }
760
761 if (of_address_to_resource(node, ret, &res)) {
762 ret = -EINVAL;
763 goto out_err;
764 }
765
766 irs_base = of_io_request_and_map(node, ret, of_node_full_name(node));
767 if (IS_ERR(irs_base)) {
768 pr_err("%pOF: unable to map GICv5 IRS registers\n", node);
769 ret = PTR_ERR(irs_base);
770 goto out_err;
771 }
772
773 irs_data->fwnode = of_fwnode_handle(node);
774 irs_data->res = res;
775 gicv5_irs_init_bases(irs_data, irs_base, of_property_read_bool(node, "dma-noncoherent"));
776
777 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR1);
778 iaffid_bits = FIELD_GET(GICV5_IRS_IDR1_IAFFID_BITS, idr) + 1;
779
780 ret = gicv5_irs_of_init_affinity(node, irs_data, iaffid_bits);
781 if (ret) {
782 pr_err("Failed to parse CPU IAFFIDs from the device tree!\n");
783 goto out_iomem;
784 }
785
786 ret = gicv5_irs_init(irs_data);
787 if (ret)
788 goto out_iomem;
789
790 if (irs_data->spi_range) {
791 pr_info("%s detected SPI range [%u-%u]\n",
792 of_node_full_name(node),
793 irs_data->spi_min,
794 irs_data->spi_min +
795 irs_data->spi_range - 1);
796 }
797
798 return ret;
799
800 out_iomem:
801 gicv5_irs_clear_affinity(irs_data);
802 gicv5_irs_disable(irs_data);
803 iounmap(irs_base);
804 release_mem_region(res.start, resource_size(&res));
805 out_err:
806 kfree(irs_data);
807 return ret;
808 }
809
gicv5_irs_remove(void)810 void __init gicv5_irs_remove(void)
811 {
812 struct gicv5_irs_chip_data *irs_data, *tmp_data;
813
814 gicv5_free_lpi_domain();
815 gicv5_deinit_lpis();
816
817 list_for_each_entry_safe(irs_data, tmp_data, &irs_nodes, entry) {
818 gicv5_irs_clear_affinity(irs_data);
819 gicv5_irs_disable(irs_data);
820 iounmap(irs_data->irs_base);
821 release_mem_region(irs_data->res.start, resource_size(&irs_data->res));
822 list_del(&irs_data->entry);
823 kfree(irs_data);
824 }
825 }
826
gicv5_irs_enable(void)827 int __init gicv5_irs_enable(void)
828 {
829 struct gicv5_irs_chip_data *irs_data;
830 int ret;
831
832 irs_data = list_first_entry_or_null(&irs_nodes,
833 struct gicv5_irs_chip_data, entry);
834 if (!irs_data)
835 return -ENODEV;
836
837 ret = gicv5_irs_init_ist(irs_data);
838 if (ret) {
839 pr_err("Failed to init IST\n");
840 return ret;
841 }
842
843 return 0;
844 }
845
gicv5_irs_its_probe(void)846 void __init gicv5_irs_its_probe(void)
847 {
848 struct gicv5_irs_chip_data *irs_data;
849
850 if (acpi_disabled)
851 list_for_each_entry(irs_data, &irs_nodes, entry)
852 gicv5_its_of_probe(to_of_node(irs_data->fwnode));
853 else
854 gicv5_its_acpi_probe();
855 }
856
gicv5_irs_of_probe(struct device_node * parent)857 int __init gicv5_irs_of_probe(struct device_node *parent)
858 {
859 struct device_node *np;
860 int ret;
861
862 for_each_available_child_of_node(parent, np) {
863 if (!of_device_is_compatible(np, "arm,gic-v5-irs"))
864 continue;
865
866 ret = gicv5_irs_of_init(np);
867 if (ret)
868 pr_err("Failed to init IRS %s\n", np->full_name);
869 }
870
871 return list_empty(&irs_nodes) ? -ENODEV : 0;
872 }
873
874 #ifdef CONFIG_ACPI
875
876 #define ACPI_GICV5_IRS_MEM_SIZE (SZ_64K)
877 static struct gicv5_irs_chip_data *current_irs_data __initdata;
878 static int current_irsid __initdata = -1;
879 static u8 current_iaffid_bits __initdata;
880
gic_acpi_parse_iaffid(union acpi_subtable_headers * header,const unsigned long end)881 static int __init gic_acpi_parse_iaffid(union acpi_subtable_headers *header,
882 const unsigned long end)
883 {
884 struct acpi_madt_generic_interrupt *gicc = (struct acpi_madt_generic_interrupt *)header;
885 int cpu;
886
887 if (!(gicc->flags & (ACPI_MADT_ENABLED | ACPI_MADT_GICC_ONLINE_CAPABLE)))
888 return 0;
889
890 if (gicc->irs_id != current_irsid)
891 return 0;
892
893 cpu = get_logical_index(gicc->arm_mpidr);
894 if (cpu < 0)
895 return 0;
896
897 if (gicc->iaffid & ~GENMASK(current_iaffid_bits - 1, 0)) {
898 pr_warn("CPU %d iaffid 0x%x exceeds IRS iaffid bits\n", cpu, gicc->iaffid);
899 return 0;
900 }
901
902 /* Bind the IAFFID and the CPU */
903 per_cpu(cpu_iaffid, cpu).iaffid = gicc->iaffid;
904 per_cpu(cpu_iaffid, cpu).valid = true;
905 pr_debug("Processed IAFFID %u for CPU%d", per_cpu(cpu_iaffid, cpu).iaffid, cpu);
906
907 /* We also know that the CPU is connected to this IRS */
908 per_cpu(per_cpu_irs_data, cpu) = current_irs_data;
909
910 return 0;
911 }
912
gicv5_irs_acpi_init_affinity(u32 irsid,struct gicv5_irs_chip_data * irs_data)913 static int __init gicv5_irs_acpi_init_affinity(u32 irsid, struct gicv5_irs_chip_data *irs_data)
914 {
915 u32 idr;
916
917 current_irsid = irsid;
918 current_irs_data = irs_data;
919
920 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR1);
921 current_iaffid_bits = FIELD_GET(GICV5_IRS_IDR1_IAFFID_BITS, idr) + 1;
922
923 acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT, gic_acpi_parse_iaffid, 0);
924
925 return 0;
926 }
927
gic_request_region(resource_size_t base,resource_size_t size,const char * name)928 static struct resource * __init gic_request_region(resource_size_t base, resource_size_t size,
929 const char *name)
930 {
931 struct resource *r = request_mem_region(base, size, name);
932
933 if (!r)
934 pr_warn_once(FW_BUG "%s region %pa has overlapping address\n", name, &base);
935
936 return r;
937 }
938
gic_acpi_parse_madt_irs(union acpi_subtable_headers * header,const unsigned long end)939 static int __init gic_acpi_parse_madt_irs(union acpi_subtable_headers *header,
940 const unsigned long end)
941 {
942 struct acpi_madt_gicv5_irs *irs = (struct acpi_madt_gicv5_irs *)header;
943 struct gicv5_irs_chip_data *irs_data;
944 void __iomem *irs_base;
945 struct resource *r;
946 int ret;
947
948 /* Per-IRS data structure */
949 irs_data = kzalloc_obj(*irs_data);
950 if (!irs_data)
951 return -ENOMEM;
952
953 /* This spinlock is used for SPI config changes */
954 raw_spin_lock_init(&irs_data->spi_config_lock);
955
956 r = gic_request_region(irs->config_base_address, ACPI_GICV5_IRS_MEM_SIZE, "GICv5 IRS");
957 if (!r) {
958 ret = -EBUSY;
959 goto out_free;
960 }
961
962 irs_base = ioremap(irs->config_base_address, ACPI_GICV5_IRS_MEM_SIZE);
963 if (!irs_base) {
964 pr_err("Unable to map GIC IRS registers\n");
965 ret = -ENOMEM;
966 goto out_release;
967 }
968
969 gicv5_irs_init_bases(irs_data, irs_base, irs->flags & ACPI_MADT_IRS_NON_COHERENT);
970 irs_data->res = *r;
971
972 gicv5_irs_acpi_init_affinity(irs->irs_id, irs_data);
973
974 ret = gicv5_irs_init(irs_data);
975 if (ret)
976 goto out_map;
977
978 if (irs_data->spi_range) {
979 pr_info("%s @%llx detected SPI range [%u-%u]\n", "IRS", irs->config_base_address,
980 irs_data->spi_min,
981 irs_data->spi_min +
982 irs_data->spi_range - 1);
983 }
984
985 return 0;
986
987 out_map:
988 gicv5_irs_clear_affinity(irs_data);
989 gicv5_irs_disable(irs_data);
990 iounmap(irs_base);
991 out_release:
992 release_mem_region(r->start, resource_size(r));
993 out_free:
994 kfree(irs_data);
995 return ret;
996 }
997
gicv5_irs_acpi_probe(void)998 int __init gicv5_irs_acpi_probe(void)
999 {
1000 acpi_table_parse_madt(ACPI_MADT_TYPE_GICV5_IRS, gic_acpi_parse_madt_irs, 0);
1001
1002 return list_empty(&irs_nodes) ? -ENODEV : 0;
1003 }
1004 #endif
1005