1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2024-2025 ARM Limited, All Rights Reserved.
4 */
5
6 #define pr_fmt(fmt) "GICv5 IRS: " fmt
7
8 #include <linux/kmemleak.h>
9 #include <linux/log2.h>
10 #include <linux/of.h>
11 #include <linux/of_address.h>
12
13 #include <linux/irqchip.h>
14 #include <linux/irqchip/arm-gic-v5.h>
15
16 /*
17 * Hardcoded ID_BITS limit for systems supporting only a 1-level IST
18 * table. Systems supporting only a 1-level IST table aren't expected
19 * to require more than 2^12 LPIs. Tweak as required.
20 */
21 #define LPI_ID_BITS_LINEAR 12
22
23 #define IRS_FLAGS_NON_COHERENT BIT(0)
24
25 static DEFINE_PER_CPU_READ_MOSTLY(struct gicv5_irs_chip_data *, per_cpu_irs_data);
26 static LIST_HEAD(irs_nodes);
27
irs_readl_relaxed(struct gicv5_irs_chip_data * irs_data,const u32 reg_offset)28 static u32 irs_readl_relaxed(struct gicv5_irs_chip_data *irs_data,
29 const u32 reg_offset)
30 {
31 return readl_relaxed(irs_data->irs_base + reg_offset);
32 }
33
irs_writel_relaxed(struct gicv5_irs_chip_data * irs_data,const u32 val,const u32 reg_offset)34 static void irs_writel_relaxed(struct gicv5_irs_chip_data *irs_data,
35 const u32 val, const u32 reg_offset)
36 {
37 writel_relaxed(val, irs_data->irs_base + reg_offset);
38 }
39
irs_readq_relaxed(struct gicv5_irs_chip_data * irs_data,const u32 reg_offset)40 static u64 irs_readq_relaxed(struct gicv5_irs_chip_data *irs_data,
41 const u32 reg_offset)
42 {
43 return readq_relaxed(irs_data->irs_base + reg_offset);
44 }
45
irs_writeq_relaxed(struct gicv5_irs_chip_data * irs_data,const u64 val,const u32 reg_offset)46 static void irs_writeq_relaxed(struct gicv5_irs_chip_data *irs_data,
47 const u64 val, const u32 reg_offset)
48 {
49 writeq_relaxed(val, irs_data->irs_base + reg_offset);
50 }
51
52 /*
53 * The polling wait (in gicv5_wait_for_op_s_atomic()) on a GIC register
54 * provides the memory barriers (through MMIO accessors)
55 * required to synchronize CPU and GIC access to IST memory.
56 */
gicv5_irs_ist_synchronise(struct gicv5_irs_chip_data * irs_data)57 static int gicv5_irs_ist_synchronise(struct gicv5_irs_chip_data *irs_data)
58 {
59 return gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_IST_STATUSR,
60 GICV5_IRS_IST_STATUSR_IDLE, NULL);
61 }
62
gicv5_irs_init_ist_linear(struct gicv5_irs_chip_data * irs_data,unsigned int lpi_id_bits,unsigned int istsz)63 static int __init gicv5_irs_init_ist_linear(struct gicv5_irs_chip_data *irs_data,
64 unsigned int lpi_id_bits,
65 unsigned int istsz)
66 {
67 size_t l2istsz;
68 u32 n, cfgr;
69 void *ist;
70 u64 baser;
71 int ret;
72
73 /* Taken from GICv5 specifications 10.2.1.13 IRS_IST_BASER */
74 n = max(5, lpi_id_bits + 1 + istsz);
75
76 l2istsz = BIT(n + 1);
77 /*
78 * Check memory requirements. For a linear IST we cap the
79 * number of ID bits to a value that should never exceed
80 * kmalloc interface memory allocation limits, so this
81 * check is really belt and braces.
82 */
83 if (l2istsz > KMALLOC_MAX_SIZE) {
84 u8 lpi_id_cap = ilog2(KMALLOC_MAX_SIZE) - 2 + istsz;
85
86 pr_warn("Limiting LPI ID bits from %u to %u\n",
87 lpi_id_bits, lpi_id_cap);
88 lpi_id_bits = lpi_id_cap;
89 l2istsz = KMALLOC_MAX_SIZE;
90 }
91
92 ist = kzalloc(l2istsz, GFP_KERNEL);
93 if (!ist)
94 return -ENOMEM;
95
96 if (irs_data->flags & IRS_FLAGS_NON_COHERENT)
97 dcache_clean_inval_poc((unsigned long)ist,
98 (unsigned long)ist + l2istsz);
99 else
100 dsb(ishst);
101
102 cfgr = FIELD_PREP(GICV5_IRS_IST_CFGR_STRUCTURE,
103 GICV5_IRS_IST_CFGR_STRUCTURE_LINEAR) |
104 FIELD_PREP(GICV5_IRS_IST_CFGR_ISTSZ, istsz) |
105 FIELD_PREP(GICV5_IRS_IST_CFGR_L2SZ,
106 GICV5_IRS_IST_CFGR_L2SZ_4K) |
107 FIELD_PREP(GICV5_IRS_IST_CFGR_LPI_ID_BITS, lpi_id_bits);
108 irs_writel_relaxed(irs_data, cfgr, GICV5_IRS_IST_CFGR);
109
110 gicv5_global_data.ist.l2 = false;
111
112 baser = (virt_to_phys(ist) & GICV5_IRS_IST_BASER_ADDR_MASK) |
113 FIELD_PREP(GICV5_IRS_IST_BASER_VALID, 0x1);
114 irs_writeq_relaxed(irs_data, baser, GICV5_IRS_IST_BASER);
115
116 ret = gicv5_irs_ist_synchronise(irs_data);
117 if (ret) {
118 kfree(ist);
119 return ret;
120 }
121 kmemleak_ignore(ist);
122
123 return 0;
124 }
125
gicv5_irs_init_ist_two_level(struct gicv5_irs_chip_data * irs_data,unsigned int lpi_id_bits,unsigned int istsz,unsigned int l2sz)126 static int __init gicv5_irs_init_ist_two_level(struct gicv5_irs_chip_data *irs_data,
127 unsigned int lpi_id_bits,
128 unsigned int istsz,
129 unsigned int l2sz)
130 {
131 __le64 *l1ist;
132 u32 cfgr, n;
133 size_t l1sz;
134 u64 baser;
135 int ret;
136
137 /* Taken from GICv5 specifications 10.2.1.13 IRS_IST_BASER */
138 n = max(5, lpi_id_bits - ((10 - istsz) + (2 * l2sz)) + 2);
139
140 l1sz = BIT(n + 1);
141
142 l1ist = kzalloc(l1sz, GFP_KERNEL);
143 if (!l1ist)
144 return -ENOMEM;
145
146 if (irs_data->flags & IRS_FLAGS_NON_COHERENT)
147 dcache_clean_inval_poc((unsigned long)l1ist,
148 (unsigned long)l1ist + l1sz);
149 else
150 dsb(ishst);
151
152 cfgr = FIELD_PREP(GICV5_IRS_IST_CFGR_STRUCTURE,
153 GICV5_IRS_IST_CFGR_STRUCTURE_TWO_LEVEL) |
154 FIELD_PREP(GICV5_IRS_IST_CFGR_ISTSZ, istsz) |
155 FIELD_PREP(GICV5_IRS_IST_CFGR_L2SZ, l2sz) |
156 FIELD_PREP(GICV5_IRS_IST_CFGR_LPI_ID_BITS, lpi_id_bits);
157 irs_writel_relaxed(irs_data, cfgr, GICV5_IRS_IST_CFGR);
158
159 /*
160 * The L2SZ determine bits required at L2 level. Number of bytes
161 * required by metadata is reported through istsz - the number of bits
162 * covered by L2 entries scales accordingly.
163 */
164 gicv5_global_data.ist.l2_size = BIT(11 + (2 * l2sz) + 1);
165 gicv5_global_data.ist.l2_bits = (10 - istsz) + (2 * l2sz);
166 gicv5_global_data.ist.l1ist_addr = l1ist;
167 gicv5_global_data.ist.l2 = true;
168
169 baser = (virt_to_phys(l1ist) & GICV5_IRS_IST_BASER_ADDR_MASK) |
170 FIELD_PREP(GICV5_IRS_IST_BASER_VALID, 0x1);
171 irs_writeq_relaxed(irs_data, baser, GICV5_IRS_IST_BASER);
172
173 ret = gicv5_irs_ist_synchronise(irs_data);
174 if (ret) {
175 kfree(l1ist);
176 return ret;
177 }
178
179 return 0;
180 }
181
182 /*
183 * Alloc L2 IST entries on demand.
184 *
185 * Locking/serialization is guaranteed by irqdomain core code by
186 * taking the hierarchical domain struct irq_domain.root->mutex.
187 */
gicv5_irs_iste_alloc(const u32 lpi)188 int gicv5_irs_iste_alloc(const u32 lpi)
189 {
190 struct gicv5_irs_chip_data *irs_data;
191 unsigned int index;
192 u32 l2istr, l2bits;
193 __le64 *l1ist;
194 size_t l2size;
195 void *l2ist;
196 int ret;
197
198 if (!gicv5_global_data.ist.l2)
199 return 0;
200
201 irs_data = per_cpu(per_cpu_irs_data, smp_processor_id());
202 if (!irs_data)
203 return -ENOENT;
204
205 l2size = gicv5_global_data.ist.l2_size;
206 l2bits = gicv5_global_data.ist.l2_bits;
207 l1ist = gicv5_global_data.ist.l1ist_addr;
208 index = lpi >> l2bits;
209
210 if (FIELD_GET(GICV5_ISTL1E_VALID, le64_to_cpu(l1ist[index])))
211 return 0;
212
213 l2ist = kzalloc(l2size, GFP_KERNEL);
214 if (!l2ist)
215 return -ENOMEM;
216
217 l1ist[index] = cpu_to_le64(virt_to_phys(l2ist) & GICV5_ISTL1E_L2_ADDR_MASK);
218
219 if (irs_data->flags & IRS_FLAGS_NON_COHERENT) {
220 dcache_clean_inval_poc((unsigned long)l2ist,
221 (unsigned long)l2ist + l2size);
222 dcache_clean_poc((unsigned long)(l1ist + index),
223 (unsigned long)(l1ist + index) + sizeof(*l1ist));
224 } else {
225 dsb(ishst);
226 }
227
228 l2istr = FIELD_PREP(GICV5_IRS_MAP_L2_ISTR_ID, lpi);
229 irs_writel_relaxed(irs_data, l2istr, GICV5_IRS_MAP_L2_ISTR);
230
231 ret = gicv5_irs_ist_synchronise(irs_data);
232 if (ret) {
233 l1ist[index] = 0;
234 kfree(l2ist);
235 return ret;
236 }
237 kmemleak_ignore(l2ist);
238
239 /*
240 * Make sure we invalidate the cache line pulled before the IRS
241 * had a chance to update the L1 entry and mark it valid.
242 */
243 if (irs_data->flags & IRS_FLAGS_NON_COHERENT) {
244 /*
245 * gicv5_irs_ist_synchronise() includes memory
246 * barriers (MMIO accessors) required to guarantee that the
247 * following dcache invalidation is not executed before the
248 * IST mapping operation has completed.
249 */
250 dcache_inval_poc((unsigned long)(l1ist + index),
251 (unsigned long)(l1ist + index) + sizeof(*l1ist));
252 }
253
254 return 0;
255 }
256
257 /*
258 * Try to match the L2 IST size to the pagesize, and if this is not possible
259 * pick the smallest supported L2 size in order to minimise the requirement for
260 * physically contiguous blocks of memory as page-sized allocations are
261 * guaranteed to be physically contiguous, and are by definition the easiest to
262 * find.
263 *
264 * Fall back to the smallest supported size (in the event that the pagesize
265 * itself is not supported) again serves to make it easier to find physically
266 * contiguous blocks of memory.
267 */
gicv5_irs_l2_sz(u32 idr2)268 static unsigned int gicv5_irs_l2_sz(u32 idr2)
269 {
270 switch (PAGE_SIZE) {
271 case SZ_64K:
272 if (GICV5_IRS_IST_L2SZ_SUPPORT_64KB(idr2))
273 return GICV5_IRS_IST_CFGR_L2SZ_64K;
274 fallthrough;
275 case SZ_4K:
276 if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(idr2))
277 return GICV5_IRS_IST_CFGR_L2SZ_4K;
278 fallthrough;
279 case SZ_16K:
280 if (GICV5_IRS_IST_L2SZ_SUPPORT_16KB(idr2))
281 return GICV5_IRS_IST_CFGR_L2SZ_16K;
282 break;
283 }
284
285 if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(idr2))
286 return GICV5_IRS_IST_CFGR_L2SZ_4K;
287
288 return GICV5_IRS_IST_CFGR_L2SZ_64K;
289 }
290
gicv5_irs_init_ist(struct gicv5_irs_chip_data * irs_data)291 static int __init gicv5_irs_init_ist(struct gicv5_irs_chip_data *irs_data)
292 {
293 u32 lpi_id_bits, idr2_id_bits, idr2_min_lpi_id_bits, l2_iste_sz, l2sz;
294 u32 l2_iste_sz_split, idr2;
295 bool two_levels, istmd;
296 u64 baser;
297 int ret;
298
299 baser = irs_readq_relaxed(irs_data, GICV5_IRS_IST_BASER);
300 if (FIELD_GET(GICV5_IRS_IST_BASER_VALID, baser)) {
301 pr_err("IST is marked as valid already; cannot allocate\n");
302 return -EPERM;
303 }
304
305 idr2 = irs_readl_relaxed(irs_data, GICV5_IRS_IDR2);
306 two_levels = !!FIELD_GET(GICV5_IRS_IDR2_IST_LEVELS, idr2);
307
308 idr2_id_bits = FIELD_GET(GICV5_IRS_IDR2_ID_BITS, idr2);
309 idr2_min_lpi_id_bits = FIELD_GET(GICV5_IRS_IDR2_MIN_LPI_ID_BITS, idr2);
310
311 /*
312 * For two level tables we are always supporting the maximum allowed
313 * number of IDs.
314 *
315 * For 1-level tables, we should support a number of bits that
316 * is >= min_lpi_id_bits but cap it to LPI_ID_BITS_LINEAR lest
317 * the level 1-table gets too large and its memory allocation
318 * may fail.
319 */
320 if (two_levels) {
321 lpi_id_bits = idr2_id_bits;
322 } else {
323 lpi_id_bits = max(LPI_ID_BITS_LINEAR, idr2_min_lpi_id_bits);
324 lpi_id_bits = min(lpi_id_bits, idr2_id_bits);
325 }
326
327 /*
328 * Cap the ID bits according to the CPUIF supported ID bits
329 */
330 lpi_id_bits = min(lpi_id_bits, gicv5_global_data.cpuif_id_bits);
331
332 if (two_levels)
333 l2sz = gicv5_irs_l2_sz(idr2);
334
335 istmd = !!FIELD_GET(GICV5_IRS_IDR2_ISTMD, idr2);
336
337 l2_iste_sz = GICV5_IRS_IST_CFGR_ISTSZ_4;
338
339 if (istmd) {
340 l2_iste_sz_split = FIELD_GET(GICV5_IRS_IDR2_ISTMD_SZ, idr2);
341
342 if (lpi_id_bits < l2_iste_sz_split)
343 l2_iste_sz = GICV5_IRS_IST_CFGR_ISTSZ_8;
344 else
345 l2_iste_sz = GICV5_IRS_IST_CFGR_ISTSZ_16;
346 }
347
348 /*
349 * Follow GICv5 specification recommendation to opt in for two
350 * level tables (ref: 10.2.1.14 IRS_IST_CFGR).
351 */
352 if (two_levels && (lpi_id_bits > ((10 - l2_iste_sz) + (2 * l2sz)))) {
353 ret = gicv5_irs_init_ist_two_level(irs_data, lpi_id_bits,
354 l2_iste_sz, l2sz);
355 } else {
356 ret = gicv5_irs_init_ist_linear(irs_data, lpi_id_bits,
357 l2_iste_sz);
358 }
359 if (ret)
360 return ret;
361
362 gicv5_init_lpis(BIT(lpi_id_bits));
363
364 return 0;
365 }
366
367 struct iaffid_entry {
368 u16 iaffid;
369 bool valid;
370 };
371
372 static DEFINE_PER_CPU(struct iaffid_entry, cpu_iaffid);
373
gicv5_irs_cpu_to_iaffid(int cpuid,u16 * iaffid)374 int gicv5_irs_cpu_to_iaffid(int cpuid, u16 *iaffid)
375 {
376 if (!per_cpu(cpu_iaffid, cpuid).valid) {
377 pr_err("IAFFID for CPU %d has not been initialised\n", cpuid);
378 return -ENODEV;
379 }
380
381 *iaffid = per_cpu(cpu_iaffid, cpuid).iaffid;
382
383 return 0;
384 }
385
gicv5_irs_lookup_by_spi_id(u32 spi_id)386 struct gicv5_irs_chip_data *gicv5_irs_lookup_by_spi_id(u32 spi_id)
387 {
388 struct gicv5_irs_chip_data *irs_data;
389 u32 min, max;
390
391 list_for_each_entry(irs_data, &irs_nodes, entry) {
392 if (!irs_data->spi_range)
393 continue;
394
395 min = irs_data->spi_min;
396 max = irs_data->spi_min + irs_data->spi_range - 1;
397 if (spi_id >= min && spi_id <= max)
398 return irs_data;
399 }
400
401 return NULL;
402 }
403
gicv5_irs_wait_for_spi_op(struct gicv5_irs_chip_data * irs_data)404 static int gicv5_irs_wait_for_spi_op(struct gicv5_irs_chip_data *irs_data)
405 {
406 u32 statusr;
407 int ret;
408
409 ret = gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_SPI_STATUSR,
410 GICV5_IRS_SPI_STATUSR_IDLE, &statusr);
411 if (ret)
412 return ret;
413
414 return !!FIELD_GET(GICV5_IRS_SPI_STATUSR_V, statusr) ? 0 : -EIO;
415 }
416
gicv5_irs_wait_for_irs_pe(struct gicv5_irs_chip_data * irs_data,bool selr)417 static int gicv5_irs_wait_for_irs_pe(struct gicv5_irs_chip_data *irs_data,
418 bool selr)
419 {
420 bool valid = true;
421 u32 statusr;
422 int ret;
423
424 ret = gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_PE_STATUSR,
425 GICV5_IRS_PE_STATUSR_IDLE, &statusr);
426 if (ret)
427 return ret;
428
429 if (selr)
430 valid = !!FIELD_GET(GICV5_IRS_PE_STATUSR_V, statusr);
431
432 return valid ? 0 : -EIO;
433 }
434
gicv5_irs_wait_for_pe_selr(struct gicv5_irs_chip_data * irs_data)435 static int gicv5_irs_wait_for_pe_selr(struct gicv5_irs_chip_data *irs_data)
436 {
437 return gicv5_irs_wait_for_irs_pe(irs_data, true);
438 }
439
gicv5_irs_wait_for_pe_cr0(struct gicv5_irs_chip_data * irs_data)440 static int gicv5_irs_wait_for_pe_cr0(struct gicv5_irs_chip_data *irs_data)
441 {
442 return gicv5_irs_wait_for_irs_pe(irs_data, false);
443 }
444
gicv5_spi_irq_set_type(struct irq_data * d,unsigned int type)445 int gicv5_spi_irq_set_type(struct irq_data *d, unsigned int type)
446 {
447 struct gicv5_irs_chip_data *irs_data = d->chip_data;
448 u32 selr, cfgr;
449 bool level;
450 int ret;
451
452 /*
453 * There is no distinction between HIGH/LOW for level IRQs
454 * and RISING/FALLING for edge IRQs in the architecture,
455 * hence consider them equivalent.
456 */
457 switch (type) {
458 case IRQ_TYPE_EDGE_RISING:
459 case IRQ_TYPE_EDGE_FALLING:
460 level = false;
461 break;
462 case IRQ_TYPE_LEVEL_HIGH:
463 case IRQ_TYPE_LEVEL_LOW:
464 level = true;
465 break;
466 default:
467 return -EINVAL;
468 }
469
470 guard(raw_spinlock)(&irs_data->spi_config_lock);
471
472 selr = FIELD_PREP(GICV5_IRS_SPI_SELR_ID, d->hwirq);
473 irs_writel_relaxed(irs_data, selr, GICV5_IRS_SPI_SELR);
474 ret = gicv5_irs_wait_for_spi_op(irs_data);
475 if (ret)
476 return ret;
477
478 cfgr = FIELD_PREP(GICV5_IRS_SPI_CFGR_TM, level);
479 irs_writel_relaxed(irs_data, cfgr, GICV5_IRS_SPI_CFGR);
480
481 return gicv5_irs_wait_for_spi_op(irs_data);
482 }
483
gicv5_irs_wait_for_idle(struct gicv5_irs_chip_data * irs_data)484 static int gicv5_irs_wait_for_idle(struct gicv5_irs_chip_data *irs_data)
485 {
486 return gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_CR0,
487 GICV5_IRS_CR0_IDLE, NULL);
488 }
489
gicv5_irs_syncr(void)490 void gicv5_irs_syncr(void)
491 {
492 struct gicv5_irs_chip_data *irs_data;
493 u32 syncr;
494
495 irs_data = list_first_entry_or_null(&irs_nodes, struct gicv5_irs_chip_data, entry);
496 if (WARN_ON_ONCE(!irs_data))
497 return;
498
499 syncr = FIELD_PREP(GICV5_IRS_SYNCR_SYNC, 1);
500 irs_writel_relaxed(irs_data, syncr, GICV5_IRS_SYNCR);
501
502 gicv5_wait_for_op(irs_data->irs_base, GICV5_IRS_SYNC_STATUSR,
503 GICV5_IRS_SYNC_STATUSR_IDLE);
504 }
505
gicv5_irs_register_cpu(int cpuid)506 int gicv5_irs_register_cpu(int cpuid)
507 {
508 struct gicv5_irs_chip_data *irs_data;
509 u32 selr, cr0;
510 u16 iaffid;
511 int ret;
512
513 ret = gicv5_irs_cpu_to_iaffid(cpuid, &iaffid);
514 if (ret) {
515 pr_err("IAFFID for CPU %d has not been initialised\n", cpuid);
516 return ret;
517 }
518
519 irs_data = per_cpu(per_cpu_irs_data, cpuid);
520 if (!irs_data) {
521 pr_err("No IRS associated with CPU %u\n", cpuid);
522 return -ENXIO;
523 }
524
525 selr = FIELD_PREP(GICV5_IRS_PE_SELR_IAFFID, iaffid);
526 irs_writel_relaxed(irs_data, selr, GICV5_IRS_PE_SELR);
527
528 ret = gicv5_irs_wait_for_pe_selr(irs_data);
529 if (ret) {
530 pr_err("IAFFID 0x%x used in IRS_PE_SELR is invalid\n", iaffid);
531 return -ENXIO;
532 }
533
534 cr0 = FIELD_PREP(GICV5_IRS_PE_CR0_DPS, 0x1);
535 irs_writel_relaxed(irs_data, cr0, GICV5_IRS_PE_CR0);
536
537 ret = gicv5_irs_wait_for_pe_cr0(irs_data);
538 if (ret)
539 return ret;
540
541 pr_debug("CPU %d enabled PE IAFFID 0x%x\n", cpuid, iaffid);
542
543 return 0;
544 }
545
gicv5_irs_init_bases(struct gicv5_irs_chip_data * irs_data,void __iomem * irs_base,struct fwnode_handle * handle)546 static void __init gicv5_irs_init_bases(struct gicv5_irs_chip_data *irs_data,
547 void __iomem *irs_base,
548 struct fwnode_handle *handle)
549 {
550 struct device_node *np = to_of_node(handle);
551 u32 cr0, cr1;
552
553 irs_data->fwnode = handle;
554 irs_data->irs_base = irs_base;
555
556 if (of_property_read_bool(np, "dma-noncoherent")) {
557 /*
558 * A non-coherent IRS implies that some cache levels cannot be
559 * used coherently by the cores and GIC. Our only option is to mark
560 * memory attributes for the GIC as non-cacheable; by default,
561 * non-cacheable memory attributes imply outer-shareable
562 * shareability, the value written into IRS_CR1_SH is ignored.
563 */
564 cr1 = FIELD_PREP(GICV5_IRS_CR1_VPED_WA, GICV5_NO_WRITE_ALLOC) |
565 FIELD_PREP(GICV5_IRS_CR1_VPED_RA, GICV5_NO_READ_ALLOC) |
566 FIELD_PREP(GICV5_IRS_CR1_VMD_WA, GICV5_NO_WRITE_ALLOC) |
567 FIELD_PREP(GICV5_IRS_CR1_VMD_RA, GICV5_NO_READ_ALLOC) |
568 FIELD_PREP(GICV5_IRS_CR1_VPET_RA, GICV5_NO_READ_ALLOC) |
569 FIELD_PREP(GICV5_IRS_CR1_VMT_RA, GICV5_NO_READ_ALLOC) |
570 FIELD_PREP(GICV5_IRS_CR1_IST_WA, GICV5_NO_WRITE_ALLOC) |
571 FIELD_PREP(GICV5_IRS_CR1_IST_RA, GICV5_NO_READ_ALLOC) |
572 FIELD_PREP(GICV5_IRS_CR1_IC, GICV5_NON_CACHE) |
573 FIELD_PREP(GICV5_IRS_CR1_OC, GICV5_NON_CACHE);
574 irs_data->flags |= IRS_FLAGS_NON_COHERENT;
575 } else {
576 cr1 = FIELD_PREP(GICV5_IRS_CR1_VPED_WA, GICV5_WRITE_ALLOC) |
577 FIELD_PREP(GICV5_IRS_CR1_VPED_RA, GICV5_READ_ALLOC) |
578 FIELD_PREP(GICV5_IRS_CR1_VMD_WA, GICV5_WRITE_ALLOC) |
579 FIELD_PREP(GICV5_IRS_CR1_VMD_RA, GICV5_READ_ALLOC) |
580 FIELD_PREP(GICV5_IRS_CR1_VPET_RA, GICV5_READ_ALLOC) |
581 FIELD_PREP(GICV5_IRS_CR1_VMT_RA, GICV5_READ_ALLOC) |
582 FIELD_PREP(GICV5_IRS_CR1_IST_WA, GICV5_WRITE_ALLOC) |
583 FIELD_PREP(GICV5_IRS_CR1_IST_RA, GICV5_READ_ALLOC) |
584 FIELD_PREP(GICV5_IRS_CR1_IC, GICV5_WB_CACHE) |
585 FIELD_PREP(GICV5_IRS_CR1_OC, GICV5_WB_CACHE) |
586 FIELD_PREP(GICV5_IRS_CR1_SH, GICV5_INNER_SHARE);
587 }
588
589 irs_writel_relaxed(irs_data, cr1, GICV5_IRS_CR1);
590
591 cr0 = FIELD_PREP(GICV5_IRS_CR0_IRSEN, 0x1);
592 irs_writel_relaxed(irs_data, cr0, GICV5_IRS_CR0);
593 gicv5_irs_wait_for_idle(irs_data);
594 }
595
gicv5_irs_of_init_affinity(struct device_node * node,struct gicv5_irs_chip_data * irs_data,u8 iaffid_bits)596 static int __init gicv5_irs_of_init_affinity(struct device_node *node,
597 struct gicv5_irs_chip_data *irs_data,
598 u8 iaffid_bits)
599 {
600 /*
601 * Detect IAFFID<->CPU mappings from the device tree and
602 * record IRS<->CPU topology information.
603 */
604 u16 iaffid_mask = GENMASK(iaffid_bits - 1, 0);
605 int ret, i, ncpus, niaffids;
606
607 ncpus = of_count_phandle_with_args(node, "cpus", NULL);
608 if (ncpus < 0)
609 return -EINVAL;
610
611 niaffids = of_property_count_elems_of_size(node, "arm,iaffids",
612 sizeof(u16));
613 if (niaffids != ncpus)
614 return -EINVAL;
615
616 u16 *iaffids __free(kfree) = kcalloc(niaffids, sizeof(*iaffids), GFP_KERNEL);
617 if (!iaffids)
618 return -ENOMEM;
619
620 ret = of_property_read_u16_array(node, "arm,iaffids", iaffids, niaffids);
621 if (ret)
622 return ret;
623
624 for (i = 0; i < ncpus; i++) {
625 struct device_node *cpu_node;
626 int cpu;
627
628 cpu_node = of_parse_phandle(node, "cpus", i);
629 if (!cpu_node) {
630 pr_warn(FW_BUG "Erroneous CPU node phandle\n");
631 continue;
632 }
633
634 cpu = of_cpu_node_to_id(cpu_node);
635 of_node_put(cpu_node);
636 if (cpu < 0)
637 continue;
638
639 if (iaffids[i] & ~iaffid_mask) {
640 pr_warn("CPU %d iaffid 0x%x exceeds IRS iaffid bits\n",
641 cpu, iaffids[i]);
642 continue;
643 }
644
645 per_cpu(cpu_iaffid, cpu).iaffid = iaffids[i];
646 per_cpu(cpu_iaffid, cpu).valid = true;
647
648 /* We also know that the CPU is connected to this IRS */
649 per_cpu(per_cpu_irs_data, cpu) = irs_data;
650 }
651
652 return ret;
653 }
654
irs_setup_pri_bits(u32 idr1)655 static void irs_setup_pri_bits(u32 idr1)
656 {
657 switch (FIELD_GET(GICV5_IRS_IDR1_PRIORITY_BITS, idr1)) {
658 case GICV5_IRS_IDR1_PRIORITY_BITS_1BITS:
659 gicv5_global_data.irs_pri_bits = 1;
660 break;
661 case GICV5_IRS_IDR1_PRIORITY_BITS_2BITS:
662 gicv5_global_data.irs_pri_bits = 2;
663 break;
664 case GICV5_IRS_IDR1_PRIORITY_BITS_3BITS:
665 gicv5_global_data.irs_pri_bits = 3;
666 break;
667 case GICV5_IRS_IDR1_PRIORITY_BITS_4BITS:
668 gicv5_global_data.irs_pri_bits = 4;
669 break;
670 case GICV5_IRS_IDR1_PRIORITY_BITS_5BITS:
671 gicv5_global_data.irs_pri_bits = 5;
672 break;
673 default:
674 pr_warn("Detected wrong IDR priority bits value 0x%lx\n",
675 FIELD_GET(GICV5_IRS_IDR1_PRIORITY_BITS, idr1));
676 gicv5_global_data.irs_pri_bits = 1;
677 break;
678 }
679 }
680
gicv5_irs_init(struct device_node * node)681 static int __init gicv5_irs_init(struct device_node *node)
682 {
683 struct gicv5_irs_chip_data *irs_data;
684 void __iomem *irs_base;
685 u32 idr, spi_count;
686 u8 iaffid_bits;
687 int ret;
688
689 irs_data = kzalloc(sizeof(*irs_data), GFP_KERNEL);
690 if (!irs_data)
691 return -ENOMEM;
692
693 raw_spin_lock_init(&irs_data->spi_config_lock);
694
695 ret = of_property_match_string(node, "reg-names", "ns-config");
696 if (ret < 0) {
697 pr_err("%pOF: ns-config reg-name not present\n", node);
698 goto out_err;
699 }
700
701 irs_base = of_io_request_and_map(node, ret, of_node_full_name(node));
702 if (IS_ERR(irs_base)) {
703 pr_err("%pOF: unable to map GICv5 IRS registers\n", node);
704 ret = PTR_ERR(irs_base);
705 goto out_err;
706 }
707
708 gicv5_irs_init_bases(irs_data, irs_base, &node->fwnode);
709
710 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR1);
711 iaffid_bits = FIELD_GET(GICV5_IRS_IDR1_IAFFID_BITS, idr) + 1;
712
713 ret = gicv5_irs_of_init_affinity(node, irs_data, iaffid_bits);
714 if (ret) {
715 pr_err("Failed to parse CPU IAFFIDs from the device tree!\n");
716 goto out_iomem;
717 }
718
719 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR2);
720 if (WARN(!FIELD_GET(GICV5_IRS_IDR2_LPI, idr),
721 "LPI support not available - no IPIs, can't proceed\n")) {
722 ret = -ENODEV;
723 goto out_iomem;
724 }
725
726 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR7);
727 irs_data->spi_min = FIELD_GET(GICV5_IRS_IDR7_SPI_BASE, idr);
728
729 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR6);
730 irs_data->spi_range = FIELD_GET(GICV5_IRS_IDR6_SPI_IRS_RANGE, idr);
731
732 if (irs_data->spi_range) {
733 pr_info("%s detected SPI range [%u-%u]\n",
734 of_node_full_name(node),
735 irs_data->spi_min,
736 irs_data->spi_min +
737 irs_data->spi_range - 1);
738 }
739
740 /*
741 * Do the global setting only on the first IRS.
742 * Global properties (iaffid_bits, global spi count) are guaranteed to
743 * be consistent across IRSes by the architecture.
744 */
745 if (list_empty(&irs_nodes)) {
746
747 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR1);
748 irs_setup_pri_bits(idr);
749
750 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR5);
751
752 spi_count = FIELD_GET(GICV5_IRS_IDR5_SPI_RANGE, idr);
753 gicv5_global_data.global_spi_count = spi_count;
754
755 gicv5_init_lpi_domain();
756
757 pr_debug("Detected %u SPIs globally\n", spi_count);
758 }
759
760 list_add_tail(&irs_data->entry, &irs_nodes);
761
762 return 0;
763
764 out_iomem:
765 iounmap(irs_base);
766 out_err:
767 kfree(irs_data);
768 return ret;
769 }
770
gicv5_irs_remove(void)771 void __init gicv5_irs_remove(void)
772 {
773 struct gicv5_irs_chip_data *irs_data, *tmp_data;
774
775 gicv5_free_lpi_domain();
776 gicv5_deinit_lpis();
777
778 list_for_each_entry_safe(irs_data, tmp_data, &irs_nodes, entry) {
779 iounmap(irs_data->irs_base);
780 list_del(&irs_data->entry);
781 kfree(irs_data);
782 }
783 }
784
gicv5_irs_enable(void)785 int __init gicv5_irs_enable(void)
786 {
787 struct gicv5_irs_chip_data *irs_data;
788 int ret;
789
790 irs_data = list_first_entry_or_null(&irs_nodes,
791 struct gicv5_irs_chip_data, entry);
792 if (!irs_data)
793 return -ENODEV;
794
795 ret = gicv5_irs_init_ist(irs_data);
796 if (ret) {
797 pr_err("Failed to init IST\n");
798 return ret;
799 }
800
801 return 0;
802 }
803
gicv5_irs_its_probe(void)804 void __init gicv5_irs_its_probe(void)
805 {
806 struct gicv5_irs_chip_data *irs_data;
807
808 list_for_each_entry(irs_data, &irs_nodes, entry)
809 gicv5_its_of_probe(to_of_node(irs_data->fwnode));
810 }
811
gicv5_irs_of_probe(struct device_node * parent)812 int __init gicv5_irs_of_probe(struct device_node *parent)
813 {
814 struct device_node *np;
815 int ret;
816
817 for_each_available_child_of_node(parent, np) {
818 if (!of_device_is_compatible(np, "arm,gic-v5-irs"))
819 continue;
820
821 ret = gicv5_irs_init(np);
822 if (ret)
823 pr_err("Failed to init IRS %s\n", np->full_name);
824 }
825
826 return list_empty(&irs_nodes) ? -ENODEV : 0;
827 }
828