1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2024-2025 ARM Limited, All Rights Reserved. 4 */ 5 6 #define pr_fmt(fmt) "GICv5 IRS: " fmt 7 8 #include <linux/kmemleak.h> 9 #include <linux/log2.h> 10 #include <linux/of.h> 11 #include <linux/of_address.h> 12 13 #include <linux/irqchip.h> 14 #include <linux/irqchip/arm-gic-v5.h> 15 16 /* 17 * Hardcoded ID_BITS limit for systems supporting only a 1-level IST 18 * table. Systems supporting only a 1-level IST table aren't expected 19 * to require more than 2^12 LPIs. Tweak as required. 20 */ 21 #define LPI_ID_BITS_LINEAR 12 22 23 #define IRS_FLAGS_NON_COHERENT BIT(0) 24 25 static DEFINE_PER_CPU_READ_MOSTLY(struct gicv5_irs_chip_data *, per_cpu_irs_data); 26 static LIST_HEAD(irs_nodes); 27 28 static u32 irs_readl_relaxed(struct gicv5_irs_chip_data *irs_data, 29 const u32 reg_offset) 30 { 31 return readl_relaxed(irs_data->irs_base + reg_offset); 32 } 33 34 static void irs_writel_relaxed(struct gicv5_irs_chip_data *irs_data, 35 const u32 val, const u32 reg_offset) 36 { 37 writel_relaxed(val, irs_data->irs_base + reg_offset); 38 } 39 40 static u64 irs_readq_relaxed(struct gicv5_irs_chip_data *irs_data, 41 const u32 reg_offset) 42 { 43 return readq_relaxed(irs_data->irs_base + reg_offset); 44 } 45 46 static void irs_writeq_relaxed(struct gicv5_irs_chip_data *irs_data, 47 const u64 val, const u32 reg_offset) 48 { 49 writeq_relaxed(val, irs_data->irs_base + reg_offset); 50 } 51 52 /* 53 * The polling wait (in gicv5_wait_for_op_s_atomic()) on a GIC register 54 * provides the memory barriers (through MMIO accessors) 55 * required to synchronize CPU and GIC access to IST memory. 56 */ 57 static int gicv5_irs_ist_synchronise(struct gicv5_irs_chip_data *irs_data) 58 { 59 return gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_IST_STATUSR, 60 GICV5_IRS_IST_STATUSR_IDLE, NULL); 61 } 62 63 static int __init gicv5_irs_init_ist_linear(struct gicv5_irs_chip_data *irs_data, 64 unsigned int lpi_id_bits, 65 unsigned int istsz) 66 { 67 size_t l2istsz; 68 u32 n, cfgr; 69 void *ist; 70 u64 baser; 71 int ret; 72 73 /* Taken from GICv5 specifications 10.2.1.13 IRS_IST_BASER */ 74 n = max(5, lpi_id_bits + 1 + istsz); 75 76 l2istsz = BIT(n + 1); 77 /* 78 * Check memory requirements. For a linear IST we cap the 79 * number of ID bits to a value that should never exceed 80 * kmalloc interface memory allocation limits, so this 81 * check is really belt and braces. 82 */ 83 if (l2istsz > KMALLOC_MAX_SIZE) { 84 u8 lpi_id_cap = ilog2(KMALLOC_MAX_SIZE) - 2 + istsz; 85 86 pr_warn("Limiting LPI ID bits from %u to %u\n", 87 lpi_id_bits, lpi_id_cap); 88 lpi_id_bits = lpi_id_cap; 89 l2istsz = KMALLOC_MAX_SIZE; 90 } 91 92 ist = kzalloc(l2istsz, GFP_KERNEL); 93 if (!ist) 94 return -ENOMEM; 95 96 if (irs_data->flags & IRS_FLAGS_NON_COHERENT) 97 dcache_clean_inval_poc((unsigned long)ist, 98 (unsigned long)ist + l2istsz); 99 else 100 dsb(ishst); 101 102 cfgr = FIELD_PREP(GICV5_IRS_IST_CFGR_STRUCTURE, 103 GICV5_IRS_IST_CFGR_STRUCTURE_LINEAR) | 104 FIELD_PREP(GICV5_IRS_IST_CFGR_ISTSZ, istsz) | 105 FIELD_PREP(GICV5_IRS_IST_CFGR_L2SZ, 106 GICV5_IRS_IST_CFGR_L2SZ_4K) | 107 FIELD_PREP(GICV5_IRS_IST_CFGR_LPI_ID_BITS, lpi_id_bits); 108 irs_writel_relaxed(irs_data, cfgr, GICV5_IRS_IST_CFGR); 109 110 gicv5_global_data.ist.l2 = false; 111 112 baser = (virt_to_phys(ist) & GICV5_IRS_IST_BASER_ADDR_MASK) | 113 FIELD_PREP(GICV5_IRS_IST_BASER_VALID, 0x1); 114 irs_writeq_relaxed(irs_data, baser, GICV5_IRS_IST_BASER); 115 116 ret = gicv5_irs_ist_synchronise(irs_data); 117 if (ret) { 118 kfree(ist); 119 return ret; 120 } 121 kmemleak_ignore(ist); 122 123 return 0; 124 } 125 126 static int __init gicv5_irs_init_ist_two_level(struct gicv5_irs_chip_data *irs_data, 127 unsigned int lpi_id_bits, 128 unsigned int istsz, 129 unsigned int l2sz) 130 { 131 __le64 *l1ist; 132 u32 cfgr, n; 133 size_t l1sz; 134 u64 baser; 135 int ret; 136 137 /* Taken from GICv5 specifications 10.2.1.13 IRS_IST_BASER */ 138 n = max(5, lpi_id_bits - ((10 - istsz) + (2 * l2sz)) + 2); 139 140 l1sz = BIT(n + 1); 141 142 l1ist = kzalloc(l1sz, GFP_KERNEL); 143 if (!l1ist) 144 return -ENOMEM; 145 146 if (irs_data->flags & IRS_FLAGS_NON_COHERENT) 147 dcache_clean_inval_poc((unsigned long)l1ist, 148 (unsigned long)l1ist + l1sz); 149 else 150 dsb(ishst); 151 152 cfgr = FIELD_PREP(GICV5_IRS_IST_CFGR_STRUCTURE, 153 GICV5_IRS_IST_CFGR_STRUCTURE_TWO_LEVEL) | 154 FIELD_PREP(GICV5_IRS_IST_CFGR_ISTSZ, istsz) | 155 FIELD_PREP(GICV5_IRS_IST_CFGR_L2SZ, l2sz) | 156 FIELD_PREP(GICV5_IRS_IST_CFGR_LPI_ID_BITS, lpi_id_bits); 157 irs_writel_relaxed(irs_data, cfgr, GICV5_IRS_IST_CFGR); 158 159 /* 160 * The L2SZ determine bits required at L2 level. Number of bytes 161 * required by metadata is reported through istsz - the number of bits 162 * covered by L2 entries scales accordingly. 163 */ 164 gicv5_global_data.ist.l2_size = BIT(11 + (2 * l2sz) + 1); 165 gicv5_global_data.ist.l2_bits = (10 - istsz) + (2 * l2sz); 166 gicv5_global_data.ist.l1ist_addr = l1ist; 167 gicv5_global_data.ist.l2 = true; 168 169 baser = (virt_to_phys(l1ist) & GICV5_IRS_IST_BASER_ADDR_MASK) | 170 FIELD_PREP(GICV5_IRS_IST_BASER_VALID, 0x1); 171 irs_writeq_relaxed(irs_data, baser, GICV5_IRS_IST_BASER); 172 173 ret = gicv5_irs_ist_synchronise(irs_data); 174 if (ret) { 175 kfree(l1ist); 176 return ret; 177 } 178 179 return 0; 180 } 181 182 /* 183 * Alloc L2 IST entries on demand. 184 * 185 * Locking/serialization is guaranteed by irqdomain core code by 186 * taking the hierarchical domain struct irq_domain.root->mutex. 187 */ 188 int gicv5_irs_iste_alloc(const u32 lpi) 189 { 190 struct gicv5_irs_chip_data *irs_data; 191 unsigned int index; 192 u32 l2istr, l2bits; 193 __le64 *l1ist; 194 size_t l2size; 195 void *l2ist; 196 int ret; 197 198 if (!gicv5_global_data.ist.l2) 199 return 0; 200 201 irs_data = per_cpu(per_cpu_irs_data, smp_processor_id()); 202 if (!irs_data) 203 return -ENOENT; 204 205 l2size = gicv5_global_data.ist.l2_size; 206 l2bits = gicv5_global_data.ist.l2_bits; 207 l1ist = gicv5_global_data.ist.l1ist_addr; 208 index = lpi >> l2bits; 209 210 if (FIELD_GET(GICV5_ISTL1E_VALID, le64_to_cpu(l1ist[index]))) 211 return 0; 212 213 l2ist = kzalloc(l2size, GFP_KERNEL); 214 if (!l2ist) 215 return -ENOMEM; 216 217 l1ist[index] = cpu_to_le64(virt_to_phys(l2ist) & GICV5_ISTL1E_L2_ADDR_MASK); 218 219 if (irs_data->flags & IRS_FLAGS_NON_COHERENT) { 220 dcache_clean_inval_poc((unsigned long)l2ist, 221 (unsigned long)l2ist + l2size); 222 dcache_clean_poc((unsigned long)(l1ist + index), 223 (unsigned long)(l1ist + index) + sizeof(*l1ist)); 224 } else { 225 dsb(ishst); 226 } 227 228 l2istr = FIELD_PREP(GICV5_IRS_MAP_L2_ISTR_ID, lpi); 229 irs_writel_relaxed(irs_data, l2istr, GICV5_IRS_MAP_L2_ISTR); 230 231 ret = gicv5_irs_ist_synchronise(irs_data); 232 if (ret) { 233 l1ist[index] = 0; 234 kfree(l2ist); 235 return ret; 236 } 237 kmemleak_ignore(l2ist); 238 239 /* 240 * Make sure we invalidate the cache line pulled before the IRS 241 * had a chance to update the L1 entry and mark it valid. 242 */ 243 if (irs_data->flags & IRS_FLAGS_NON_COHERENT) { 244 /* 245 * gicv5_irs_ist_synchronise() includes memory 246 * barriers (MMIO accessors) required to guarantee that the 247 * following dcache invalidation is not executed before the 248 * IST mapping operation has completed. 249 */ 250 dcache_inval_poc((unsigned long)(l1ist + index), 251 (unsigned long)(l1ist + index) + sizeof(*l1ist)); 252 } 253 254 return 0; 255 } 256 257 /* 258 * Try to match the L2 IST size to the pagesize, and if this is not possible 259 * pick the smallest supported L2 size in order to minimise the requirement for 260 * physically contiguous blocks of memory as page-sized allocations are 261 * guaranteed to be physically contiguous, and are by definition the easiest to 262 * find. 263 * 264 * Fall back to the smallest supported size (in the event that the pagesize 265 * itself is not supported) again serves to make it easier to find physically 266 * contiguous blocks of memory. 267 */ 268 static unsigned int gicv5_irs_l2_sz(u32 idr2) 269 { 270 switch (PAGE_SIZE) { 271 case SZ_64K: 272 if (GICV5_IRS_IST_L2SZ_SUPPORT_64KB(idr2)) 273 return GICV5_IRS_IST_CFGR_L2SZ_64K; 274 fallthrough; 275 case SZ_4K: 276 if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(idr2)) 277 return GICV5_IRS_IST_CFGR_L2SZ_4K; 278 fallthrough; 279 case SZ_16K: 280 if (GICV5_IRS_IST_L2SZ_SUPPORT_16KB(idr2)) 281 return GICV5_IRS_IST_CFGR_L2SZ_16K; 282 break; 283 } 284 285 if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(idr2)) 286 return GICV5_IRS_IST_CFGR_L2SZ_4K; 287 288 return GICV5_IRS_IST_CFGR_L2SZ_64K; 289 } 290 291 static int __init gicv5_irs_init_ist(struct gicv5_irs_chip_data *irs_data) 292 { 293 u32 lpi_id_bits, idr2_id_bits, idr2_min_lpi_id_bits, l2_iste_sz, l2sz; 294 u32 l2_iste_sz_split, idr2; 295 bool two_levels, istmd; 296 u64 baser; 297 int ret; 298 299 baser = irs_readq_relaxed(irs_data, GICV5_IRS_IST_BASER); 300 if (FIELD_GET(GICV5_IRS_IST_BASER_VALID, baser)) { 301 pr_err("IST is marked as valid already; cannot allocate\n"); 302 return -EPERM; 303 } 304 305 idr2 = irs_readl_relaxed(irs_data, GICV5_IRS_IDR2); 306 two_levels = !!FIELD_GET(GICV5_IRS_IDR2_IST_LEVELS, idr2); 307 308 idr2_id_bits = FIELD_GET(GICV5_IRS_IDR2_ID_BITS, idr2); 309 idr2_min_lpi_id_bits = FIELD_GET(GICV5_IRS_IDR2_MIN_LPI_ID_BITS, idr2); 310 311 /* 312 * For two level tables we are always supporting the maximum allowed 313 * number of IDs. 314 * 315 * For 1-level tables, we should support a number of bits that 316 * is >= min_lpi_id_bits but cap it to LPI_ID_BITS_LINEAR lest 317 * the level 1-table gets too large and its memory allocation 318 * may fail. 319 */ 320 if (two_levels) { 321 lpi_id_bits = idr2_id_bits; 322 } else { 323 lpi_id_bits = max(LPI_ID_BITS_LINEAR, idr2_min_lpi_id_bits); 324 lpi_id_bits = min(lpi_id_bits, idr2_id_bits); 325 } 326 327 /* 328 * Cap the ID bits according to the CPUIF supported ID bits 329 */ 330 lpi_id_bits = min(lpi_id_bits, gicv5_global_data.cpuif_id_bits); 331 332 if (two_levels) 333 l2sz = gicv5_irs_l2_sz(idr2); 334 335 istmd = !!FIELD_GET(GICV5_IRS_IDR2_ISTMD, idr2); 336 337 l2_iste_sz = GICV5_IRS_IST_CFGR_ISTSZ_4; 338 339 if (istmd) { 340 l2_iste_sz_split = FIELD_GET(GICV5_IRS_IDR2_ISTMD_SZ, idr2); 341 342 if (lpi_id_bits < l2_iste_sz_split) 343 l2_iste_sz = GICV5_IRS_IST_CFGR_ISTSZ_8; 344 else 345 l2_iste_sz = GICV5_IRS_IST_CFGR_ISTSZ_16; 346 } 347 348 /* 349 * Follow GICv5 specification recommendation to opt in for two 350 * level tables (ref: 10.2.1.14 IRS_IST_CFGR). 351 */ 352 if (two_levels && (lpi_id_bits > ((10 - l2_iste_sz) + (2 * l2sz)))) { 353 ret = gicv5_irs_init_ist_two_level(irs_data, lpi_id_bits, 354 l2_iste_sz, l2sz); 355 } else { 356 ret = gicv5_irs_init_ist_linear(irs_data, lpi_id_bits, 357 l2_iste_sz); 358 } 359 if (ret) 360 return ret; 361 362 gicv5_init_lpis(BIT(lpi_id_bits)); 363 364 return 0; 365 } 366 367 struct iaffid_entry { 368 u16 iaffid; 369 bool valid; 370 }; 371 372 static DEFINE_PER_CPU(struct iaffid_entry, cpu_iaffid); 373 374 int gicv5_irs_cpu_to_iaffid(int cpuid, u16 *iaffid) 375 { 376 if (!per_cpu(cpu_iaffid, cpuid).valid) { 377 pr_err("IAFFID for CPU %d has not been initialised\n", cpuid); 378 return -ENODEV; 379 } 380 381 *iaffid = per_cpu(cpu_iaffid, cpuid).iaffid; 382 383 return 0; 384 } 385 386 struct gicv5_irs_chip_data *gicv5_irs_lookup_by_spi_id(u32 spi_id) 387 { 388 struct gicv5_irs_chip_data *irs_data; 389 u32 min, max; 390 391 list_for_each_entry(irs_data, &irs_nodes, entry) { 392 if (!irs_data->spi_range) 393 continue; 394 395 min = irs_data->spi_min; 396 max = irs_data->spi_min + irs_data->spi_range - 1; 397 if (spi_id >= min && spi_id <= max) 398 return irs_data; 399 } 400 401 return NULL; 402 } 403 404 static int gicv5_irs_wait_for_spi_op(struct gicv5_irs_chip_data *irs_data) 405 { 406 u32 statusr; 407 int ret; 408 409 ret = gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_SPI_STATUSR, 410 GICV5_IRS_SPI_STATUSR_IDLE, &statusr); 411 if (ret) 412 return ret; 413 414 return !!FIELD_GET(GICV5_IRS_SPI_STATUSR_V, statusr) ? 0 : -EIO; 415 } 416 417 static int gicv5_irs_wait_for_irs_pe(struct gicv5_irs_chip_data *irs_data, 418 bool selr) 419 { 420 bool valid = true; 421 u32 statusr; 422 int ret; 423 424 ret = gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_PE_STATUSR, 425 GICV5_IRS_PE_STATUSR_IDLE, &statusr); 426 if (ret) 427 return ret; 428 429 if (selr) 430 valid = !!FIELD_GET(GICV5_IRS_PE_STATUSR_V, statusr); 431 432 return valid ? 0 : -EIO; 433 } 434 435 static int gicv5_irs_wait_for_pe_selr(struct gicv5_irs_chip_data *irs_data) 436 { 437 return gicv5_irs_wait_for_irs_pe(irs_data, true); 438 } 439 440 static int gicv5_irs_wait_for_pe_cr0(struct gicv5_irs_chip_data *irs_data) 441 { 442 return gicv5_irs_wait_for_irs_pe(irs_data, false); 443 } 444 445 int gicv5_spi_irq_set_type(struct irq_data *d, unsigned int type) 446 { 447 struct gicv5_irs_chip_data *irs_data = d->chip_data; 448 u32 selr, cfgr; 449 bool level; 450 int ret; 451 452 /* 453 * There is no distinction between HIGH/LOW for level IRQs 454 * and RISING/FALLING for edge IRQs in the architecture, 455 * hence consider them equivalent. 456 */ 457 switch (type) { 458 case IRQ_TYPE_EDGE_RISING: 459 case IRQ_TYPE_EDGE_FALLING: 460 level = false; 461 break; 462 case IRQ_TYPE_LEVEL_HIGH: 463 case IRQ_TYPE_LEVEL_LOW: 464 level = true; 465 break; 466 default: 467 return -EINVAL; 468 } 469 470 guard(raw_spinlock)(&irs_data->spi_config_lock); 471 472 selr = FIELD_PREP(GICV5_IRS_SPI_SELR_ID, d->hwirq); 473 irs_writel_relaxed(irs_data, selr, GICV5_IRS_SPI_SELR); 474 ret = gicv5_irs_wait_for_spi_op(irs_data); 475 if (ret) 476 return ret; 477 478 cfgr = FIELD_PREP(GICV5_IRS_SPI_CFGR_TM, level); 479 irs_writel_relaxed(irs_data, cfgr, GICV5_IRS_SPI_CFGR); 480 481 return gicv5_irs_wait_for_spi_op(irs_data); 482 } 483 484 static int gicv5_irs_wait_for_idle(struct gicv5_irs_chip_data *irs_data) 485 { 486 return gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_CR0, 487 GICV5_IRS_CR0_IDLE, NULL); 488 } 489 490 void gicv5_irs_syncr(void) 491 { 492 struct gicv5_irs_chip_data *irs_data; 493 u32 syncr; 494 495 irs_data = list_first_entry_or_null(&irs_nodes, struct gicv5_irs_chip_data, entry); 496 if (WARN_ON_ONCE(!irs_data)) 497 return; 498 499 syncr = FIELD_PREP(GICV5_IRS_SYNCR_SYNC, 1); 500 irs_writel_relaxed(irs_data, syncr, GICV5_IRS_SYNCR); 501 502 gicv5_wait_for_op(irs_data->irs_base, GICV5_IRS_SYNC_STATUSR, 503 GICV5_IRS_SYNC_STATUSR_IDLE); 504 } 505 506 int gicv5_irs_register_cpu(int cpuid) 507 { 508 struct gicv5_irs_chip_data *irs_data; 509 u32 selr, cr0; 510 u16 iaffid; 511 int ret; 512 513 ret = gicv5_irs_cpu_to_iaffid(cpuid, &iaffid); 514 if (ret) { 515 pr_err("IAFFID for CPU %d has not been initialised\n", cpuid); 516 return ret; 517 } 518 519 irs_data = per_cpu(per_cpu_irs_data, cpuid); 520 if (!irs_data) { 521 pr_err("No IRS associated with CPU %u\n", cpuid); 522 return -ENXIO; 523 } 524 525 selr = FIELD_PREP(GICV5_IRS_PE_SELR_IAFFID, iaffid); 526 irs_writel_relaxed(irs_data, selr, GICV5_IRS_PE_SELR); 527 528 ret = gicv5_irs_wait_for_pe_selr(irs_data); 529 if (ret) { 530 pr_err("IAFFID 0x%x used in IRS_PE_SELR is invalid\n", iaffid); 531 return -ENXIO; 532 } 533 534 cr0 = FIELD_PREP(GICV5_IRS_PE_CR0_DPS, 0x1); 535 irs_writel_relaxed(irs_data, cr0, GICV5_IRS_PE_CR0); 536 537 ret = gicv5_irs_wait_for_pe_cr0(irs_data); 538 if (ret) 539 return ret; 540 541 pr_debug("CPU %d enabled PE IAFFID 0x%x\n", cpuid, iaffid); 542 543 return 0; 544 } 545 546 static void __init gicv5_irs_init_bases(struct gicv5_irs_chip_data *irs_data, 547 void __iomem *irs_base, 548 struct fwnode_handle *handle) 549 { 550 struct device_node *np = to_of_node(handle); 551 u32 cr0, cr1; 552 553 irs_data->fwnode = handle; 554 irs_data->irs_base = irs_base; 555 556 if (of_property_read_bool(np, "dma-noncoherent")) { 557 /* 558 * A non-coherent IRS implies that some cache levels cannot be 559 * used coherently by the cores and GIC. Our only option is to mark 560 * memory attributes for the GIC as non-cacheable; by default, 561 * non-cacheable memory attributes imply outer-shareable 562 * shareability, the value written into IRS_CR1_SH is ignored. 563 */ 564 cr1 = FIELD_PREP(GICV5_IRS_CR1_VPED_WA, GICV5_NO_WRITE_ALLOC) | 565 FIELD_PREP(GICV5_IRS_CR1_VPED_RA, GICV5_NO_READ_ALLOC) | 566 FIELD_PREP(GICV5_IRS_CR1_VMD_WA, GICV5_NO_WRITE_ALLOC) | 567 FIELD_PREP(GICV5_IRS_CR1_VMD_RA, GICV5_NO_READ_ALLOC) | 568 FIELD_PREP(GICV5_IRS_CR1_VPET_RA, GICV5_NO_READ_ALLOC) | 569 FIELD_PREP(GICV5_IRS_CR1_VMT_RA, GICV5_NO_READ_ALLOC) | 570 FIELD_PREP(GICV5_IRS_CR1_IST_WA, GICV5_NO_WRITE_ALLOC) | 571 FIELD_PREP(GICV5_IRS_CR1_IST_RA, GICV5_NO_READ_ALLOC) | 572 FIELD_PREP(GICV5_IRS_CR1_IC, GICV5_NON_CACHE) | 573 FIELD_PREP(GICV5_IRS_CR1_OC, GICV5_NON_CACHE); 574 irs_data->flags |= IRS_FLAGS_NON_COHERENT; 575 } else { 576 cr1 = FIELD_PREP(GICV5_IRS_CR1_VPED_WA, GICV5_WRITE_ALLOC) | 577 FIELD_PREP(GICV5_IRS_CR1_VPED_RA, GICV5_READ_ALLOC) | 578 FIELD_PREP(GICV5_IRS_CR1_VMD_WA, GICV5_WRITE_ALLOC) | 579 FIELD_PREP(GICV5_IRS_CR1_VMD_RA, GICV5_READ_ALLOC) | 580 FIELD_PREP(GICV5_IRS_CR1_VPET_RA, GICV5_READ_ALLOC) | 581 FIELD_PREP(GICV5_IRS_CR1_VMT_RA, GICV5_READ_ALLOC) | 582 FIELD_PREP(GICV5_IRS_CR1_IST_WA, GICV5_WRITE_ALLOC) | 583 FIELD_PREP(GICV5_IRS_CR1_IST_RA, GICV5_READ_ALLOC) | 584 FIELD_PREP(GICV5_IRS_CR1_IC, GICV5_WB_CACHE) | 585 FIELD_PREP(GICV5_IRS_CR1_OC, GICV5_WB_CACHE) | 586 FIELD_PREP(GICV5_IRS_CR1_SH, GICV5_INNER_SHARE); 587 } 588 589 irs_writel_relaxed(irs_data, cr1, GICV5_IRS_CR1); 590 591 cr0 = FIELD_PREP(GICV5_IRS_CR0_IRSEN, 0x1); 592 irs_writel_relaxed(irs_data, cr0, GICV5_IRS_CR0); 593 gicv5_irs_wait_for_idle(irs_data); 594 } 595 596 static int __init gicv5_irs_of_init_affinity(struct device_node *node, 597 struct gicv5_irs_chip_data *irs_data, 598 u8 iaffid_bits) 599 { 600 /* 601 * Detect IAFFID<->CPU mappings from the device tree and 602 * record IRS<->CPU topology information. 603 */ 604 u16 iaffid_mask = GENMASK(iaffid_bits - 1, 0); 605 int ret, i, ncpus, niaffids; 606 607 ncpus = of_count_phandle_with_args(node, "cpus", NULL); 608 if (ncpus < 0) 609 return -EINVAL; 610 611 niaffids = of_property_count_elems_of_size(node, "arm,iaffids", 612 sizeof(u16)); 613 if (niaffids != ncpus) 614 return -EINVAL; 615 616 u16 *iaffids __free(kfree) = kcalloc(niaffids, sizeof(*iaffids), GFP_KERNEL); 617 if (!iaffids) 618 return -ENOMEM; 619 620 ret = of_property_read_u16_array(node, "arm,iaffids", iaffids, niaffids); 621 if (ret) 622 return ret; 623 624 for (i = 0; i < ncpus; i++) { 625 struct device_node *cpu_node; 626 int cpu; 627 628 cpu_node = of_parse_phandle(node, "cpus", i); 629 if (!cpu_node) { 630 pr_warn(FW_BUG "Erroneous CPU node phandle\n"); 631 continue; 632 } 633 634 cpu = of_cpu_node_to_id(cpu_node); 635 of_node_put(cpu_node); 636 if (cpu < 0) 637 continue; 638 639 if (iaffids[i] & ~iaffid_mask) { 640 pr_warn("CPU %d iaffid 0x%x exceeds IRS iaffid bits\n", 641 cpu, iaffids[i]); 642 continue; 643 } 644 645 per_cpu(cpu_iaffid, cpu).iaffid = iaffids[i]; 646 per_cpu(cpu_iaffid, cpu).valid = true; 647 648 /* We also know that the CPU is connected to this IRS */ 649 per_cpu(per_cpu_irs_data, cpu) = irs_data; 650 } 651 652 return ret; 653 } 654 655 static void irs_setup_pri_bits(u32 idr1) 656 { 657 switch (FIELD_GET(GICV5_IRS_IDR1_PRIORITY_BITS, idr1)) { 658 case GICV5_IRS_IDR1_PRIORITY_BITS_1BITS: 659 gicv5_global_data.irs_pri_bits = 1; 660 break; 661 case GICV5_IRS_IDR1_PRIORITY_BITS_2BITS: 662 gicv5_global_data.irs_pri_bits = 2; 663 break; 664 case GICV5_IRS_IDR1_PRIORITY_BITS_3BITS: 665 gicv5_global_data.irs_pri_bits = 3; 666 break; 667 case GICV5_IRS_IDR1_PRIORITY_BITS_4BITS: 668 gicv5_global_data.irs_pri_bits = 4; 669 break; 670 case GICV5_IRS_IDR1_PRIORITY_BITS_5BITS: 671 gicv5_global_data.irs_pri_bits = 5; 672 break; 673 default: 674 pr_warn("Detected wrong IDR priority bits value 0x%lx\n", 675 FIELD_GET(GICV5_IRS_IDR1_PRIORITY_BITS, idr1)); 676 gicv5_global_data.irs_pri_bits = 1; 677 break; 678 } 679 } 680 681 static int __init gicv5_irs_init(struct device_node *node) 682 { 683 struct gicv5_irs_chip_data *irs_data; 684 void __iomem *irs_base; 685 u32 idr, spi_count; 686 u8 iaffid_bits; 687 int ret; 688 689 irs_data = kzalloc(sizeof(*irs_data), GFP_KERNEL); 690 if (!irs_data) 691 return -ENOMEM; 692 693 raw_spin_lock_init(&irs_data->spi_config_lock); 694 695 ret = of_property_match_string(node, "reg-names", "ns-config"); 696 if (ret < 0) { 697 pr_err("%pOF: ns-config reg-name not present\n", node); 698 goto out_err; 699 } 700 701 irs_base = of_io_request_and_map(node, ret, of_node_full_name(node)); 702 if (IS_ERR(irs_base)) { 703 pr_err("%pOF: unable to map GICv5 IRS registers\n", node); 704 ret = PTR_ERR(irs_base); 705 goto out_err; 706 } 707 708 gicv5_irs_init_bases(irs_data, irs_base, &node->fwnode); 709 710 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR1); 711 iaffid_bits = FIELD_GET(GICV5_IRS_IDR1_IAFFID_BITS, idr) + 1; 712 713 ret = gicv5_irs_of_init_affinity(node, irs_data, iaffid_bits); 714 if (ret) { 715 pr_err("Failed to parse CPU IAFFIDs from the device tree!\n"); 716 goto out_iomem; 717 } 718 719 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR2); 720 if (WARN(!FIELD_GET(GICV5_IRS_IDR2_LPI, idr), 721 "LPI support not available - no IPIs, can't proceed\n")) { 722 ret = -ENODEV; 723 goto out_iomem; 724 } 725 726 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR7); 727 irs_data->spi_min = FIELD_GET(GICV5_IRS_IDR7_SPI_BASE, idr); 728 729 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR6); 730 irs_data->spi_range = FIELD_GET(GICV5_IRS_IDR6_SPI_IRS_RANGE, idr); 731 732 if (irs_data->spi_range) { 733 pr_info("%s detected SPI range [%u-%u]\n", 734 of_node_full_name(node), 735 irs_data->spi_min, 736 irs_data->spi_min + 737 irs_data->spi_range - 1); 738 } 739 740 /* 741 * Do the global setting only on the first IRS. 742 * Global properties (iaffid_bits, global spi count) are guaranteed to 743 * be consistent across IRSes by the architecture. 744 */ 745 if (list_empty(&irs_nodes)) { 746 747 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR1); 748 irs_setup_pri_bits(idr); 749 750 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR5); 751 752 spi_count = FIELD_GET(GICV5_IRS_IDR5_SPI_RANGE, idr); 753 gicv5_global_data.global_spi_count = spi_count; 754 755 gicv5_init_lpi_domain(); 756 757 pr_debug("Detected %u SPIs globally\n", spi_count); 758 } 759 760 list_add_tail(&irs_data->entry, &irs_nodes); 761 762 return 0; 763 764 out_iomem: 765 iounmap(irs_base); 766 out_err: 767 kfree(irs_data); 768 return ret; 769 } 770 771 void __init gicv5_irs_remove(void) 772 { 773 struct gicv5_irs_chip_data *irs_data, *tmp_data; 774 775 gicv5_free_lpi_domain(); 776 gicv5_deinit_lpis(); 777 778 list_for_each_entry_safe(irs_data, tmp_data, &irs_nodes, entry) { 779 iounmap(irs_data->irs_base); 780 list_del(&irs_data->entry); 781 kfree(irs_data); 782 } 783 } 784 785 int __init gicv5_irs_enable(void) 786 { 787 struct gicv5_irs_chip_data *irs_data; 788 int ret; 789 790 irs_data = list_first_entry_or_null(&irs_nodes, 791 struct gicv5_irs_chip_data, entry); 792 if (!irs_data) 793 return -ENODEV; 794 795 ret = gicv5_irs_init_ist(irs_data); 796 if (ret) { 797 pr_err("Failed to init IST\n"); 798 return ret; 799 } 800 801 return 0; 802 } 803 804 void __init gicv5_irs_its_probe(void) 805 { 806 struct gicv5_irs_chip_data *irs_data; 807 808 list_for_each_entry(irs_data, &irs_nodes, entry) 809 gicv5_its_of_probe(to_of_node(irs_data->fwnode)); 810 } 811 812 int __init gicv5_irs_of_probe(struct device_node *parent) 813 { 814 struct device_node *np; 815 int ret; 816 817 for_each_available_child_of_node(parent, np) { 818 if (!of_device_is_compatible(np, "arm,gic-v5-irs")) 819 continue; 820 821 ret = gicv5_irs_init(np); 822 if (ret) 823 pr_err("Failed to init IRS %s\n", np->full_name); 824 } 825 826 return list_empty(&irs_nodes) ? -ENODEV : 0; 827 } 828