1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2024-2025 ARM Limited, All Rights Reserved. 4 */ 5 6 #define pr_fmt(fmt) "GICv5 IRS: " fmt 7 8 #include <linux/log2.h> 9 #include <linux/of.h> 10 #include <linux/of_address.h> 11 12 #include <linux/irqchip.h> 13 #include <linux/irqchip/arm-gic-v5.h> 14 15 /* 16 * Hardcoded ID_BITS limit for systems supporting only a 1-level IST 17 * table. Systems supporting only a 1-level IST table aren't expected 18 * to require more than 2^12 LPIs. Tweak as required. 19 */ 20 #define LPI_ID_BITS_LINEAR 12 21 22 #define IRS_FLAGS_NON_COHERENT BIT(0) 23 24 static DEFINE_PER_CPU_READ_MOSTLY(struct gicv5_irs_chip_data *, per_cpu_irs_data); 25 static LIST_HEAD(irs_nodes); 26 27 static u32 irs_readl_relaxed(struct gicv5_irs_chip_data *irs_data, 28 const u32 reg_offset) 29 { 30 return readl_relaxed(irs_data->irs_base + reg_offset); 31 } 32 33 static void irs_writel_relaxed(struct gicv5_irs_chip_data *irs_data, 34 const u32 val, const u32 reg_offset) 35 { 36 writel_relaxed(val, irs_data->irs_base + reg_offset); 37 } 38 39 static u64 irs_readq_relaxed(struct gicv5_irs_chip_data *irs_data, 40 const u32 reg_offset) 41 { 42 return readq_relaxed(irs_data->irs_base + reg_offset); 43 } 44 45 static void irs_writeq_relaxed(struct gicv5_irs_chip_data *irs_data, 46 const u64 val, const u32 reg_offset) 47 { 48 writeq_relaxed(val, irs_data->irs_base + reg_offset); 49 } 50 51 /* 52 * The polling wait (in gicv5_wait_for_op_s_atomic()) on a GIC register 53 * provides the memory barriers (through MMIO accessors) 54 * required to synchronize CPU and GIC access to IST memory. 55 */ 56 static int gicv5_irs_ist_synchronise(struct gicv5_irs_chip_data *irs_data) 57 { 58 return gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_IST_STATUSR, 59 GICV5_IRS_IST_STATUSR_IDLE, NULL); 60 } 61 62 static int __init gicv5_irs_init_ist_linear(struct gicv5_irs_chip_data *irs_data, 63 unsigned int lpi_id_bits, 64 unsigned int istsz) 65 { 66 size_t l2istsz; 67 u32 n, cfgr; 68 void *ist; 69 u64 baser; 70 int ret; 71 72 /* Taken from GICv5 specifications 10.2.1.13 IRS_IST_BASER */ 73 n = max(5, lpi_id_bits + 1 + istsz); 74 75 l2istsz = BIT(n + 1); 76 /* 77 * Check memory requirements. For a linear IST we cap the 78 * number of ID bits to a value that should never exceed 79 * kmalloc interface memory allocation limits, so this 80 * check is really belt and braces. 81 */ 82 if (l2istsz > KMALLOC_MAX_SIZE) { 83 u8 lpi_id_cap = ilog2(KMALLOC_MAX_SIZE) - 2 + istsz; 84 85 pr_warn("Limiting LPI ID bits from %u to %u\n", 86 lpi_id_bits, lpi_id_cap); 87 lpi_id_bits = lpi_id_cap; 88 l2istsz = KMALLOC_MAX_SIZE; 89 } 90 91 ist = kzalloc(l2istsz, GFP_KERNEL); 92 if (!ist) 93 return -ENOMEM; 94 95 if (irs_data->flags & IRS_FLAGS_NON_COHERENT) 96 dcache_clean_inval_poc((unsigned long)ist, 97 (unsigned long)ist + l2istsz); 98 else 99 dsb(ishst); 100 101 cfgr = FIELD_PREP(GICV5_IRS_IST_CFGR_STRUCTURE, 102 GICV5_IRS_IST_CFGR_STRUCTURE_LINEAR) | 103 FIELD_PREP(GICV5_IRS_IST_CFGR_ISTSZ, istsz) | 104 FIELD_PREP(GICV5_IRS_IST_CFGR_L2SZ, 105 GICV5_IRS_IST_CFGR_L2SZ_4K) | 106 FIELD_PREP(GICV5_IRS_IST_CFGR_LPI_ID_BITS, lpi_id_bits); 107 irs_writel_relaxed(irs_data, cfgr, GICV5_IRS_IST_CFGR); 108 109 gicv5_global_data.ist.l2 = false; 110 111 baser = (virt_to_phys(ist) & GICV5_IRS_IST_BASER_ADDR_MASK) | 112 FIELD_PREP(GICV5_IRS_IST_BASER_VALID, 0x1); 113 irs_writeq_relaxed(irs_data, baser, GICV5_IRS_IST_BASER); 114 115 ret = gicv5_irs_ist_synchronise(irs_data); 116 if (ret) { 117 kfree(ist); 118 return ret; 119 } 120 121 return 0; 122 } 123 124 static int __init gicv5_irs_init_ist_two_level(struct gicv5_irs_chip_data *irs_data, 125 unsigned int lpi_id_bits, 126 unsigned int istsz, 127 unsigned int l2sz) 128 { 129 __le64 *l1ist; 130 u32 cfgr, n; 131 size_t l1sz; 132 u64 baser; 133 int ret; 134 135 /* Taken from GICv5 specifications 10.2.1.13 IRS_IST_BASER */ 136 n = max(5, lpi_id_bits - ((10 - istsz) + (2 * l2sz)) + 2); 137 138 l1sz = BIT(n + 1); 139 140 l1ist = kzalloc(l1sz, GFP_KERNEL); 141 if (!l1ist) 142 return -ENOMEM; 143 144 if (irs_data->flags & IRS_FLAGS_NON_COHERENT) 145 dcache_clean_inval_poc((unsigned long)l1ist, 146 (unsigned long)l1ist + l1sz); 147 else 148 dsb(ishst); 149 150 cfgr = FIELD_PREP(GICV5_IRS_IST_CFGR_STRUCTURE, 151 GICV5_IRS_IST_CFGR_STRUCTURE_TWO_LEVEL) | 152 FIELD_PREP(GICV5_IRS_IST_CFGR_ISTSZ, istsz) | 153 FIELD_PREP(GICV5_IRS_IST_CFGR_L2SZ, l2sz) | 154 FIELD_PREP(GICV5_IRS_IST_CFGR_LPI_ID_BITS, lpi_id_bits); 155 irs_writel_relaxed(irs_data, cfgr, GICV5_IRS_IST_CFGR); 156 157 /* 158 * The L2SZ determine bits required at L2 level. Number of bytes 159 * required by metadata is reported through istsz - the number of bits 160 * covered by L2 entries scales accordingly. 161 */ 162 gicv5_global_data.ist.l2_size = BIT(11 + (2 * l2sz) + 1); 163 gicv5_global_data.ist.l2_bits = (10 - istsz) + (2 * l2sz); 164 gicv5_global_data.ist.l1ist_addr = l1ist; 165 gicv5_global_data.ist.l2 = true; 166 167 baser = (virt_to_phys(l1ist) & GICV5_IRS_IST_BASER_ADDR_MASK) | 168 FIELD_PREP(GICV5_IRS_IST_BASER_VALID, 0x1); 169 irs_writeq_relaxed(irs_data, baser, GICV5_IRS_IST_BASER); 170 171 ret = gicv5_irs_ist_synchronise(irs_data); 172 if (ret) { 173 kfree(l1ist); 174 return ret; 175 } 176 177 return 0; 178 } 179 180 /* 181 * Alloc L2 IST entries on demand. 182 * 183 * Locking/serialization is guaranteed by irqdomain core code by 184 * taking the hierarchical domain struct irq_domain.root->mutex. 185 */ 186 int gicv5_irs_iste_alloc(const u32 lpi) 187 { 188 struct gicv5_irs_chip_data *irs_data; 189 unsigned int index; 190 u32 l2istr, l2bits; 191 __le64 *l1ist; 192 size_t l2size; 193 void *l2ist; 194 int ret; 195 196 if (!gicv5_global_data.ist.l2) 197 return 0; 198 199 irs_data = per_cpu(per_cpu_irs_data, smp_processor_id()); 200 if (!irs_data) 201 return -ENOENT; 202 203 l2size = gicv5_global_data.ist.l2_size; 204 l2bits = gicv5_global_data.ist.l2_bits; 205 l1ist = gicv5_global_data.ist.l1ist_addr; 206 index = lpi >> l2bits; 207 208 if (FIELD_GET(GICV5_ISTL1E_VALID, le64_to_cpu(l1ist[index]))) 209 return 0; 210 211 l2ist = kzalloc(l2size, GFP_KERNEL); 212 if (!l2ist) 213 return -ENOMEM; 214 215 l1ist[index] = cpu_to_le64(virt_to_phys(l2ist) & GICV5_ISTL1E_L2_ADDR_MASK); 216 217 if (irs_data->flags & IRS_FLAGS_NON_COHERENT) { 218 dcache_clean_inval_poc((unsigned long)l2ist, 219 (unsigned long)l2ist + l2size); 220 dcache_clean_poc((unsigned long)(l1ist + index), 221 (unsigned long)(l1ist + index) + sizeof(*l1ist)); 222 } else { 223 dsb(ishst); 224 } 225 226 l2istr = FIELD_PREP(GICV5_IRS_MAP_L2_ISTR_ID, lpi); 227 irs_writel_relaxed(irs_data, l2istr, GICV5_IRS_MAP_L2_ISTR); 228 229 ret = gicv5_irs_ist_synchronise(irs_data); 230 if (ret) { 231 l1ist[index] = 0; 232 kfree(l2ist); 233 return ret; 234 } 235 236 /* 237 * Make sure we invalidate the cache line pulled before the IRS 238 * had a chance to update the L1 entry and mark it valid. 239 */ 240 if (irs_data->flags & IRS_FLAGS_NON_COHERENT) { 241 /* 242 * gicv5_irs_ist_synchronise() includes memory 243 * barriers (MMIO accessors) required to guarantee that the 244 * following dcache invalidation is not executed before the 245 * IST mapping operation has completed. 246 */ 247 dcache_inval_poc((unsigned long)(l1ist + index), 248 (unsigned long)(l1ist + index) + sizeof(*l1ist)); 249 } 250 251 return 0; 252 } 253 254 /* 255 * Try to match the L2 IST size to the pagesize, and if this is not possible 256 * pick the smallest supported L2 size in order to minimise the requirement for 257 * physically contiguous blocks of memory as page-sized allocations are 258 * guaranteed to be physically contiguous, and are by definition the easiest to 259 * find. 260 * 261 * Fall back to the smallest supported size (in the event that the pagesize 262 * itself is not supported) again serves to make it easier to find physically 263 * contiguous blocks of memory. 264 */ 265 static unsigned int gicv5_irs_l2_sz(u32 idr2) 266 { 267 switch (PAGE_SIZE) { 268 case SZ_64K: 269 if (GICV5_IRS_IST_L2SZ_SUPPORT_64KB(idr2)) 270 return GICV5_IRS_IST_CFGR_L2SZ_64K; 271 fallthrough; 272 case SZ_4K: 273 if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(idr2)) 274 return GICV5_IRS_IST_CFGR_L2SZ_4K; 275 fallthrough; 276 case SZ_16K: 277 if (GICV5_IRS_IST_L2SZ_SUPPORT_16KB(idr2)) 278 return GICV5_IRS_IST_CFGR_L2SZ_16K; 279 break; 280 } 281 282 if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(idr2)) 283 return GICV5_IRS_IST_CFGR_L2SZ_4K; 284 285 return GICV5_IRS_IST_CFGR_L2SZ_64K; 286 } 287 288 static int __init gicv5_irs_init_ist(struct gicv5_irs_chip_data *irs_data) 289 { 290 u32 lpi_id_bits, idr2_id_bits, idr2_min_lpi_id_bits, l2_iste_sz, l2sz; 291 u32 l2_iste_sz_split, idr2; 292 bool two_levels, istmd; 293 u64 baser; 294 int ret; 295 296 baser = irs_readq_relaxed(irs_data, GICV5_IRS_IST_BASER); 297 if (FIELD_GET(GICV5_IRS_IST_BASER_VALID, baser)) { 298 pr_err("IST is marked as valid already; cannot allocate\n"); 299 return -EPERM; 300 } 301 302 idr2 = irs_readl_relaxed(irs_data, GICV5_IRS_IDR2); 303 two_levels = !!FIELD_GET(GICV5_IRS_IDR2_IST_LEVELS, idr2); 304 305 idr2_id_bits = FIELD_GET(GICV5_IRS_IDR2_ID_BITS, idr2); 306 idr2_min_lpi_id_bits = FIELD_GET(GICV5_IRS_IDR2_MIN_LPI_ID_BITS, idr2); 307 308 /* 309 * For two level tables we are always supporting the maximum allowed 310 * number of IDs. 311 * 312 * For 1-level tables, we should support a number of bits that 313 * is >= min_lpi_id_bits but cap it to LPI_ID_BITS_LINEAR lest 314 * the level 1-table gets too large and its memory allocation 315 * may fail. 316 */ 317 if (two_levels) { 318 lpi_id_bits = idr2_id_bits; 319 } else { 320 lpi_id_bits = max(LPI_ID_BITS_LINEAR, idr2_min_lpi_id_bits); 321 lpi_id_bits = min(lpi_id_bits, idr2_id_bits); 322 } 323 324 /* 325 * Cap the ID bits according to the CPUIF supported ID bits 326 */ 327 lpi_id_bits = min(lpi_id_bits, gicv5_global_data.cpuif_id_bits); 328 329 if (two_levels) 330 l2sz = gicv5_irs_l2_sz(idr2); 331 332 istmd = !!FIELD_GET(GICV5_IRS_IDR2_ISTMD, idr2); 333 334 l2_iste_sz = GICV5_IRS_IST_CFGR_ISTSZ_4; 335 336 if (istmd) { 337 l2_iste_sz_split = FIELD_GET(GICV5_IRS_IDR2_ISTMD_SZ, idr2); 338 339 if (lpi_id_bits < l2_iste_sz_split) 340 l2_iste_sz = GICV5_IRS_IST_CFGR_ISTSZ_8; 341 else 342 l2_iste_sz = GICV5_IRS_IST_CFGR_ISTSZ_16; 343 } 344 345 /* 346 * Follow GICv5 specification recommendation to opt in for two 347 * level tables (ref: 10.2.1.14 IRS_IST_CFGR). 348 */ 349 if (two_levels && (lpi_id_bits > ((10 - l2_iste_sz) + (2 * l2sz)))) { 350 ret = gicv5_irs_init_ist_two_level(irs_data, lpi_id_bits, 351 l2_iste_sz, l2sz); 352 } else { 353 ret = gicv5_irs_init_ist_linear(irs_data, lpi_id_bits, 354 l2_iste_sz); 355 } 356 if (ret) 357 return ret; 358 359 gicv5_init_lpis(BIT(lpi_id_bits)); 360 361 return 0; 362 } 363 364 struct iaffid_entry { 365 u16 iaffid; 366 bool valid; 367 }; 368 369 static DEFINE_PER_CPU(struct iaffid_entry, cpu_iaffid); 370 371 int gicv5_irs_cpu_to_iaffid(int cpuid, u16 *iaffid) 372 { 373 if (!per_cpu(cpu_iaffid, cpuid).valid) { 374 pr_err("IAFFID for CPU %d has not been initialised\n", cpuid); 375 return -ENODEV; 376 } 377 378 *iaffid = per_cpu(cpu_iaffid, cpuid).iaffid; 379 380 return 0; 381 } 382 383 struct gicv5_irs_chip_data *gicv5_irs_lookup_by_spi_id(u32 spi_id) 384 { 385 struct gicv5_irs_chip_data *irs_data; 386 u32 min, max; 387 388 list_for_each_entry(irs_data, &irs_nodes, entry) { 389 if (!irs_data->spi_range) 390 continue; 391 392 min = irs_data->spi_min; 393 max = irs_data->spi_min + irs_data->spi_range - 1; 394 if (spi_id >= min && spi_id <= max) 395 return irs_data; 396 } 397 398 return NULL; 399 } 400 401 static int gicv5_irs_wait_for_spi_op(struct gicv5_irs_chip_data *irs_data) 402 { 403 u32 statusr; 404 int ret; 405 406 ret = gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_SPI_STATUSR, 407 GICV5_IRS_SPI_STATUSR_IDLE, &statusr); 408 if (ret) 409 return ret; 410 411 return !!FIELD_GET(GICV5_IRS_SPI_STATUSR_V, statusr) ? 0 : -EIO; 412 } 413 414 static int gicv5_irs_wait_for_irs_pe(struct gicv5_irs_chip_data *irs_data, 415 bool selr) 416 { 417 bool valid = true; 418 u32 statusr; 419 int ret; 420 421 ret = gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_PE_STATUSR, 422 GICV5_IRS_PE_STATUSR_IDLE, &statusr); 423 if (ret) 424 return ret; 425 426 if (selr) 427 valid = !!FIELD_GET(GICV5_IRS_PE_STATUSR_V, statusr); 428 429 return valid ? 0 : -EIO; 430 } 431 432 static int gicv5_irs_wait_for_pe_selr(struct gicv5_irs_chip_data *irs_data) 433 { 434 return gicv5_irs_wait_for_irs_pe(irs_data, true); 435 } 436 437 static int gicv5_irs_wait_for_pe_cr0(struct gicv5_irs_chip_data *irs_data) 438 { 439 return gicv5_irs_wait_for_irs_pe(irs_data, false); 440 } 441 442 int gicv5_spi_irq_set_type(struct irq_data *d, unsigned int type) 443 { 444 struct gicv5_irs_chip_data *irs_data = d->chip_data; 445 u32 selr, cfgr; 446 bool level; 447 int ret; 448 449 /* 450 * There is no distinction between HIGH/LOW for level IRQs 451 * and RISING/FALLING for edge IRQs in the architecture, 452 * hence consider them equivalent. 453 */ 454 switch (type) { 455 case IRQ_TYPE_EDGE_RISING: 456 case IRQ_TYPE_EDGE_FALLING: 457 level = false; 458 break; 459 case IRQ_TYPE_LEVEL_HIGH: 460 case IRQ_TYPE_LEVEL_LOW: 461 level = true; 462 break; 463 default: 464 return -EINVAL; 465 } 466 467 guard(raw_spinlock)(&irs_data->spi_config_lock); 468 469 selr = FIELD_PREP(GICV5_IRS_SPI_SELR_ID, d->hwirq); 470 irs_writel_relaxed(irs_data, selr, GICV5_IRS_SPI_SELR); 471 ret = gicv5_irs_wait_for_spi_op(irs_data); 472 if (ret) 473 return ret; 474 475 cfgr = FIELD_PREP(GICV5_IRS_SPI_CFGR_TM, level); 476 irs_writel_relaxed(irs_data, cfgr, GICV5_IRS_SPI_CFGR); 477 478 return gicv5_irs_wait_for_spi_op(irs_data); 479 } 480 481 static int gicv5_irs_wait_for_idle(struct gicv5_irs_chip_data *irs_data) 482 { 483 return gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_CR0, 484 GICV5_IRS_CR0_IDLE, NULL); 485 } 486 487 void gicv5_irs_syncr(void) 488 { 489 struct gicv5_irs_chip_data *irs_data; 490 u32 syncr; 491 492 irs_data = list_first_entry_or_null(&irs_nodes, struct gicv5_irs_chip_data, entry); 493 if (WARN_ON_ONCE(!irs_data)) 494 return; 495 496 syncr = FIELD_PREP(GICV5_IRS_SYNCR_SYNC, 1); 497 irs_writel_relaxed(irs_data, syncr, GICV5_IRS_SYNCR); 498 499 gicv5_wait_for_op(irs_data->irs_base, GICV5_IRS_SYNC_STATUSR, 500 GICV5_IRS_SYNC_STATUSR_IDLE); 501 } 502 503 int gicv5_irs_register_cpu(int cpuid) 504 { 505 struct gicv5_irs_chip_data *irs_data; 506 u32 selr, cr0; 507 u16 iaffid; 508 int ret; 509 510 ret = gicv5_irs_cpu_to_iaffid(cpuid, &iaffid); 511 if (ret) { 512 pr_err("IAFFID for CPU %d has not been initialised\n", cpuid); 513 return ret; 514 } 515 516 irs_data = per_cpu(per_cpu_irs_data, cpuid); 517 if (!irs_data) { 518 pr_err("No IRS associated with CPU %u\n", cpuid); 519 return -ENXIO; 520 } 521 522 selr = FIELD_PREP(GICV5_IRS_PE_SELR_IAFFID, iaffid); 523 irs_writel_relaxed(irs_data, selr, GICV5_IRS_PE_SELR); 524 525 ret = gicv5_irs_wait_for_pe_selr(irs_data); 526 if (ret) { 527 pr_err("IAFFID 0x%x used in IRS_PE_SELR is invalid\n", iaffid); 528 return -ENXIO; 529 } 530 531 cr0 = FIELD_PREP(GICV5_IRS_PE_CR0_DPS, 0x1); 532 irs_writel_relaxed(irs_data, cr0, GICV5_IRS_PE_CR0); 533 534 ret = gicv5_irs_wait_for_pe_cr0(irs_data); 535 if (ret) 536 return ret; 537 538 pr_debug("CPU %d enabled PE IAFFID 0x%x\n", cpuid, iaffid); 539 540 return 0; 541 } 542 543 static void __init gicv5_irs_init_bases(struct gicv5_irs_chip_data *irs_data, 544 void __iomem *irs_base, 545 struct fwnode_handle *handle) 546 { 547 struct device_node *np = to_of_node(handle); 548 u32 cr0, cr1; 549 550 irs_data->fwnode = handle; 551 irs_data->irs_base = irs_base; 552 553 if (of_property_read_bool(np, "dma-noncoherent")) { 554 /* 555 * A non-coherent IRS implies that some cache levels cannot be 556 * used coherently by the cores and GIC. Our only option is to mark 557 * memory attributes for the GIC as non-cacheable; by default, 558 * non-cacheable memory attributes imply outer-shareable 559 * shareability, the value written into IRS_CR1_SH is ignored. 560 */ 561 cr1 = FIELD_PREP(GICV5_IRS_CR1_VPED_WA, GICV5_NO_WRITE_ALLOC) | 562 FIELD_PREP(GICV5_IRS_CR1_VPED_RA, GICV5_NO_READ_ALLOC) | 563 FIELD_PREP(GICV5_IRS_CR1_VMD_WA, GICV5_NO_WRITE_ALLOC) | 564 FIELD_PREP(GICV5_IRS_CR1_VMD_RA, GICV5_NO_READ_ALLOC) | 565 FIELD_PREP(GICV5_IRS_CR1_VPET_RA, GICV5_NO_READ_ALLOC) | 566 FIELD_PREP(GICV5_IRS_CR1_VMT_RA, GICV5_NO_READ_ALLOC) | 567 FIELD_PREP(GICV5_IRS_CR1_IST_WA, GICV5_NO_WRITE_ALLOC) | 568 FIELD_PREP(GICV5_IRS_CR1_IST_RA, GICV5_NO_READ_ALLOC) | 569 FIELD_PREP(GICV5_IRS_CR1_IC, GICV5_NON_CACHE) | 570 FIELD_PREP(GICV5_IRS_CR1_OC, GICV5_NON_CACHE); 571 irs_data->flags |= IRS_FLAGS_NON_COHERENT; 572 } else { 573 cr1 = FIELD_PREP(GICV5_IRS_CR1_VPED_WA, GICV5_WRITE_ALLOC) | 574 FIELD_PREP(GICV5_IRS_CR1_VPED_RA, GICV5_READ_ALLOC) | 575 FIELD_PREP(GICV5_IRS_CR1_VMD_WA, GICV5_WRITE_ALLOC) | 576 FIELD_PREP(GICV5_IRS_CR1_VMD_RA, GICV5_READ_ALLOC) | 577 FIELD_PREP(GICV5_IRS_CR1_VPET_RA, GICV5_READ_ALLOC) | 578 FIELD_PREP(GICV5_IRS_CR1_VMT_RA, GICV5_READ_ALLOC) | 579 FIELD_PREP(GICV5_IRS_CR1_IST_WA, GICV5_WRITE_ALLOC) | 580 FIELD_PREP(GICV5_IRS_CR1_IST_RA, GICV5_READ_ALLOC) | 581 FIELD_PREP(GICV5_IRS_CR1_IC, GICV5_WB_CACHE) | 582 FIELD_PREP(GICV5_IRS_CR1_OC, GICV5_WB_CACHE) | 583 FIELD_PREP(GICV5_IRS_CR1_SH, GICV5_INNER_SHARE); 584 } 585 586 irs_writel_relaxed(irs_data, cr1, GICV5_IRS_CR1); 587 588 cr0 = FIELD_PREP(GICV5_IRS_CR0_IRSEN, 0x1); 589 irs_writel_relaxed(irs_data, cr0, GICV5_IRS_CR0); 590 gicv5_irs_wait_for_idle(irs_data); 591 } 592 593 static int __init gicv5_irs_of_init_affinity(struct device_node *node, 594 struct gicv5_irs_chip_data *irs_data, 595 u8 iaffid_bits) 596 { 597 /* 598 * Detect IAFFID<->CPU mappings from the device tree and 599 * record IRS<->CPU topology information. 600 */ 601 u16 iaffid_mask = GENMASK(iaffid_bits - 1, 0); 602 int ret, i, ncpus, niaffids; 603 604 ncpus = of_count_phandle_with_args(node, "cpus", NULL); 605 if (ncpus < 0) 606 return -EINVAL; 607 608 niaffids = of_property_count_elems_of_size(node, "arm,iaffids", 609 sizeof(u16)); 610 if (niaffids != ncpus) 611 return -EINVAL; 612 613 u16 *iaffids __free(kfree) = kcalloc(niaffids, sizeof(*iaffids), GFP_KERNEL); 614 if (!iaffids) 615 return -ENOMEM; 616 617 ret = of_property_read_u16_array(node, "arm,iaffids", iaffids, niaffids); 618 if (ret) 619 return ret; 620 621 for (i = 0; i < ncpus; i++) { 622 struct device_node *cpu_node; 623 int cpu; 624 625 cpu_node = of_parse_phandle(node, "cpus", i); 626 if (WARN_ON(!cpu_node)) 627 continue; 628 629 cpu = of_cpu_node_to_id(cpu_node); 630 of_node_put(cpu_node); 631 if (WARN_ON(cpu < 0)) 632 continue; 633 634 if (iaffids[i] & ~iaffid_mask) { 635 pr_warn("CPU %d iaffid 0x%x exceeds IRS iaffid bits\n", 636 cpu, iaffids[i]); 637 continue; 638 } 639 640 per_cpu(cpu_iaffid, cpu).iaffid = iaffids[i]; 641 per_cpu(cpu_iaffid, cpu).valid = true; 642 643 /* We also know that the CPU is connected to this IRS */ 644 per_cpu(per_cpu_irs_data, cpu) = irs_data; 645 } 646 647 return ret; 648 } 649 650 static void irs_setup_pri_bits(u32 idr1) 651 { 652 switch (FIELD_GET(GICV5_IRS_IDR1_PRIORITY_BITS, idr1)) { 653 case GICV5_IRS_IDR1_PRIORITY_BITS_1BITS: 654 gicv5_global_data.irs_pri_bits = 1; 655 break; 656 case GICV5_IRS_IDR1_PRIORITY_BITS_2BITS: 657 gicv5_global_data.irs_pri_bits = 2; 658 break; 659 case GICV5_IRS_IDR1_PRIORITY_BITS_3BITS: 660 gicv5_global_data.irs_pri_bits = 3; 661 break; 662 case GICV5_IRS_IDR1_PRIORITY_BITS_4BITS: 663 gicv5_global_data.irs_pri_bits = 4; 664 break; 665 case GICV5_IRS_IDR1_PRIORITY_BITS_5BITS: 666 gicv5_global_data.irs_pri_bits = 5; 667 break; 668 default: 669 pr_warn("Detected wrong IDR priority bits value 0x%lx\n", 670 FIELD_GET(GICV5_IRS_IDR1_PRIORITY_BITS, idr1)); 671 gicv5_global_data.irs_pri_bits = 1; 672 break; 673 } 674 } 675 676 static int __init gicv5_irs_init(struct device_node *node) 677 { 678 struct gicv5_irs_chip_data *irs_data; 679 void __iomem *irs_base; 680 u32 idr, spi_count; 681 u8 iaffid_bits; 682 int ret; 683 684 irs_data = kzalloc(sizeof(*irs_data), GFP_KERNEL); 685 if (!irs_data) 686 return -ENOMEM; 687 688 raw_spin_lock_init(&irs_data->spi_config_lock); 689 690 ret = of_property_match_string(node, "reg-names", "ns-config"); 691 if (ret < 0) { 692 pr_err("%pOF: ns-config reg-name not present\n", node); 693 goto out_err; 694 } 695 696 irs_base = of_io_request_and_map(node, ret, of_node_full_name(node)); 697 if (IS_ERR(irs_base)) { 698 pr_err("%pOF: unable to map GICv5 IRS registers\n", node); 699 ret = PTR_ERR(irs_base); 700 goto out_err; 701 } 702 703 gicv5_irs_init_bases(irs_data, irs_base, &node->fwnode); 704 705 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR1); 706 iaffid_bits = FIELD_GET(GICV5_IRS_IDR1_IAFFID_BITS, idr) + 1; 707 708 ret = gicv5_irs_of_init_affinity(node, irs_data, iaffid_bits); 709 if (ret) { 710 pr_err("Failed to parse CPU IAFFIDs from the device tree!\n"); 711 goto out_iomem; 712 } 713 714 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR2); 715 if (WARN(!FIELD_GET(GICV5_IRS_IDR2_LPI, idr), 716 "LPI support not available - no IPIs, can't proceed\n")) { 717 ret = -ENODEV; 718 goto out_iomem; 719 } 720 721 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR7); 722 irs_data->spi_min = FIELD_GET(GICV5_IRS_IDR7_SPI_BASE, idr); 723 724 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR6); 725 irs_data->spi_range = FIELD_GET(GICV5_IRS_IDR6_SPI_IRS_RANGE, idr); 726 727 if (irs_data->spi_range) { 728 pr_info("%s detected SPI range [%u-%u]\n", 729 of_node_full_name(node), 730 irs_data->spi_min, 731 irs_data->spi_min + 732 irs_data->spi_range - 1); 733 } 734 735 /* 736 * Do the global setting only on the first IRS. 737 * Global properties (iaffid_bits, global spi count) are guaranteed to 738 * be consistent across IRSes by the architecture. 739 */ 740 if (list_empty(&irs_nodes)) { 741 742 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR1); 743 irs_setup_pri_bits(idr); 744 745 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR5); 746 747 spi_count = FIELD_GET(GICV5_IRS_IDR5_SPI_RANGE, idr); 748 gicv5_global_data.global_spi_count = spi_count; 749 750 gicv5_init_lpi_domain(); 751 752 pr_debug("Detected %u SPIs globally\n", spi_count); 753 } 754 755 list_add_tail(&irs_data->entry, &irs_nodes); 756 757 return 0; 758 759 out_iomem: 760 iounmap(irs_base); 761 out_err: 762 kfree(irs_data); 763 return ret; 764 } 765 766 void __init gicv5_irs_remove(void) 767 { 768 struct gicv5_irs_chip_data *irs_data, *tmp_data; 769 770 gicv5_free_lpi_domain(); 771 gicv5_deinit_lpis(); 772 773 list_for_each_entry_safe(irs_data, tmp_data, &irs_nodes, entry) { 774 iounmap(irs_data->irs_base); 775 list_del(&irs_data->entry); 776 kfree(irs_data); 777 } 778 } 779 780 int __init gicv5_irs_enable(void) 781 { 782 struct gicv5_irs_chip_data *irs_data; 783 int ret; 784 785 irs_data = list_first_entry_or_null(&irs_nodes, 786 struct gicv5_irs_chip_data, entry); 787 if (!irs_data) 788 return -ENODEV; 789 790 ret = gicv5_irs_init_ist(irs_data); 791 if (ret) { 792 pr_err("Failed to init IST\n"); 793 return ret; 794 } 795 796 return 0; 797 } 798 799 void __init gicv5_irs_its_probe(void) 800 { 801 struct gicv5_irs_chip_data *irs_data; 802 803 list_for_each_entry(irs_data, &irs_nodes, entry) 804 gicv5_its_of_probe(to_of_node(irs_data->fwnode)); 805 } 806 807 int __init gicv5_irs_of_probe(struct device_node *parent) 808 { 809 struct device_node *np; 810 int ret; 811 812 for_each_available_child_of_node(parent, np) { 813 if (!of_device_is_compatible(np, "arm,gic-v5-irs")) 814 continue; 815 816 ret = gicv5_irs_init(np); 817 if (ret) 818 pr_err("Failed to init IRS %s\n", np->full_name); 819 } 820 821 return list_empty(&irs_nodes) ? -ENODEV : 0; 822 } 823