1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2024-2025 ARM Limited, All Rights Reserved. 4 */ 5 6 #define pr_fmt(fmt) "GICv5 IRS: " fmt 7 8 #include <linux/acpi.h> 9 #include <linux/kmemleak.h> 10 #include <linux/log2.h> 11 #include <linux/of.h> 12 #include <linux/of_address.h> 13 14 #include <linux/irqchip.h> 15 #include <linux/irqchip/arm-gic-v5.h> 16 17 /* 18 * Hardcoded ID_BITS limit for systems supporting only a 1-level IST 19 * table. Systems supporting only a 1-level IST table aren't expected 20 * to require more than 2^12 LPIs. Tweak as required. 21 */ 22 #define LPI_ID_BITS_LINEAR 12 23 24 #define IRS_FLAGS_NON_COHERENT BIT(0) 25 26 static DEFINE_PER_CPU_READ_MOSTLY(struct gicv5_irs_chip_data *, per_cpu_irs_data); 27 static LIST_HEAD(irs_nodes); 28 29 static u32 irs_readl_relaxed(struct gicv5_irs_chip_data *irs_data, 30 const u32 reg_offset) 31 { 32 return readl_relaxed(irs_data->irs_base + reg_offset); 33 } 34 35 static void irs_writel_relaxed(struct gicv5_irs_chip_data *irs_data, 36 const u32 val, const u32 reg_offset) 37 { 38 writel_relaxed(val, irs_data->irs_base + reg_offset); 39 } 40 41 static u64 irs_readq_relaxed(struct gicv5_irs_chip_data *irs_data, 42 const u32 reg_offset) 43 { 44 return readq_relaxed(irs_data->irs_base + reg_offset); 45 } 46 47 static void irs_writeq_relaxed(struct gicv5_irs_chip_data *irs_data, 48 const u64 val, const u32 reg_offset) 49 { 50 writeq_relaxed(val, irs_data->irs_base + reg_offset); 51 } 52 53 /* 54 * The polling wait (in gicv5_wait_for_op_s_atomic()) on a GIC register 55 * provides the memory barriers (through MMIO accessors) 56 * required to synchronize CPU and GIC access to IST memory. 57 */ 58 static int gicv5_irs_ist_synchronise(struct gicv5_irs_chip_data *irs_data) 59 { 60 return gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_IST_STATUSR, 61 GICV5_IRS_IST_STATUSR_IDLE, NULL); 62 } 63 64 static int __init gicv5_irs_init_ist_linear(struct gicv5_irs_chip_data *irs_data, 65 unsigned int lpi_id_bits, 66 unsigned int istsz) 67 { 68 size_t l2istsz; 69 u32 n, cfgr; 70 void *ist; 71 u64 baser; 72 int ret; 73 74 /* Taken from GICv5 specifications 10.2.1.13 IRS_IST_BASER */ 75 n = max(5, lpi_id_bits + 1 + istsz); 76 77 l2istsz = BIT(n + 1); 78 /* 79 * Check memory requirements. For a linear IST we cap the 80 * number of ID bits to a value that should never exceed 81 * kmalloc interface memory allocation limits, so this 82 * check is really belt and braces. 83 */ 84 if (l2istsz > KMALLOC_MAX_SIZE) { 85 u8 lpi_id_cap = ilog2(KMALLOC_MAX_SIZE) - 2 + istsz; 86 87 pr_warn("Limiting LPI ID bits from %u to %u\n", 88 lpi_id_bits, lpi_id_cap); 89 lpi_id_bits = lpi_id_cap; 90 l2istsz = KMALLOC_MAX_SIZE; 91 } 92 93 ist = kzalloc(l2istsz, GFP_KERNEL); 94 if (!ist) 95 return -ENOMEM; 96 97 if (irs_data->flags & IRS_FLAGS_NON_COHERENT) 98 dcache_clean_inval_poc((unsigned long)ist, 99 (unsigned long)ist + l2istsz); 100 else 101 dsb(ishst); 102 103 cfgr = FIELD_PREP(GICV5_IRS_IST_CFGR_STRUCTURE, 104 GICV5_IRS_IST_CFGR_STRUCTURE_LINEAR) | 105 FIELD_PREP(GICV5_IRS_IST_CFGR_ISTSZ, istsz) | 106 FIELD_PREP(GICV5_IRS_IST_CFGR_L2SZ, 107 GICV5_IRS_IST_CFGR_L2SZ_4K) | 108 FIELD_PREP(GICV5_IRS_IST_CFGR_LPI_ID_BITS, lpi_id_bits); 109 irs_writel_relaxed(irs_data, cfgr, GICV5_IRS_IST_CFGR); 110 111 gicv5_global_data.ist.l2 = false; 112 113 baser = (virt_to_phys(ist) & GICV5_IRS_IST_BASER_ADDR_MASK) | 114 FIELD_PREP(GICV5_IRS_IST_BASER_VALID, 0x1); 115 irs_writeq_relaxed(irs_data, baser, GICV5_IRS_IST_BASER); 116 117 ret = gicv5_irs_ist_synchronise(irs_data); 118 if (ret) { 119 kfree(ist); 120 return ret; 121 } 122 kmemleak_ignore(ist); 123 124 return 0; 125 } 126 127 static int __init gicv5_irs_init_ist_two_level(struct gicv5_irs_chip_data *irs_data, 128 unsigned int lpi_id_bits, 129 unsigned int istsz, 130 unsigned int l2sz) 131 { 132 __le64 *l1ist; 133 u32 cfgr, n; 134 size_t l1sz; 135 u64 baser; 136 int ret; 137 138 /* Taken from GICv5 specifications 10.2.1.13 IRS_IST_BASER */ 139 n = max(5, lpi_id_bits - ((10 - istsz) + (2 * l2sz)) + 2); 140 141 l1sz = BIT(n + 1); 142 143 l1ist = kzalloc(l1sz, GFP_KERNEL); 144 if (!l1ist) 145 return -ENOMEM; 146 147 if (irs_data->flags & IRS_FLAGS_NON_COHERENT) 148 dcache_clean_inval_poc((unsigned long)l1ist, 149 (unsigned long)l1ist + l1sz); 150 else 151 dsb(ishst); 152 153 cfgr = FIELD_PREP(GICV5_IRS_IST_CFGR_STRUCTURE, 154 GICV5_IRS_IST_CFGR_STRUCTURE_TWO_LEVEL) | 155 FIELD_PREP(GICV5_IRS_IST_CFGR_ISTSZ, istsz) | 156 FIELD_PREP(GICV5_IRS_IST_CFGR_L2SZ, l2sz) | 157 FIELD_PREP(GICV5_IRS_IST_CFGR_LPI_ID_BITS, lpi_id_bits); 158 irs_writel_relaxed(irs_data, cfgr, GICV5_IRS_IST_CFGR); 159 160 /* 161 * The L2SZ determine bits required at L2 level. Number of bytes 162 * required by metadata is reported through istsz - the number of bits 163 * covered by L2 entries scales accordingly. 164 */ 165 gicv5_global_data.ist.l2_size = BIT(11 + (2 * l2sz) + 1); 166 gicv5_global_data.ist.l2_bits = (10 - istsz) + (2 * l2sz); 167 gicv5_global_data.ist.l1ist_addr = l1ist; 168 gicv5_global_data.ist.l2 = true; 169 170 baser = (virt_to_phys(l1ist) & GICV5_IRS_IST_BASER_ADDR_MASK) | 171 FIELD_PREP(GICV5_IRS_IST_BASER_VALID, 0x1); 172 irs_writeq_relaxed(irs_data, baser, GICV5_IRS_IST_BASER); 173 174 ret = gicv5_irs_ist_synchronise(irs_data); 175 if (ret) { 176 kfree(l1ist); 177 return ret; 178 } 179 180 return 0; 181 } 182 183 /* 184 * Alloc L2 IST entries on demand. 185 * 186 * Locking/serialization is guaranteed by irqdomain core code by 187 * taking the hierarchical domain struct irq_domain.root->mutex. 188 */ 189 int gicv5_irs_iste_alloc(const u32 lpi) 190 { 191 struct gicv5_irs_chip_data *irs_data; 192 unsigned int index; 193 u32 l2istr, l2bits; 194 __le64 *l1ist; 195 size_t l2size; 196 void *l2ist; 197 int ret; 198 199 if (!gicv5_global_data.ist.l2) 200 return 0; 201 202 irs_data = per_cpu(per_cpu_irs_data, 0); 203 if (!irs_data) 204 return -ENOENT; 205 206 l2size = gicv5_global_data.ist.l2_size; 207 l2bits = gicv5_global_data.ist.l2_bits; 208 l1ist = gicv5_global_data.ist.l1ist_addr; 209 index = lpi >> l2bits; 210 211 if (FIELD_GET(GICV5_ISTL1E_VALID, le64_to_cpu(l1ist[index]))) 212 return 0; 213 214 l2ist = kzalloc(l2size, GFP_KERNEL); 215 if (!l2ist) 216 return -ENOMEM; 217 218 l1ist[index] = cpu_to_le64(virt_to_phys(l2ist) & GICV5_ISTL1E_L2_ADDR_MASK); 219 220 if (irs_data->flags & IRS_FLAGS_NON_COHERENT) { 221 dcache_clean_inval_poc((unsigned long)l2ist, 222 (unsigned long)l2ist + l2size); 223 dcache_clean_poc((unsigned long)(l1ist + index), 224 (unsigned long)(l1ist + index) + sizeof(*l1ist)); 225 } else { 226 dsb(ishst); 227 } 228 229 l2istr = FIELD_PREP(GICV5_IRS_MAP_L2_ISTR_ID, lpi); 230 irs_writel_relaxed(irs_data, l2istr, GICV5_IRS_MAP_L2_ISTR); 231 232 ret = gicv5_irs_ist_synchronise(irs_data); 233 if (ret) { 234 l1ist[index] = 0; 235 kfree(l2ist); 236 return ret; 237 } 238 kmemleak_ignore(l2ist); 239 240 /* 241 * Make sure we invalidate the cache line pulled before the IRS 242 * had a chance to update the L1 entry and mark it valid. 243 */ 244 if (irs_data->flags & IRS_FLAGS_NON_COHERENT) { 245 /* 246 * gicv5_irs_ist_synchronise() includes memory 247 * barriers (MMIO accessors) required to guarantee that the 248 * following dcache invalidation is not executed before the 249 * IST mapping operation has completed. 250 */ 251 dcache_inval_poc((unsigned long)(l1ist + index), 252 (unsigned long)(l1ist + index) + sizeof(*l1ist)); 253 } 254 255 return 0; 256 } 257 258 /* 259 * Try to match the L2 IST size to the pagesize, and if this is not possible 260 * pick the smallest supported L2 size in order to minimise the requirement for 261 * physically contiguous blocks of memory as page-sized allocations are 262 * guaranteed to be physically contiguous, and are by definition the easiest to 263 * find. 264 * 265 * Fall back to the smallest supported size (in the event that the pagesize 266 * itself is not supported) again serves to make it easier to find physically 267 * contiguous blocks of memory. 268 */ 269 static unsigned int gicv5_irs_l2_sz(u32 idr2) 270 { 271 switch (PAGE_SIZE) { 272 case SZ_64K: 273 if (GICV5_IRS_IST_L2SZ_SUPPORT_64KB(idr2)) 274 return GICV5_IRS_IST_CFGR_L2SZ_64K; 275 fallthrough; 276 case SZ_4K: 277 if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(idr2)) 278 return GICV5_IRS_IST_CFGR_L2SZ_4K; 279 fallthrough; 280 case SZ_16K: 281 if (GICV5_IRS_IST_L2SZ_SUPPORT_16KB(idr2)) 282 return GICV5_IRS_IST_CFGR_L2SZ_16K; 283 break; 284 } 285 286 if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(idr2)) 287 return GICV5_IRS_IST_CFGR_L2SZ_4K; 288 289 return GICV5_IRS_IST_CFGR_L2SZ_64K; 290 } 291 292 static int __init gicv5_irs_init_ist(struct gicv5_irs_chip_data *irs_data) 293 { 294 u32 lpi_id_bits, idr2_id_bits, idr2_min_lpi_id_bits, l2_iste_sz, l2sz; 295 u32 l2_iste_sz_split, idr2; 296 bool two_levels, istmd; 297 u64 baser; 298 int ret; 299 300 baser = irs_readq_relaxed(irs_data, GICV5_IRS_IST_BASER); 301 if (FIELD_GET(GICV5_IRS_IST_BASER_VALID, baser)) { 302 pr_err("IST is marked as valid already; cannot allocate\n"); 303 return -EPERM; 304 } 305 306 idr2 = irs_readl_relaxed(irs_data, GICV5_IRS_IDR2); 307 two_levels = !!FIELD_GET(GICV5_IRS_IDR2_IST_LEVELS, idr2); 308 309 idr2_id_bits = FIELD_GET(GICV5_IRS_IDR2_ID_BITS, idr2); 310 idr2_min_lpi_id_bits = FIELD_GET(GICV5_IRS_IDR2_MIN_LPI_ID_BITS, idr2); 311 312 /* 313 * For two level tables we are always supporting the maximum allowed 314 * number of IDs. 315 * 316 * For 1-level tables, we should support a number of bits that 317 * is >= min_lpi_id_bits but cap it to LPI_ID_BITS_LINEAR lest 318 * the level 1-table gets too large and its memory allocation 319 * may fail. 320 */ 321 if (two_levels) { 322 lpi_id_bits = idr2_id_bits; 323 } else { 324 lpi_id_bits = max(LPI_ID_BITS_LINEAR, idr2_min_lpi_id_bits); 325 lpi_id_bits = min(lpi_id_bits, idr2_id_bits); 326 } 327 328 /* 329 * Cap the ID bits according to the CPUIF supported ID bits 330 */ 331 lpi_id_bits = min(lpi_id_bits, gicv5_global_data.cpuif_id_bits); 332 333 if (two_levels) 334 l2sz = gicv5_irs_l2_sz(idr2); 335 336 istmd = !!FIELD_GET(GICV5_IRS_IDR2_ISTMD, idr2); 337 338 l2_iste_sz = GICV5_IRS_IST_CFGR_ISTSZ_4; 339 340 if (istmd) { 341 l2_iste_sz_split = FIELD_GET(GICV5_IRS_IDR2_ISTMD_SZ, idr2); 342 343 if (lpi_id_bits < l2_iste_sz_split) 344 l2_iste_sz = GICV5_IRS_IST_CFGR_ISTSZ_8; 345 else 346 l2_iste_sz = GICV5_IRS_IST_CFGR_ISTSZ_16; 347 } 348 349 /* 350 * Follow GICv5 specification recommendation to opt in for two 351 * level tables (ref: 10.2.1.14 IRS_IST_CFGR). 352 */ 353 if (two_levels && (lpi_id_bits > ((10 - l2_iste_sz) + (2 * l2sz)))) { 354 ret = gicv5_irs_init_ist_two_level(irs_data, lpi_id_bits, 355 l2_iste_sz, l2sz); 356 } else { 357 ret = gicv5_irs_init_ist_linear(irs_data, lpi_id_bits, 358 l2_iste_sz); 359 } 360 if (ret) 361 return ret; 362 363 gicv5_init_lpis(BIT(lpi_id_bits)); 364 365 return 0; 366 } 367 368 struct iaffid_entry { 369 u16 iaffid; 370 bool valid; 371 }; 372 373 static DEFINE_PER_CPU(struct iaffid_entry, cpu_iaffid); 374 375 int gicv5_irs_cpu_to_iaffid(int cpuid, u16 *iaffid) 376 { 377 if (!per_cpu(cpu_iaffid, cpuid).valid) { 378 pr_err("IAFFID for CPU %d has not been initialised\n", cpuid); 379 return -ENODEV; 380 } 381 382 *iaffid = per_cpu(cpu_iaffid, cpuid).iaffid; 383 384 return 0; 385 } 386 387 struct gicv5_irs_chip_data *gicv5_irs_lookup_by_spi_id(u32 spi_id) 388 { 389 struct gicv5_irs_chip_data *irs_data; 390 u32 min, max; 391 392 list_for_each_entry(irs_data, &irs_nodes, entry) { 393 if (!irs_data->spi_range) 394 continue; 395 396 min = irs_data->spi_min; 397 max = irs_data->spi_min + irs_data->spi_range - 1; 398 if (spi_id >= min && spi_id <= max) 399 return irs_data; 400 } 401 402 return NULL; 403 } 404 405 static int gicv5_irs_wait_for_spi_op(struct gicv5_irs_chip_data *irs_data) 406 { 407 u32 statusr; 408 int ret; 409 410 ret = gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_SPI_STATUSR, 411 GICV5_IRS_SPI_STATUSR_IDLE, &statusr); 412 if (ret) 413 return ret; 414 415 return !!FIELD_GET(GICV5_IRS_SPI_STATUSR_V, statusr) ? 0 : -EIO; 416 } 417 418 static int gicv5_irs_wait_for_irs_pe(struct gicv5_irs_chip_data *irs_data, 419 bool selr) 420 { 421 bool valid = true; 422 u32 statusr; 423 int ret; 424 425 ret = gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_PE_STATUSR, 426 GICV5_IRS_PE_STATUSR_IDLE, &statusr); 427 if (ret) 428 return ret; 429 430 if (selr) 431 valid = !!FIELD_GET(GICV5_IRS_PE_STATUSR_V, statusr); 432 433 return valid ? 0 : -EIO; 434 } 435 436 static int gicv5_irs_wait_for_pe_selr(struct gicv5_irs_chip_data *irs_data) 437 { 438 return gicv5_irs_wait_for_irs_pe(irs_data, true); 439 } 440 441 static int gicv5_irs_wait_for_pe_cr0(struct gicv5_irs_chip_data *irs_data) 442 { 443 return gicv5_irs_wait_for_irs_pe(irs_data, false); 444 } 445 446 int gicv5_spi_irq_set_type(struct irq_data *d, unsigned int type) 447 { 448 struct gicv5_irs_chip_data *irs_data = d->chip_data; 449 u32 selr, cfgr; 450 bool level; 451 int ret; 452 453 /* 454 * There is no distinction between HIGH/LOW for level IRQs 455 * and RISING/FALLING for edge IRQs in the architecture, 456 * hence consider them equivalent. 457 */ 458 switch (type) { 459 case IRQ_TYPE_EDGE_RISING: 460 case IRQ_TYPE_EDGE_FALLING: 461 level = false; 462 break; 463 case IRQ_TYPE_LEVEL_HIGH: 464 case IRQ_TYPE_LEVEL_LOW: 465 level = true; 466 break; 467 default: 468 return -EINVAL; 469 } 470 471 guard(raw_spinlock)(&irs_data->spi_config_lock); 472 473 selr = FIELD_PREP(GICV5_IRS_SPI_SELR_ID, d->hwirq); 474 irs_writel_relaxed(irs_data, selr, GICV5_IRS_SPI_SELR); 475 ret = gicv5_irs_wait_for_spi_op(irs_data); 476 if (ret) 477 return ret; 478 479 cfgr = FIELD_PREP(GICV5_IRS_SPI_CFGR_TM, level); 480 irs_writel_relaxed(irs_data, cfgr, GICV5_IRS_SPI_CFGR); 481 482 return gicv5_irs_wait_for_spi_op(irs_data); 483 } 484 485 static int gicv5_irs_wait_for_idle(struct gicv5_irs_chip_data *irs_data) 486 { 487 return gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_CR0, 488 GICV5_IRS_CR0_IDLE, NULL); 489 } 490 491 static void __init gicv5_irs_disable(struct gicv5_irs_chip_data *irs_data) 492 { 493 irs_writel_relaxed(irs_data, 0, GICV5_IRS_CR0); 494 gicv5_irs_wait_for_idle(irs_data); 495 } 496 497 void gicv5_irs_syncr(void) 498 { 499 struct gicv5_irs_chip_data *irs_data; 500 u32 syncr; 501 502 irs_data = list_first_entry_or_null(&irs_nodes, struct gicv5_irs_chip_data, entry); 503 if (WARN_ON_ONCE(!irs_data)) 504 return; 505 506 syncr = FIELD_PREP(GICV5_IRS_SYNCR_SYNC, 1); 507 irs_writel_relaxed(irs_data, syncr, GICV5_IRS_SYNCR); 508 509 gicv5_wait_for_op(irs_data->irs_base, GICV5_IRS_SYNC_STATUSR, 510 GICV5_IRS_SYNC_STATUSR_IDLE); 511 } 512 513 int gicv5_irs_register_cpu(int cpuid) 514 { 515 struct gicv5_irs_chip_data *irs_data; 516 u32 selr, cr0; 517 u16 iaffid; 518 int ret; 519 520 ret = gicv5_irs_cpu_to_iaffid(cpuid, &iaffid); 521 if (ret) { 522 pr_err("IAFFID for CPU %d has not been initialised\n", cpuid); 523 return ret; 524 } 525 526 irs_data = per_cpu(per_cpu_irs_data, cpuid); 527 if (!irs_data) { 528 pr_err("No IRS associated with CPU %u\n", cpuid); 529 return -ENXIO; 530 } 531 532 selr = FIELD_PREP(GICV5_IRS_PE_SELR_IAFFID, iaffid); 533 irs_writel_relaxed(irs_data, selr, GICV5_IRS_PE_SELR); 534 535 ret = gicv5_irs_wait_for_pe_selr(irs_data); 536 if (ret) { 537 pr_err("IAFFID 0x%x used in IRS_PE_SELR is invalid\n", iaffid); 538 return -ENXIO; 539 } 540 541 cr0 = FIELD_PREP(GICV5_IRS_PE_CR0_DPS, 0x1); 542 irs_writel_relaxed(irs_data, cr0, GICV5_IRS_PE_CR0); 543 544 ret = gicv5_irs_wait_for_pe_cr0(irs_data); 545 if (ret) 546 return ret; 547 548 pr_debug("CPU %d enabled PE IAFFID 0x%x\n", cpuid, iaffid); 549 550 return 0; 551 } 552 553 static void __init gicv5_irs_init_bases(struct gicv5_irs_chip_data *irs_data, 554 void __iomem *irs_base, 555 bool noncoherent) 556 { 557 u32 cr0, cr1; 558 559 irs_data->irs_base = irs_base; 560 561 if (noncoherent) { 562 /* 563 * A non-coherent IRS implies that some cache levels cannot be 564 * used coherently by the cores and GIC. Our only option is to mark 565 * memory attributes for the GIC as non-cacheable; by default, 566 * non-cacheable memory attributes imply outer-shareable 567 * shareability, the value written into IRS_CR1_SH is ignored. 568 */ 569 cr1 = FIELD_PREP(GICV5_IRS_CR1_VPED_WA, GICV5_NO_WRITE_ALLOC) | 570 FIELD_PREP(GICV5_IRS_CR1_VPED_RA, GICV5_NO_READ_ALLOC) | 571 FIELD_PREP(GICV5_IRS_CR1_VMD_WA, GICV5_NO_WRITE_ALLOC) | 572 FIELD_PREP(GICV5_IRS_CR1_VMD_RA, GICV5_NO_READ_ALLOC) | 573 FIELD_PREP(GICV5_IRS_CR1_VPET_RA, GICV5_NO_READ_ALLOC) | 574 FIELD_PREP(GICV5_IRS_CR1_VMT_RA, GICV5_NO_READ_ALLOC) | 575 FIELD_PREP(GICV5_IRS_CR1_IST_WA, GICV5_NO_WRITE_ALLOC) | 576 FIELD_PREP(GICV5_IRS_CR1_IST_RA, GICV5_NO_READ_ALLOC) | 577 FIELD_PREP(GICV5_IRS_CR1_IC, GICV5_NON_CACHE) | 578 FIELD_PREP(GICV5_IRS_CR1_OC, GICV5_NON_CACHE); 579 irs_data->flags |= IRS_FLAGS_NON_COHERENT; 580 } else { 581 cr1 = FIELD_PREP(GICV5_IRS_CR1_VPED_WA, GICV5_WRITE_ALLOC) | 582 FIELD_PREP(GICV5_IRS_CR1_VPED_RA, GICV5_READ_ALLOC) | 583 FIELD_PREP(GICV5_IRS_CR1_VMD_WA, GICV5_WRITE_ALLOC) | 584 FIELD_PREP(GICV5_IRS_CR1_VMD_RA, GICV5_READ_ALLOC) | 585 FIELD_PREP(GICV5_IRS_CR1_VPET_RA, GICV5_READ_ALLOC) | 586 FIELD_PREP(GICV5_IRS_CR1_VMT_RA, GICV5_READ_ALLOC) | 587 FIELD_PREP(GICV5_IRS_CR1_IST_WA, GICV5_WRITE_ALLOC) | 588 FIELD_PREP(GICV5_IRS_CR1_IST_RA, GICV5_READ_ALLOC) | 589 FIELD_PREP(GICV5_IRS_CR1_IC, GICV5_WB_CACHE) | 590 FIELD_PREP(GICV5_IRS_CR1_OC, GICV5_WB_CACHE) | 591 FIELD_PREP(GICV5_IRS_CR1_SH, GICV5_INNER_SHARE); 592 } 593 594 irs_writel_relaxed(irs_data, cr1, GICV5_IRS_CR1); 595 596 cr0 = FIELD_PREP(GICV5_IRS_CR0_IRSEN, 0x1); 597 irs_writel_relaxed(irs_data, cr0, GICV5_IRS_CR0); 598 gicv5_irs_wait_for_idle(irs_data); 599 } 600 601 static int __init gicv5_irs_of_init_affinity(struct device_node *node, 602 struct gicv5_irs_chip_data *irs_data, 603 u8 iaffid_bits) 604 { 605 /* 606 * Detect IAFFID<->CPU mappings from the device tree and 607 * record IRS<->CPU topology information. 608 */ 609 u16 iaffid_mask = GENMASK(iaffid_bits - 1, 0); 610 int ret, i, ncpus, niaffids; 611 612 /* An IRS is architecturally allowed to have no cpus associated with it */ 613 if (!of_property_present(node, "cpus") && !of_property_present(node, "arm,iaffids")) 614 return 0; 615 616 ncpus = of_count_phandle_with_args(node, "cpus", NULL); 617 if (ncpus < 0) 618 return -EINVAL; 619 620 niaffids = of_property_count_elems_of_size(node, "arm,iaffids", 621 sizeof(u16)); 622 if (niaffids != ncpus) 623 return -EINVAL; 624 625 u16 *iaffids __free(kfree) = kzalloc_objs(*iaffids, niaffids); 626 if (!iaffids) 627 return -ENOMEM; 628 629 ret = of_property_read_u16_array(node, "arm,iaffids", iaffids, niaffids); 630 if (ret) 631 return ret; 632 633 for (i = 0; i < ncpus; i++) { 634 struct device_node *cpu_node; 635 int cpu; 636 637 cpu_node = of_parse_phandle(node, "cpus", i); 638 if (!cpu_node) { 639 pr_warn(FW_BUG "Erroneous CPU node phandle\n"); 640 continue; 641 } 642 643 cpu = of_cpu_node_to_id(cpu_node); 644 of_node_put(cpu_node); 645 if (cpu < 0) 646 continue; 647 648 if (iaffids[i] & ~iaffid_mask) { 649 pr_warn("CPU %d iaffid 0x%x exceeds IRS iaffid bits\n", 650 cpu, iaffids[i]); 651 continue; 652 } 653 654 per_cpu(cpu_iaffid, cpu).iaffid = iaffids[i]; 655 per_cpu(cpu_iaffid, cpu).valid = true; 656 657 /* We also know that the CPU is connected to this IRS */ 658 per_cpu(per_cpu_irs_data, cpu) = irs_data; 659 } 660 661 return ret; 662 } 663 664 static void __init gicv5_irs_clear_affinity(struct gicv5_irs_chip_data *irs_data) 665 { 666 int cpu; 667 668 for_each_possible_cpu(cpu) { 669 if (per_cpu(per_cpu_irs_data, cpu) == irs_data) { 670 per_cpu(cpu_iaffid, cpu).valid = false; 671 per_cpu(per_cpu_irs_data, cpu) = NULL; 672 } 673 } 674 } 675 676 static void irs_setup_pri_bits(u32 idr1) 677 { 678 switch (FIELD_GET(GICV5_IRS_IDR1_PRIORITY_BITS, idr1)) { 679 case GICV5_IRS_IDR1_PRIORITY_BITS_1BITS: 680 gicv5_global_data.irs_pri_bits = 1; 681 break; 682 case GICV5_IRS_IDR1_PRIORITY_BITS_2BITS: 683 gicv5_global_data.irs_pri_bits = 2; 684 break; 685 case GICV5_IRS_IDR1_PRIORITY_BITS_3BITS: 686 gicv5_global_data.irs_pri_bits = 3; 687 break; 688 case GICV5_IRS_IDR1_PRIORITY_BITS_4BITS: 689 gicv5_global_data.irs_pri_bits = 4; 690 break; 691 case GICV5_IRS_IDR1_PRIORITY_BITS_5BITS: 692 gicv5_global_data.irs_pri_bits = 5; 693 break; 694 default: 695 pr_warn("Detected wrong IDR priority bits value 0x%lx\n", 696 FIELD_GET(GICV5_IRS_IDR1_PRIORITY_BITS, idr1)); 697 gicv5_global_data.irs_pri_bits = 1; 698 break; 699 } 700 } 701 702 static int __init gicv5_irs_init(struct gicv5_irs_chip_data *irs_data) 703 { 704 u32 spi_count, idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR2); 705 706 if (WARN(!FIELD_GET(GICV5_IRS_IDR2_LPI, idr), 707 "LPI support not available - no IPIs, can't proceed\n")) { 708 return -ENODEV; 709 } 710 711 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR7); 712 irs_data->spi_min = FIELD_GET(GICV5_IRS_IDR7_SPI_BASE, idr); 713 714 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR6); 715 irs_data->spi_range = FIELD_GET(GICV5_IRS_IDR6_SPI_IRS_RANGE, idr); 716 717 /* 718 * Do the global setting only on the first IRS. 719 * Global properties (iaffid_bits, global spi count) are guaranteed to 720 * be consistent across IRSes by the architecture. 721 */ 722 if (list_empty(&irs_nodes)) { 723 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR0); 724 gicv5_global_data.virt_capable = !!FIELD_GET(GICV5_IRS_IDR0_VIRT, idr); 725 726 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR1); 727 irs_setup_pri_bits(idr); 728 729 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR5); 730 731 spi_count = FIELD_GET(GICV5_IRS_IDR5_SPI_RANGE, idr); 732 gicv5_global_data.global_spi_count = spi_count; 733 734 gicv5_init_lpi_domain(); 735 736 pr_debug("Detected %u SPIs globally\n", spi_count); 737 } 738 739 list_add_tail(&irs_data->entry, &irs_nodes); 740 741 return 0; 742 } 743 744 static int __init gicv5_irs_of_init(struct device_node *node) 745 { 746 struct gicv5_irs_chip_data *irs_data; 747 struct resource res; 748 void __iomem *irs_base; 749 u8 iaffid_bits; 750 u32 idr; 751 int ret; 752 753 irs_data = kzalloc_obj(*irs_data); 754 if (!irs_data) 755 return -ENOMEM; 756 757 raw_spin_lock_init(&irs_data->spi_config_lock); 758 759 ret = of_property_match_string(node, "reg-names", "ns-config"); 760 if (ret < 0) { 761 pr_err("%pOF: ns-config reg-name not present\n", node); 762 goto out_err; 763 } 764 765 if (of_address_to_resource(node, ret, &res)) { 766 ret = -EINVAL; 767 goto out_err; 768 } 769 770 irs_base = of_io_request_and_map(node, ret, of_node_full_name(node)); 771 if (IS_ERR(irs_base)) { 772 pr_err("%pOF: unable to map GICv5 IRS registers\n", node); 773 ret = PTR_ERR(irs_base); 774 goto out_err; 775 } 776 777 irs_data->fwnode = of_fwnode_handle(node); 778 irs_data->res = res; 779 gicv5_irs_init_bases(irs_data, irs_base, of_property_read_bool(node, "dma-noncoherent")); 780 781 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR1); 782 iaffid_bits = FIELD_GET(GICV5_IRS_IDR1_IAFFID_BITS, idr) + 1; 783 784 ret = gicv5_irs_of_init_affinity(node, irs_data, iaffid_bits); 785 if (ret) { 786 pr_err("Failed to parse CPU IAFFIDs from the device tree!\n"); 787 goto out_iomem; 788 } 789 790 ret = gicv5_irs_init(irs_data); 791 if (ret) 792 goto out_iomem; 793 794 if (irs_data->spi_range) { 795 pr_info("%s detected SPI range [%u-%u]\n", 796 of_node_full_name(node), 797 irs_data->spi_min, 798 irs_data->spi_min + 799 irs_data->spi_range - 1); 800 } 801 802 return ret; 803 804 out_iomem: 805 gicv5_irs_clear_affinity(irs_data); 806 gicv5_irs_disable(irs_data); 807 iounmap(irs_base); 808 release_mem_region(res.start, resource_size(&res)); 809 out_err: 810 kfree(irs_data); 811 return ret; 812 } 813 814 void __init gicv5_irs_remove(void) 815 { 816 struct gicv5_irs_chip_data *irs_data, *tmp_data; 817 818 gicv5_free_lpi_domain(); 819 gicv5_deinit_lpis(); 820 821 list_for_each_entry_safe(irs_data, tmp_data, &irs_nodes, entry) { 822 gicv5_irs_clear_affinity(irs_data); 823 gicv5_irs_disable(irs_data); 824 iounmap(irs_data->irs_base); 825 release_mem_region(irs_data->res.start, resource_size(&irs_data->res)); 826 list_del(&irs_data->entry); 827 kfree(irs_data); 828 } 829 } 830 831 int __init gicv5_irs_enable(void) 832 { 833 struct gicv5_irs_chip_data *irs_data; 834 int ret; 835 836 irs_data = list_first_entry_or_null(&irs_nodes, 837 struct gicv5_irs_chip_data, entry); 838 if (!irs_data) 839 return -ENODEV; 840 841 ret = gicv5_irs_init_ist(irs_data); 842 if (ret) { 843 pr_err("Failed to init IST\n"); 844 return ret; 845 } 846 847 return 0; 848 } 849 850 void __init gicv5_irs_its_probe(void) 851 { 852 struct gicv5_irs_chip_data *irs_data; 853 854 if (acpi_disabled) 855 list_for_each_entry(irs_data, &irs_nodes, entry) 856 gicv5_its_of_probe(to_of_node(irs_data->fwnode)); 857 else 858 gicv5_its_acpi_probe(); 859 } 860 861 int __init gicv5_irs_of_probe(struct device_node *parent) 862 { 863 struct device_node *np; 864 int ret; 865 866 for_each_available_child_of_node(parent, np) { 867 if (!of_device_is_compatible(np, "arm,gic-v5-irs")) 868 continue; 869 870 ret = gicv5_irs_of_init(np); 871 if (ret) 872 pr_err("Failed to init IRS %s\n", np->full_name); 873 } 874 875 return list_empty(&irs_nodes) ? -ENODEV : 0; 876 } 877 878 #ifdef CONFIG_ACPI 879 880 #define ACPI_GICV5_IRS_MEM_SIZE (SZ_64K) 881 static struct gicv5_irs_chip_data *current_irs_data __initdata; 882 static int current_irsid __initdata = -1; 883 static u8 current_iaffid_bits __initdata; 884 885 static int __init gic_acpi_parse_iaffid(union acpi_subtable_headers *header, 886 const unsigned long end) 887 { 888 struct acpi_madt_generic_interrupt *gicc = (struct acpi_madt_generic_interrupt *)header; 889 int cpu; 890 891 if (!(gicc->flags & (ACPI_MADT_ENABLED | ACPI_MADT_GICC_ONLINE_CAPABLE))) 892 return 0; 893 894 if (gicc->irs_id != current_irsid) 895 return 0; 896 897 cpu = get_logical_index(gicc->arm_mpidr); 898 if (cpu < 0) 899 return 0; 900 901 if (gicc->iaffid & ~GENMASK(current_iaffid_bits - 1, 0)) { 902 pr_warn("CPU %d iaffid 0x%x exceeds IRS iaffid bits\n", cpu, gicc->iaffid); 903 return 0; 904 } 905 906 /* Bind the IAFFID and the CPU */ 907 per_cpu(cpu_iaffid, cpu).iaffid = gicc->iaffid; 908 per_cpu(cpu_iaffid, cpu).valid = true; 909 pr_debug("Processed IAFFID %u for CPU%d", per_cpu(cpu_iaffid, cpu).iaffid, cpu); 910 911 /* We also know that the CPU is connected to this IRS */ 912 per_cpu(per_cpu_irs_data, cpu) = current_irs_data; 913 914 return 0; 915 } 916 917 static int __init gicv5_irs_acpi_init_affinity(u32 irsid, struct gicv5_irs_chip_data *irs_data) 918 { 919 u32 idr; 920 921 current_irsid = irsid; 922 current_irs_data = irs_data; 923 924 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR1); 925 current_iaffid_bits = FIELD_GET(GICV5_IRS_IDR1_IAFFID_BITS, idr) + 1; 926 927 acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT, gic_acpi_parse_iaffid, 0); 928 929 return 0; 930 } 931 932 static struct resource * __init gic_request_region(resource_size_t base, resource_size_t size, 933 const char *name) 934 { 935 struct resource *r = request_mem_region(base, size, name); 936 937 if (!r) 938 pr_warn_once(FW_BUG "%s region %pa has overlapping address\n", name, &base); 939 940 return r; 941 } 942 943 static int __init gic_acpi_parse_madt_irs(union acpi_subtable_headers *header, 944 const unsigned long end) 945 { 946 struct acpi_madt_gicv5_irs *irs = (struct acpi_madt_gicv5_irs *)header; 947 struct gicv5_irs_chip_data *irs_data; 948 void __iomem *irs_base; 949 struct resource *r; 950 int ret; 951 952 /* Per-IRS data structure */ 953 irs_data = kzalloc_obj(*irs_data); 954 if (!irs_data) 955 return -ENOMEM; 956 957 /* This spinlock is used for SPI config changes */ 958 raw_spin_lock_init(&irs_data->spi_config_lock); 959 960 r = gic_request_region(irs->config_base_address, ACPI_GICV5_IRS_MEM_SIZE, "GICv5 IRS"); 961 if (!r) { 962 ret = -EBUSY; 963 goto out_free; 964 } 965 966 irs_base = ioremap(irs->config_base_address, ACPI_GICV5_IRS_MEM_SIZE); 967 if (!irs_base) { 968 pr_err("Unable to map GIC IRS registers\n"); 969 ret = -ENOMEM; 970 goto out_release; 971 } 972 973 gicv5_irs_init_bases(irs_data, irs_base, irs->flags & ACPI_MADT_IRS_NON_COHERENT); 974 irs_data->res = *r; 975 976 gicv5_irs_acpi_init_affinity(irs->irs_id, irs_data); 977 978 ret = gicv5_irs_init(irs_data); 979 if (ret) 980 goto out_map; 981 982 if (irs_data->spi_range) { 983 pr_info("%s @%llx detected SPI range [%u-%u]\n", "IRS", irs->config_base_address, 984 irs_data->spi_min, 985 irs_data->spi_min + 986 irs_data->spi_range - 1); 987 } 988 989 return 0; 990 991 out_map: 992 gicv5_irs_clear_affinity(irs_data); 993 gicv5_irs_disable(irs_data); 994 iounmap(irs_base); 995 out_release: 996 release_mem_region(r->start, resource_size(r)); 997 out_free: 998 kfree(irs_data); 999 return ret; 1000 } 1001 1002 int __init gicv5_irs_acpi_probe(void) 1003 { 1004 acpi_table_parse_madt(ACPI_MADT_TYPE_GICV5_IRS, gic_acpi_parse_madt_irs, 0); 1005 1006 return list_empty(&irs_nodes) ? -ENODEV : 0; 1007 } 1008 #endif 1009