1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2024-2025 ARM Limited, All Rights Reserved. 4 */ 5 6 #define pr_fmt(fmt) "GICv5 IRS: " fmt 7 8 #include <linux/acpi.h> 9 #include <linux/kmemleak.h> 10 #include <linux/log2.h> 11 #include <linux/of.h> 12 #include <linux/of_address.h> 13 14 #include <linux/irqchip.h> 15 #include <linux/irqchip/arm-gic-v5.h> 16 17 /* 18 * Hardcoded ID_BITS limit for systems supporting only a 1-level IST 19 * table. Systems supporting only a 1-level IST table aren't expected 20 * to require more than 2^12 LPIs. Tweak as required. 21 */ 22 #define LPI_ID_BITS_LINEAR 12 23 24 static DEFINE_PER_CPU_READ_MOSTLY(struct gicv5_irs_chip_data *, per_cpu_irs_data); 25 static LIST_HEAD(irs_nodes); 26 27 static u32 irs_readl_relaxed(struct gicv5_irs_chip_data *irs_data, 28 const u32 reg_offset) 29 { 30 return readl_relaxed(irs_data->irs_base + reg_offset); 31 } 32 33 static void irs_writel_relaxed(struct gicv5_irs_chip_data *irs_data, 34 const u32 val, const u32 reg_offset) 35 { 36 writel_relaxed(val, irs_data->irs_base + reg_offset); 37 } 38 39 static u64 irs_readq_relaxed(struct gicv5_irs_chip_data *irs_data, 40 const u32 reg_offset) 41 { 42 return readq_relaxed(irs_data->irs_base + reg_offset); 43 } 44 45 static void irs_writeq_relaxed(struct gicv5_irs_chip_data *irs_data, 46 const u64 val, const u32 reg_offset) 47 { 48 writeq_relaxed(val, irs_data->irs_base + reg_offset); 49 } 50 51 struct gicv5_irs_chip_data *gicv5_irs_get_chip_data(void) 52 { 53 return per_cpu(per_cpu_irs_data, 0); 54 } 55 56 /* 57 * The polling wait (in gicv5_wait_for_op_s_atomic()) on a GIC register 58 * provides the memory barriers (through MMIO accessors) 59 * required to synchronize CPU and GIC access to IST memory. 60 */ 61 static int gicv5_irs_ist_synchronise(struct gicv5_irs_chip_data *irs_data) 62 { 63 return gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_IST_STATUSR, 64 GICV5_IRS_IST_STATUSR_IDLE, NULL); 65 } 66 67 static int __init gicv5_irs_init_ist_linear(struct gicv5_irs_chip_data *irs_data, 68 unsigned int lpi_id_bits, 69 unsigned int istsz) 70 { 71 size_t l2istsz; 72 u32 n, cfgr; 73 void *ist; 74 u64 baser; 75 int ret; 76 77 /* Taken from GICv5 specifications 10.2.1.13 IRS_IST_BASER */ 78 n = max(5, lpi_id_bits + 1 + istsz); 79 80 l2istsz = BIT(n + 1); 81 /* 82 * Check memory requirements. For a linear IST we cap the 83 * number of ID bits to a value that should never exceed 84 * kmalloc interface memory allocation limits, so this 85 * check is really belt and braces. 86 */ 87 if (l2istsz > KMALLOC_MAX_SIZE) { 88 u8 lpi_id_cap = ilog2(KMALLOC_MAX_SIZE) - 2 + istsz; 89 90 pr_warn("Limiting LPI ID bits from %u to %u\n", 91 lpi_id_bits, lpi_id_cap); 92 lpi_id_bits = lpi_id_cap; 93 l2istsz = KMALLOC_MAX_SIZE; 94 } 95 96 ist = kzalloc(l2istsz, GFP_KERNEL); 97 if (!ist) 98 return -ENOMEM; 99 100 if (irs_data->flags & IRS_FLAGS_NON_COHERENT) 101 dcache_clean_inval_poc((unsigned long)ist, 102 (unsigned long)ist + l2istsz); 103 else 104 dsb(ishst); 105 106 cfgr = FIELD_PREP(GICV5_IRS_IST_CFGR_STRUCTURE, 107 GICV5_IRS_IST_CFGR_STRUCTURE_LINEAR) | 108 FIELD_PREP(GICV5_IRS_IST_CFGR_ISTSZ, istsz) | 109 FIELD_PREP(GICV5_IRS_IST_CFGR_L2SZ, 110 GICV5_IRS_IST_CFGR_L2SZ_4K) | 111 FIELD_PREP(GICV5_IRS_IST_CFGR_LPI_ID_BITS, lpi_id_bits); 112 irs_writel_relaxed(irs_data, cfgr, GICV5_IRS_IST_CFGR); 113 114 gicv5_global_data.ist.l2 = false; 115 116 baser = (virt_to_phys(ist) & GICV5_IRS_IST_BASER_ADDR_MASK) | 117 FIELD_PREP(GICV5_IRS_IST_BASER_VALID, 0x1); 118 irs_writeq_relaxed(irs_data, baser, GICV5_IRS_IST_BASER); 119 120 ret = gicv5_irs_ist_synchronise(irs_data); 121 if (ret) { 122 kfree(ist); 123 return ret; 124 } 125 kmemleak_ignore(ist); 126 127 return 0; 128 } 129 130 static int __init gicv5_irs_init_ist_two_level(struct gicv5_irs_chip_data *irs_data, 131 unsigned int lpi_id_bits, 132 unsigned int istsz, 133 unsigned int l2sz) 134 { 135 __le64 *l1ist; 136 u32 cfgr, n; 137 size_t l1sz; 138 u64 baser; 139 int ret; 140 141 /* Taken from GICv5 specifications 10.2.1.13 IRS_IST_BASER */ 142 n = max(5, lpi_id_bits - ((10 - istsz) + (2 * l2sz)) + 2); 143 144 l1sz = BIT(n + 1); 145 146 l1ist = kzalloc(l1sz, GFP_KERNEL); 147 if (!l1ist) 148 return -ENOMEM; 149 150 if (irs_data->flags & IRS_FLAGS_NON_COHERENT) 151 dcache_clean_inval_poc((unsigned long)l1ist, 152 (unsigned long)l1ist + l1sz); 153 else 154 dsb(ishst); 155 156 cfgr = FIELD_PREP(GICV5_IRS_IST_CFGR_STRUCTURE, 157 GICV5_IRS_IST_CFGR_STRUCTURE_TWO_LEVEL) | 158 FIELD_PREP(GICV5_IRS_IST_CFGR_ISTSZ, istsz) | 159 FIELD_PREP(GICV5_IRS_IST_CFGR_L2SZ, l2sz) | 160 FIELD_PREP(GICV5_IRS_IST_CFGR_LPI_ID_BITS, lpi_id_bits); 161 irs_writel_relaxed(irs_data, cfgr, GICV5_IRS_IST_CFGR); 162 163 /* 164 * The L2SZ determine bits required at L2 level. Number of bytes 165 * required by metadata is reported through istsz - the number of bits 166 * covered by L2 entries scales accordingly. 167 */ 168 gicv5_global_data.ist.l2_size = BIT(11 + (2 * l2sz) + 1); 169 gicv5_global_data.ist.l2_bits = (10 - istsz) + (2 * l2sz); 170 gicv5_global_data.ist.l1ist_addr = l1ist; 171 gicv5_global_data.ist.l2 = true; 172 173 baser = (virt_to_phys(l1ist) & GICV5_IRS_IST_BASER_ADDR_MASK) | 174 FIELD_PREP(GICV5_IRS_IST_BASER_VALID, 0x1); 175 irs_writeq_relaxed(irs_data, baser, GICV5_IRS_IST_BASER); 176 177 ret = gicv5_irs_ist_synchronise(irs_data); 178 if (ret) { 179 kfree(l1ist); 180 return ret; 181 } 182 183 return 0; 184 } 185 186 /* 187 * Alloc L2 IST entries on demand. 188 * 189 * Locking/serialization is guaranteed by irqdomain core code by 190 * taking the hierarchical domain struct irq_domain.root->mutex. 191 */ 192 int gicv5_irs_iste_alloc(const u32 lpi) 193 { 194 struct gicv5_irs_chip_data *irs_data; 195 unsigned int index; 196 u32 l2istr, l2bits; 197 __le64 *l1ist; 198 size_t l2size; 199 void *l2ist; 200 int ret; 201 202 if (!gicv5_global_data.ist.l2) 203 return 0; 204 205 irs_data = per_cpu(per_cpu_irs_data, 0); 206 if (!irs_data) 207 return -ENOENT; 208 209 l2size = gicv5_global_data.ist.l2_size; 210 l2bits = gicv5_global_data.ist.l2_bits; 211 l1ist = gicv5_global_data.ist.l1ist_addr; 212 index = lpi >> l2bits; 213 214 if (FIELD_GET(GICV5_ISTL1E_VALID, le64_to_cpu(l1ist[index]))) 215 return 0; 216 217 l2ist = kzalloc(l2size, GFP_KERNEL); 218 if (!l2ist) 219 return -ENOMEM; 220 221 l1ist[index] = cpu_to_le64(virt_to_phys(l2ist) & GICV5_ISTL1E_L2_ADDR_MASK); 222 223 if (irs_data->flags & IRS_FLAGS_NON_COHERENT) { 224 dcache_clean_inval_poc((unsigned long)l2ist, 225 (unsigned long)l2ist + l2size); 226 dcache_clean_poc((unsigned long)(l1ist + index), 227 (unsigned long)(l1ist + index) + sizeof(*l1ist)); 228 } else { 229 dsb(ishst); 230 } 231 232 l2istr = FIELD_PREP(GICV5_IRS_MAP_L2_ISTR_ID, lpi); 233 irs_writel_relaxed(irs_data, l2istr, GICV5_IRS_MAP_L2_ISTR); 234 235 ret = gicv5_irs_ist_synchronise(irs_data); 236 if (ret) { 237 l1ist[index] = 0; 238 kfree(l2ist); 239 return ret; 240 } 241 kmemleak_ignore(l2ist); 242 243 /* 244 * Make sure we invalidate the cache line pulled before the IRS 245 * had a chance to update the L1 entry and mark it valid. 246 */ 247 if (irs_data->flags & IRS_FLAGS_NON_COHERENT) { 248 /* 249 * gicv5_irs_ist_synchronise() includes memory 250 * barriers (MMIO accessors) required to guarantee that the 251 * following dcache invalidation is not executed before the 252 * IST mapping operation has completed. 253 */ 254 dcache_inval_poc((unsigned long)(l1ist + index), 255 (unsigned long)(l1ist + index) + sizeof(*l1ist)); 256 } 257 258 return 0; 259 } 260 261 /* 262 * Try to match the L2 IST size to the pagesize, and if this is not possible 263 * pick the smallest supported L2 size in order to minimise the requirement for 264 * physically contiguous blocks of memory as page-sized allocations are 265 * guaranteed to be physically contiguous, and are by definition the easiest to 266 * find. 267 * 268 * Fall back to the smallest supported size (in the event that the pagesize 269 * itself is not supported) again serves to make it easier to find physically 270 * contiguous blocks of memory. 271 */ 272 unsigned int gicv5_irs_l2_sz(u32 l2sz) 273 { 274 switch (PAGE_SIZE) { 275 case SZ_64K: 276 if (GICV5_IRS_IST_L2SZ_SUPPORT_64KB(l2sz)) 277 return GICV5_IRS_IST_CFGR_L2SZ_64K; 278 fallthrough; 279 case SZ_4K: 280 if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(l2sz)) 281 return GICV5_IRS_IST_CFGR_L2SZ_4K; 282 fallthrough; 283 case SZ_16K: 284 if (GICV5_IRS_IST_L2SZ_SUPPORT_16KB(l2sz)) 285 return GICV5_IRS_IST_CFGR_L2SZ_16K; 286 break; 287 } 288 289 if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(l2sz)) 290 return GICV5_IRS_IST_CFGR_L2SZ_4K; 291 292 return GICV5_IRS_IST_CFGR_L2SZ_64K; 293 } 294 295 static int __init gicv5_irs_init_ist(struct gicv5_irs_chip_data *irs_data) 296 { 297 u32 lpi_id_bits, idr2_id_bits, idr2_min_lpi_id_bits, l2_iste_sz, l2sz; 298 u32 l2_iste_sz_split, idr2; 299 bool two_levels, istmd; 300 u64 baser; 301 int ret; 302 303 baser = irs_readq_relaxed(irs_data, GICV5_IRS_IST_BASER); 304 if (FIELD_GET(GICV5_IRS_IST_BASER_VALID, baser)) { 305 pr_err("IST is marked as valid already; cannot allocate\n"); 306 return -EPERM; 307 } 308 309 idr2 = irs_readl_relaxed(irs_data, GICV5_IRS_IDR2); 310 two_levels = !!FIELD_GET(GICV5_IRS_IDR2_IST_LEVELS, idr2); 311 312 idr2_id_bits = FIELD_GET(GICV5_IRS_IDR2_ID_BITS, idr2); 313 idr2_min_lpi_id_bits = FIELD_GET(GICV5_IRS_IDR2_MIN_LPI_ID_BITS, idr2); 314 315 /* 316 * For two level tables we are always supporting the maximum allowed 317 * number of IDs. 318 * 319 * For 1-level tables, we should support a number of bits that 320 * is >= min_lpi_id_bits but cap it to LPI_ID_BITS_LINEAR lest 321 * the level 1-table gets too large and its memory allocation 322 * may fail. 323 */ 324 if (two_levels) { 325 lpi_id_bits = idr2_id_bits; 326 } else { 327 lpi_id_bits = max(LPI_ID_BITS_LINEAR, idr2_min_lpi_id_bits); 328 lpi_id_bits = min(lpi_id_bits, idr2_id_bits); 329 } 330 331 /* 332 * Cap the ID bits according to the CPUIF supported ID bits 333 */ 334 lpi_id_bits = min(lpi_id_bits, gicv5_global_data.cpuif_id_bits); 335 336 if (two_levels) 337 l2sz = gicv5_irs_l2_sz(FIELD_GET(GICV5_IRS_IDR2_IST_L2SZ, idr2)); 338 339 istmd = !!FIELD_GET(GICV5_IRS_IDR2_ISTMD, idr2); 340 341 l2_iste_sz = GICV5_IRS_IST_CFGR_ISTSZ_4; 342 343 if (istmd) { 344 l2_iste_sz_split = FIELD_GET(GICV5_IRS_IDR2_ISTMD_SZ, idr2); 345 346 if (lpi_id_bits < l2_iste_sz_split) 347 l2_iste_sz = GICV5_IRS_IST_CFGR_ISTSZ_8; 348 else 349 l2_iste_sz = GICV5_IRS_IST_CFGR_ISTSZ_16; 350 } 351 352 /* 353 * Follow GICv5 specification recommendation to opt in for two 354 * level tables (ref: 10.2.1.14 IRS_IST_CFGR). 355 */ 356 if (two_levels && (lpi_id_bits > ((10 - l2_iste_sz) + (2 * l2sz)))) { 357 ret = gicv5_irs_init_ist_two_level(irs_data, lpi_id_bits, 358 l2_iste_sz, l2sz); 359 } else { 360 ret = gicv5_irs_init_ist_linear(irs_data, lpi_id_bits, 361 l2_iste_sz); 362 } 363 if (ret) 364 return ret; 365 366 gicv5_init_lpis(BIT(lpi_id_bits)); 367 368 return 0; 369 } 370 371 struct iaffid_entry { 372 u16 iaffid; 373 bool valid; 374 }; 375 376 static DEFINE_PER_CPU(struct iaffid_entry, cpu_iaffid); 377 378 int gicv5_irs_cpu_to_iaffid(int cpuid, u16 *iaffid) 379 { 380 if (!per_cpu(cpu_iaffid, cpuid).valid) { 381 pr_err("IAFFID for CPU %d has not been initialised\n", cpuid); 382 return -ENODEV; 383 } 384 385 *iaffid = per_cpu(cpu_iaffid, cpuid).iaffid; 386 387 return 0; 388 } 389 390 struct gicv5_irs_chip_data *gicv5_irs_lookup_by_spi_id(u32 spi_id) 391 { 392 struct gicv5_irs_chip_data *irs_data; 393 u32 min, max; 394 395 list_for_each_entry(irs_data, &irs_nodes, entry) { 396 if (!irs_data->spi_range) 397 continue; 398 399 min = irs_data->spi_min; 400 max = irs_data->spi_min + irs_data->spi_range - 1; 401 if (spi_id >= min && spi_id <= max) 402 return irs_data; 403 } 404 405 return NULL; 406 } 407 408 static int gicv5_irs_wait_for_spi_op(struct gicv5_irs_chip_data *irs_data) 409 { 410 u32 statusr; 411 int ret; 412 413 ret = gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_SPI_STATUSR, 414 GICV5_IRS_SPI_STATUSR_IDLE, &statusr); 415 if (ret) 416 return ret; 417 418 return !!FIELD_GET(GICV5_IRS_SPI_STATUSR_V, statusr) ? 0 : -EIO; 419 } 420 421 static int gicv5_irs_wait_for_irs_pe(struct gicv5_irs_chip_data *irs_data, 422 bool selr) 423 { 424 bool valid = true; 425 u32 statusr; 426 int ret; 427 428 ret = gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_PE_STATUSR, 429 GICV5_IRS_PE_STATUSR_IDLE, &statusr); 430 if (ret) 431 return ret; 432 433 if (selr) 434 valid = !!FIELD_GET(GICV5_IRS_PE_STATUSR_V, statusr); 435 436 return valid ? 0 : -EIO; 437 } 438 439 static int gicv5_irs_wait_for_pe_selr(struct gicv5_irs_chip_data *irs_data) 440 { 441 return gicv5_irs_wait_for_irs_pe(irs_data, true); 442 } 443 444 static int gicv5_irs_wait_for_pe_cr0(struct gicv5_irs_chip_data *irs_data) 445 { 446 return gicv5_irs_wait_for_irs_pe(irs_data, false); 447 } 448 449 int gicv5_spi_irq_set_type(struct irq_data *d, unsigned int type) 450 { 451 struct gicv5_irs_chip_data *irs_data = d->chip_data; 452 u32 selr, cfgr; 453 bool level; 454 int ret; 455 456 /* 457 * There is no distinction between HIGH/LOW for level IRQs 458 * and RISING/FALLING for edge IRQs in the architecture, 459 * hence consider them equivalent. 460 */ 461 switch (type) { 462 case IRQ_TYPE_EDGE_RISING: 463 case IRQ_TYPE_EDGE_FALLING: 464 level = false; 465 break; 466 case IRQ_TYPE_LEVEL_HIGH: 467 case IRQ_TYPE_LEVEL_LOW: 468 level = true; 469 break; 470 default: 471 return -EINVAL; 472 } 473 474 guard(raw_spinlock)(&irs_data->spi_config_lock); 475 476 selr = FIELD_PREP(GICV5_IRS_SPI_SELR_ID, d->hwirq); 477 irs_writel_relaxed(irs_data, selr, GICV5_IRS_SPI_SELR); 478 ret = gicv5_irs_wait_for_spi_op(irs_data); 479 if (ret) 480 return ret; 481 482 cfgr = FIELD_PREP(GICV5_IRS_SPI_CFGR_TM, level); 483 irs_writel_relaxed(irs_data, cfgr, GICV5_IRS_SPI_CFGR); 484 485 return gicv5_irs_wait_for_spi_op(irs_data); 486 } 487 488 static int gicv5_irs_wait_for_idle(struct gicv5_irs_chip_data *irs_data) 489 { 490 return gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_CR0, 491 GICV5_IRS_CR0_IDLE, NULL); 492 } 493 494 static void __init gicv5_irs_disable(struct gicv5_irs_chip_data *irs_data) 495 { 496 irs_writel_relaxed(irs_data, 0, GICV5_IRS_CR0); 497 gicv5_irs_wait_for_idle(irs_data); 498 } 499 500 void gicv5_irs_syncr(void) 501 { 502 struct gicv5_irs_chip_data *irs_data; 503 u32 syncr; 504 505 irs_data = list_first_entry_or_null(&irs_nodes, struct gicv5_irs_chip_data, entry); 506 if (WARN_ON_ONCE(!irs_data)) 507 return; 508 509 syncr = FIELD_PREP(GICV5_IRS_SYNCR_SYNC, 1); 510 irs_writel_relaxed(irs_data, syncr, GICV5_IRS_SYNCR); 511 512 gicv5_wait_for_op(irs_data->irs_base, GICV5_IRS_SYNC_STATUSR, 513 GICV5_IRS_SYNC_STATUSR_IDLE); 514 } 515 516 int gicv5_irs_register_cpu(int cpuid) 517 { 518 struct gicv5_irs_chip_data *irs_data; 519 u32 selr, cr0; 520 u16 iaffid; 521 int ret; 522 523 ret = gicv5_irs_cpu_to_iaffid(cpuid, &iaffid); 524 if (ret) { 525 pr_err("IAFFID for CPU %d has not been initialised\n", cpuid); 526 return ret; 527 } 528 529 irs_data = per_cpu(per_cpu_irs_data, cpuid); 530 if (!irs_data) { 531 pr_err("No IRS associated with CPU %u\n", cpuid); 532 return -ENXIO; 533 } 534 535 selr = FIELD_PREP(GICV5_IRS_PE_SELR_IAFFID, iaffid); 536 irs_writel_relaxed(irs_data, selr, GICV5_IRS_PE_SELR); 537 538 ret = gicv5_irs_wait_for_pe_selr(irs_data); 539 if (ret) { 540 pr_err("IAFFID 0x%x used in IRS_PE_SELR is invalid\n", iaffid); 541 return -ENXIO; 542 } 543 544 cr0 = FIELD_PREP(GICV5_IRS_PE_CR0_DPS, 0x1); 545 irs_writel_relaxed(irs_data, cr0, GICV5_IRS_PE_CR0); 546 547 ret = gicv5_irs_wait_for_pe_cr0(irs_data); 548 if (ret) 549 return ret; 550 551 pr_debug("CPU %d enabled PE IAFFID 0x%x\n", cpuid, iaffid); 552 553 return 0; 554 } 555 556 static void __init gicv5_irs_init_bases(struct gicv5_irs_chip_data *irs_data, 557 void __iomem *irs_base, 558 bool noncoherent) 559 { 560 u32 cr0, cr1; 561 562 irs_data->irs_base = irs_base; 563 564 if (noncoherent) { 565 /* 566 * A non-coherent IRS implies that some cache levels cannot be 567 * used coherently by the cores and GIC. Our only option is to mark 568 * memory attributes for the GIC as non-cacheable; by default, 569 * non-cacheable memory attributes imply outer-shareable 570 * shareability, the value written into IRS_CR1_SH is ignored. 571 */ 572 cr1 = FIELD_PREP(GICV5_IRS_CR1_VPED_WA, GICV5_NO_WRITE_ALLOC) | 573 FIELD_PREP(GICV5_IRS_CR1_VPED_RA, GICV5_NO_READ_ALLOC) | 574 FIELD_PREP(GICV5_IRS_CR1_VMD_WA, GICV5_NO_WRITE_ALLOC) | 575 FIELD_PREP(GICV5_IRS_CR1_VMD_RA, GICV5_NO_READ_ALLOC) | 576 FIELD_PREP(GICV5_IRS_CR1_VPET_RA, GICV5_NO_READ_ALLOC) | 577 FIELD_PREP(GICV5_IRS_CR1_VMT_RA, GICV5_NO_READ_ALLOC) | 578 FIELD_PREP(GICV5_IRS_CR1_IST_WA, GICV5_NO_WRITE_ALLOC) | 579 FIELD_PREP(GICV5_IRS_CR1_IST_RA, GICV5_NO_READ_ALLOC) | 580 FIELD_PREP(GICV5_IRS_CR1_IC, GICV5_NON_CACHE) | 581 FIELD_PREP(GICV5_IRS_CR1_OC, GICV5_NON_CACHE); 582 irs_data->flags |= IRS_FLAGS_NON_COHERENT; 583 } else { 584 cr1 = FIELD_PREP(GICV5_IRS_CR1_VPED_WA, GICV5_WRITE_ALLOC) | 585 FIELD_PREP(GICV5_IRS_CR1_VPED_RA, GICV5_READ_ALLOC) | 586 FIELD_PREP(GICV5_IRS_CR1_VMD_WA, GICV5_WRITE_ALLOC) | 587 FIELD_PREP(GICV5_IRS_CR1_VMD_RA, GICV5_READ_ALLOC) | 588 FIELD_PREP(GICV5_IRS_CR1_VPET_RA, GICV5_READ_ALLOC) | 589 FIELD_PREP(GICV5_IRS_CR1_VMT_RA, GICV5_READ_ALLOC) | 590 FIELD_PREP(GICV5_IRS_CR1_IST_WA, GICV5_WRITE_ALLOC) | 591 FIELD_PREP(GICV5_IRS_CR1_IST_RA, GICV5_READ_ALLOC) | 592 FIELD_PREP(GICV5_IRS_CR1_IC, GICV5_WB_CACHE) | 593 FIELD_PREP(GICV5_IRS_CR1_OC, GICV5_WB_CACHE) | 594 FIELD_PREP(GICV5_IRS_CR1_SH, GICV5_INNER_SHARE); 595 } 596 597 irs_writel_relaxed(irs_data, cr1, GICV5_IRS_CR1); 598 599 cr0 = FIELD_PREP(GICV5_IRS_CR0_IRSEN, 0x1); 600 irs_writel_relaxed(irs_data, cr0, GICV5_IRS_CR0); 601 gicv5_irs_wait_for_idle(irs_data); 602 } 603 604 static int __init gicv5_irs_of_init_affinity(struct device_node *node, 605 struct gicv5_irs_chip_data *irs_data, 606 u8 iaffid_bits) 607 { 608 /* 609 * Detect IAFFID<->CPU mappings from the device tree and 610 * record IRS<->CPU topology information. 611 */ 612 u16 iaffid_mask = GENMASK(iaffid_bits - 1, 0); 613 int ret, i, ncpus, niaffids; 614 615 /* An IRS is architecturally allowed to have no cpus associated with it */ 616 if (!of_property_present(node, "cpus") && !of_property_present(node, "arm,iaffids")) 617 return 0; 618 619 ncpus = of_count_phandle_with_args(node, "cpus", NULL); 620 if (ncpus < 0) 621 return -EINVAL; 622 623 niaffids = of_property_count_elems_of_size(node, "arm,iaffids", 624 sizeof(u16)); 625 if (niaffids != ncpus) 626 return -EINVAL; 627 628 u16 *iaffids __free(kfree) = kzalloc_objs(*iaffids, niaffids); 629 if (!iaffids) 630 return -ENOMEM; 631 632 ret = of_property_read_u16_array(node, "arm,iaffids", iaffids, niaffids); 633 if (ret) 634 return ret; 635 636 for (i = 0; i < ncpus; i++) { 637 struct device_node *cpu_node; 638 int cpu; 639 640 cpu_node = of_parse_phandle(node, "cpus", i); 641 if (!cpu_node) { 642 pr_warn(FW_BUG "Erroneous CPU node phandle\n"); 643 continue; 644 } 645 646 cpu = of_cpu_node_to_id(cpu_node); 647 of_node_put(cpu_node); 648 if (cpu < 0) 649 continue; 650 651 if (iaffids[i] & ~iaffid_mask) { 652 pr_warn("CPU %d iaffid 0x%x exceeds IRS iaffid bits\n", 653 cpu, iaffids[i]); 654 continue; 655 } 656 657 per_cpu(cpu_iaffid, cpu).iaffid = iaffids[i]; 658 per_cpu(cpu_iaffid, cpu).valid = true; 659 660 /* We also know that the CPU is connected to this IRS */ 661 per_cpu(per_cpu_irs_data, cpu) = irs_data; 662 } 663 664 return ret; 665 } 666 667 static void __init gicv5_irs_clear_affinity(struct gicv5_irs_chip_data *irs_data) 668 { 669 int cpu; 670 671 for_each_possible_cpu(cpu) { 672 if (per_cpu(per_cpu_irs_data, cpu) == irs_data) { 673 per_cpu(cpu_iaffid, cpu).valid = false; 674 per_cpu(per_cpu_irs_data, cpu) = NULL; 675 } 676 } 677 } 678 679 static void irs_setup_pri_bits(u32 idr1) 680 { 681 switch (FIELD_GET(GICV5_IRS_IDR1_PRIORITY_BITS, idr1)) { 682 case GICV5_IRS_IDR1_PRIORITY_BITS_1BITS: 683 gicv5_global_data.irs_pri_bits = 1; 684 break; 685 case GICV5_IRS_IDR1_PRIORITY_BITS_2BITS: 686 gicv5_global_data.irs_pri_bits = 2; 687 break; 688 case GICV5_IRS_IDR1_PRIORITY_BITS_3BITS: 689 gicv5_global_data.irs_pri_bits = 3; 690 break; 691 case GICV5_IRS_IDR1_PRIORITY_BITS_4BITS: 692 gicv5_global_data.irs_pri_bits = 4; 693 break; 694 case GICV5_IRS_IDR1_PRIORITY_BITS_5BITS: 695 gicv5_global_data.irs_pri_bits = 5; 696 break; 697 default: 698 pr_warn("Detected wrong IDR priority bits value 0x%lx\n", 699 FIELD_GET(GICV5_IRS_IDR1_PRIORITY_BITS, idr1)); 700 gicv5_global_data.irs_pri_bits = 1; 701 break; 702 } 703 } 704 705 static int __init gicv5_irs_init(struct gicv5_irs_chip_data *irs_data) 706 { 707 u32 spi_count, idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR2); 708 709 if (WARN(!FIELD_GET(GICV5_IRS_IDR2_LPI, idr), 710 "LPI support not available - no IPIs, can't proceed\n")) { 711 return -ENODEV; 712 } 713 714 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR7); 715 irs_data->spi_min = FIELD_GET(GICV5_IRS_IDR7_SPI_BASE, idr); 716 717 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR6); 718 irs_data->spi_range = FIELD_GET(GICV5_IRS_IDR6_SPI_IRS_RANGE, idr); 719 720 /* 721 * Do the global setting only on the first IRS. 722 * Global properties (iaffid_bits, global spi count) are guaranteed to 723 * be consistent across IRSes by the architecture. 724 */ 725 if (list_empty(&irs_nodes)) { 726 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR0); 727 gicv5_global_data.virt_capable = !!FIELD_GET(GICV5_IRS_IDR0_VIRT, idr); 728 729 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR1); 730 irs_setup_pri_bits(idr); 731 732 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR5); 733 734 spi_count = FIELD_GET(GICV5_IRS_IDR5_SPI_RANGE, idr); 735 gicv5_global_data.global_spi_count = spi_count; 736 737 gicv5_init_lpi_domain(); 738 739 pr_debug("Detected %u SPIs globally\n", spi_count); 740 } 741 742 list_add_tail(&irs_data->entry, &irs_nodes); 743 744 return 0; 745 } 746 747 static int __init gicv5_irs_of_init(struct device_node *node) 748 { 749 struct gicv5_irs_chip_data *irs_data; 750 struct resource res; 751 void __iomem *irs_base; 752 u8 iaffid_bits; 753 u32 idr; 754 int ret; 755 756 irs_data = kzalloc_obj(*irs_data); 757 if (!irs_data) 758 return -ENOMEM; 759 760 raw_spin_lock_init(&irs_data->spi_config_lock); 761 762 ret = of_property_match_string(node, "reg-names", "ns-config"); 763 if (ret < 0) { 764 pr_err("%pOF: ns-config reg-name not present\n", node); 765 goto out_err; 766 } 767 768 if (of_address_to_resource(node, ret, &res)) { 769 ret = -EINVAL; 770 goto out_err; 771 } 772 773 irs_base = of_io_request_and_map(node, ret, of_node_full_name(node)); 774 if (IS_ERR(irs_base)) { 775 pr_err("%pOF: unable to map GICv5 IRS registers\n", node); 776 ret = PTR_ERR(irs_base); 777 goto out_err; 778 } 779 780 irs_data->fwnode = of_fwnode_handle(node); 781 irs_data->res = res; 782 gicv5_irs_init_bases(irs_data, irs_base, of_property_read_bool(node, "dma-noncoherent")); 783 784 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR1); 785 iaffid_bits = FIELD_GET(GICV5_IRS_IDR1_IAFFID_BITS, idr) + 1; 786 787 ret = gicv5_irs_of_init_affinity(node, irs_data, iaffid_bits); 788 if (ret) { 789 pr_err("Failed to parse CPU IAFFIDs from the device tree!\n"); 790 goto out_iomem; 791 } 792 793 ret = gicv5_irs_init(irs_data); 794 if (ret) 795 goto out_iomem; 796 797 if (irs_data->spi_range) { 798 pr_info("%s detected SPI range [%u-%u]\n", 799 of_node_full_name(node), 800 irs_data->spi_min, 801 irs_data->spi_min + 802 irs_data->spi_range - 1); 803 } 804 805 return ret; 806 807 out_iomem: 808 gicv5_irs_clear_affinity(irs_data); 809 gicv5_irs_disable(irs_data); 810 iounmap(irs_base); 811 release_mem_region(res.start, resource_size(&res)); 812 out_err: 813 kfree(irs_data); 814 return ret; 815 } 816 817 void __init gicv5_irs_remove(void) 818 { 819 struct gicv5_irs_chip_data *irs_data, *tmp_data; 820 821 gicv5_free_lpi_domain(); 822 gicv5_deinit_lpis(); 823 824 list_for_each_entry_safe(irs_data, tmp_data, &irs_nodes, entry) { 825 gicv5_irs_clear_affinity(irs_data); 826 gicv5_irs_disable(irs_data); 827 iounmap(irs_data->irs_base); 828 release_mem_region(irs_data->res.start, resource_size(&irs_data->res)); 829 list_del(&irs_data->entry); 830 kfree(irs_data); 831 } 832 } 833 834 int __init gicv5_irs_enable(void) 835 { 836 struct gicv5_irs_chip_data *irs_data; 837 int ret; 838 839 irs_data = list_first_entry_or_null(&irs_nodes, 840 struct gicv5_irs_chip_data, entry); 841 if (!irs_data) 842 return -ENODEV; 843 844 ret = gicv5_irs_init_ist(irs_data); 845 if (ret) { 846 pr_err("Failed to init IST\n"); 847 return ret; 848 } 849 850 return 0; 851 } 852 853 void __init gicv5_irs_its_probe(void) 854 { 855 struct gicv5_irs_chip_data *irs_data; 856 857 if (acpi_disabled) 858 list_for_each_entry(irs_data, &irs_nodes, entry) 859 gicv5_its_of_probe(to_of_node(irs_data->fwnode)); 860 else 861 gicv5_its_acpi_probe(); 862 } 863 864 int __init gicv5_irs_of_probe(struct device_node *parent) 865 { 866 struct device_node *np; 867 int ret; 868 869 for_each_available_child_of_node(parent, np) { 870 if (!of_device_is_compatible(np, "arm,gic-v5-irs")) 871 continue; 872 873 ret = gicv5_irs_of_init(np); 874 if (ret) 875 pr_err("Failed to init IRS %s\n", np->full_name); 876 } 877 878 return list_empty(&irs_nodes) ? -ENODEV : 0; 879 } 880 881 #ifdef CONFIG_ACPI 882 883 #define ACPI_GICV5_IRS_MEM_SIZE (SZ_64K) 884 static struct gicv5_irs_chip_data *current_irs_data __initdata; 885 static int current_irsid __initdata = -1; 886 static u8 current_iaffid_bits __initdata; 887 888 static int __init gic_acpi_parse_iaffid(union acpi_subtable_headers *header, 889 const unsigned long end) 890 { 891 struct acpi_madt_generic_interrupt *gicc = (struct acpi_madt_generic_interrupt *)header; 892 int cpu; 893 894 if (!(gicc->flags & (ACPI_MADT_ENABLED | ACPI_MADT_GICC_ONLINE_CAPABLE))) 895 return 0; 896 897 if (gicc->irs_id != current_irsid) 898 return 0; 899 900 cpu = get_logical_index(gicc->arm_mpidr); 901 if (cpu < 0) 902 return 0; 903 904 if (gicc->iaffid & ~GENMASK(current_iaffid_bits - 1, 0)) { 905 pr_warn("CPU %d iaffid 0x%x exceeds IRS iaffid bits\n", cpu, gicc->iaffid); 906 return 0; 907 } 908 909 /* Bind the IAFFID and the CPU */ 910 per_cpu(cpu_iaffid, cpu).iaffid = gicc->iaffid; 911 per_cpu(cpu_iaffid, cpu).valid = true; 912 pr_debug("Processed IAFFID %u for CPU%d", per_cpu(cpu_iaffid, cpu).iaffid, cpu); 913 914 /* We also know that the CPU is connected to this IRS */ 915 per_cpu(per_cpu_irs_data, cpu) = current_irs_data; 916 917 return 0; 918 } 919 920 static int __init gicv5_irs_acpi_init_affinity(u32 irsid, struct gicv5_irs_chip_data *irs_data) 921 { 922 u32 idr; 923 924 current_irsid = irsid; 925 current_irs_data = irs_data; 926 927 idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR1); 928 current_iaffid_bits = FIELD_GET(GICV5_IRS_IDR1_IAFFID_BITS, idr) + 1; 929 930 acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT, gic_acpi_parse_iaffid, 0); 931 932 return 0; 933 } 934 935 static struct resource * __init gic_request_region(resource_size_t base, resource_size_t size, 936 const char *name) 937 { 938 struct resource *r = request_mem_region(base, size, name); 939 940 if (!r) 941 pr_warn_once(FW_BUG "%s region %pa has overlapping address\n", name, &base); 942 943 return r; 944 } 945 946 static int __init gic_acpi_parse_madt_irs(union acpi_subtable_headers *header, 947 const unsigned long end) 948 { 949 struct acpi_madt_gicv5_irs *irs = (struct acpi_madt_gicv5_irs *)header; 950 struct gicv5_irs_chip_data *irs_data; 951 void __iomem *irs_base; 952 struct resource *r; 953 int ret; 954 955 /* Per-IRS data structure */ 956 irs_data = kzalloc_obj(*irs_data); 957 if (!irs_data) 958 return -ENOMEM; 959 960 /* This spinlock is used for SPI config changes */ 961 raw_spin_lock_init(&irs_data->spi_config_lock); 962 963 r = gic_request_region(irs->config_base_address, ACPI_GICV5_IRS_MEM_SIZE, "GICv5 IRS"); 964 if (!r) { 965 ret = -EBUSY; 966 goto out_free; 967 } 968 969 irs_base = ioremap(irs->config_base_address, ACPI_GICV5_IRS_MEM_SIZE); 970 if (!irs_base) { 971 pr_err("Unable to map GIC IRS registers\n"); 972 ret = -ENOMEM; 973 goto out_release; 974 } 975 976 gicv5_irs_init_bases(irs_data, irs_base, irs->flags & ACPI_MADT_IRS_NON_COHERENT); 977 irs_data->res = *r; 978 979 gicv5_irs_acpi_init_affinity(irs->irs_id, irs_data); 980 981 ret = gicv5_irs_init(irs_data); 982 if (ret) 983 goto out_map; 984 985 if (irs_data->spi_range) { 986 pr_info("%s @%llx detected SPI range [%u-%u]\n", "IRS", irs->config_base_address, 987 irs_data->spi_min, 988 irs_data->spi_min + 989 irs_data->spi_range - 1); 990 } 991 992 return 0; 993 994 out_map: 995 gicv5_irs_clear_affinity(irs_data); 996 gicv5_irs_disable(irs_data); 997 iounmap(irs_base); 998 out_release: 999 release_mem_region(r->start, resource_size(r)); 1000 out_free: 1001 kfree(irs_data); 1002 return ret; 1003 } 1004 1005 int __init gicv5_irs_acpi_probe(void) 1006 { 1007 acpi_table_parse_madt(ACPI_MADT_TYPE_GICV5_IRS, gic_acpi_parse_madt_irs, 0); 1008 1009 return list_empty(&irs_nodes) ? -ENODEV : 0; 1010 } 1011 #endif 1012