1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2024-2025 ARM Limited, All Rights Reserved. 4 */ 5 6 #define pr_fmt(fmt) "GICv5: " fmt 7 8 #include <linux/acpi_iort.h> 9 #include <linux/cpuhotplug.h> 10 #include <linux/idr.h> 11 #include <linux/irqdomain.h> 12 #include <linux/slab.h> 13 #include <linux/wordpart.h> 14 15 #include <linux/irqchip.h> 16 #include <linux/irqchip/arm-gic-v5.h> 17 #include <linux/irqchip/arm-vgic-info.h> 18 19 #include <asm/cpufeature.h> 20 #include <asm/exception.h> 21 22 static u8 pri_bits __ro_after_init = 5; 23 24 #define GICV5_IRQ_PRI_MASK 0x1f 25 #define GICV5_IRQ_PRI_MI (GICV5_IRQ_PRI_MASK & GENMASK(4, 5 - pri_bits)) 26 27 #define PPI_NR 128 28 29 static bool gicv5_cpuif_has_gcie(void) 30 { 31 return this_cpu_has_cap(ARM64_HAS_GICV5_CPUIF); 32 } 33 34 struct gicv5_chip_data gicv5_global_data __read_mostly; 35 36 static DEFINE_IDA(lpi_ida); 37 static u32 num_lpis __ro_after_init; 38 39 void __init gicv5_init_lpis(u32 lpis) 40 { 41 num_lpis = lpis; 42 } 43 44 void __init gicv5_deinit_lpis(void) 45 { 46 num_lpis = 0; 47 } 48 49 static int alloc_lpi(void) 50 { 51 if (!num_lpis) 52 return -ENOSPC; 53 54 return ida_alloc_max(&lpi_ida, num_lpis - 1, GFP_KERNEL); 55 } 56 57 static void release_lpi(u32 lpi) 58 { 59 ida_free(&lpi_ida, lpi); 60 } 61 62 int gicv5_alloc_lpi(void) 63 { 64 return alloc_lpi(); 65 } 66 67 void gicv5_free_lpi(u32 lpi) 68 { 69 release_lpi(lpi); 70 } 71 72 static void gicv5_ppi_priority_init(void) 73 { 74 write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR0_EL1); 75 write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR1_EL1); 76 write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR2_EL1); 77 write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR3_EL1); 78 write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR4_EL1); 79 write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR5_EL1); 80 write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR6_EL1); 81 write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR7_EL1); 82 write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR8_EL1); 83 write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR9_EL1); 84 write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR10_EL1); 85 write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR11_EL1); 86 write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR12_EL1); 87 write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR13_EL1); 88 write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR14_EL1); 89 write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR15_EL1); 90 91 /* 92 * Context syncronization required to make sure system register writes 93 * effects are synchronised. 94 */ 95 isb(); 96 } 97 98 static void gicv5_hwirq_init(irq_hw_number_t hwirq, u8 priority, u8 hwirq_type) 99 { 100 u64 cdpri, cdaff; 101 u16 iaffid; 102 int ret; 103 104 if (hwirq_type == GICV5_HWIRQ_TYPE_LPI || hwirq_type == GICV5_HWIRQ_TYPE_SPI) { 105 cdpri = FIELD_PREP(GICV5_GIC_CDPRI_PRIORITY_MASK, priority) | 106 FIELD_PREP(GICV5_GIC_CDPRI_TYPE_MASK, hwirq_type) | 107 FIELD_PREP(GICV5_GIC_CDPRI_ID_MASK, hwirq); 108 gic_insn(cdpri, CDPRI); 109 110 ret = gicv5_irs_cpu_to_iaffid(smp_processor_id(), &iaffid); 111 112 if (WARN_ON_ONCE(ret)) 113 return; 114 115 cdaff = FIELD_PREP(GICV5_GIC_CDAFF_IAFFID_MASK, iaffid) | 116 FIELD_PREP(GICV5_GIC_CDAFF_TYPE_MASK, hwirq_type) | 117 FIELD_PREP(GICV5_GIC_CDAFF_ID_MASK, hwirq); 118 gic_insn(cdaff, CDAFF); 119 } 120 } 121 122 static void gicv5_ppi_irq_mask(struct irq_data *d) 123 { 124 u64 hwirq_id_bit = BIT_ULL(d->hwirq % 64); 125 126 if (d->hwirq < 64) 127 sysreg_clear_set_s(SYS_ICC_PPI_ENABLER0_EL1, hwirq_id_bit, 0); 128 else 129 sysreg_clear_set_s(SYS_ICC_PPI_ENABLER1_EL1, hwirq_id_bit, 0); 130 131 /* 132 * We must ensure that the disable takes effect immediately to 133 * guarantee that the lazy-disabled IRQ mechanism works. 134 * A context synchronization event is required to guarantee it. 135 * Reference: I_ZLTKB/R_YRGMH GICv5 specification - section 2.9.1. 136 */ 137 isb(); 138 } 139 140 static void gicv5_iri_irq_mask(struct irq_data *d, u8 hwirq_type) 141 { 142 u64 cddis; 143 144 cddis = FIELD_PREP(GICV5_GIC_CDDIS_ID_MASK, d->hwirq) | 145 FIELD_PREP(GICV5_GIC_CDDIS_TYPE_MASK, hwirq_type); 146 147 gic_insn(cddis, CDDIS); 148 /* 149 * We must make sure that GIC CDDIS write effects are propagated 150 * immediately to make sure the disable takes effect to guarantee 151 * that the lazy-disabled IRQ mechanism works. 152 * Rule R_XCLJC states that the effects of a GIC system instruction 153 * complete in finite time. 154 * The GSB ensures completion of the GIC instruction and prevents 155 * loads, stores and GIC instructions from executing part of their 156 * functionality before the GSB SYS. 157 */ 158 gsb_sys(); 159 } 160 161 static void gicv5_spi_irq_mask(struct irq_data *d) 162 { 163 gicv5_iri_irq_mask(d, GICV5_HWIRQ_TYPE_SPI); 164 } 165 166 static void gicv5_lpi_irq_mask(struct irq_data *d) 167 { 168 gicv5_iri_irq_mask(d, GICV5_HWIRQ_TYPE_LPI); 169 } 170 171 static void gicv5_ppi_irq_unmask(struct irq_data *d) 172 { 173 u64 hwirq_id_bit = BIT_ULL(d->hwirq % 64); 174 175 if (d->hwirq < 64) 176 sysreg_clear_set_s(SYS_ICC_PPI_ENABLER0_EL1, 0, hwirq_id_bit); 177 else 178 sysreg_clear_set_s(SYS_ICC_PPI_ENABLER1_EL1, 0, hwirq_id_bit); 179 /* 180 * We must ensure that the enable takes effect in finite time - a 181 * context synchronization event is required to guarantee it, we 182 * can not take for granted that would happen (eg a core going straight 183 * into idle after enabling a PPI). 184 * Reference: I_ZLTKB/R_YRGMH GICv5 specification - section 2.9.1. 185 */ 186 isb(); 187 } 188 189 static void gicv5_iri_irq_unmask(struct irq_data *d, u8 hwirq_type) 190 { 191 u64 cden; 192 193 cden = FIELD_PREP(GICV5_GIC_CDEN_ID_MASK, d->hwirq) | 194 FIELD_PREP(GICV5_GIC_CDEN_TYPE_MASK, hwirq_type); 195 /* 196 * Rule R_XCLJC states that the effects of a GIC system instruction 197 * complete in finite time and that's the only requirement when 198 * unmasking an SPI/LPI IRQ. 199 */ 200 gic_insn(cden, CDEN); 201 } 202 203 static void gicv5_spi_irq_unmask(struct irq_data *d) 204 { 205 gicv5_iri_irq_unmask(d, GICV5_HWIRQ_TYPE_SPI); 206 } 207 208 static void gicv5_lpi_irq_unmask(struct irq_data *d) 209 { 210 gicv5_iri_irq_unmask(d, GICV5_HWIRQ_TYPE_LPI); 211 } 212 213 static void gicv5_hwirq_eoi(u32 hwirq_id, u8 hwirq_type) 214 { 215 u64 cddi; 216 217 cddi = FIELD_PREP(GICV5_GIC_CDDI_ID_MASK, hwirq_id) | 218 FIELD_PREP(GICV5_GIC_CDDI_TYPE_MASK, hwirq_type); 219 220 gic_insn(cddi, CDDI); 221 222 gic_insn(0, CDEOI); 223 } 224 225 static void gicv5_ppi_irq_eoi(struct irq_data *d) 226 { 227 /* Skip deactivate for forwarded PPI interrupts */ 228 if (irqd_is_forwarded_to_vcpu(d)) { 229 gic_insn(0, CDEOI); 230 return; 231 } 232 233 gicv5_hwirq_eoi(d->hwirq, GICV5_HWIRQ_TYPE_PPI); 234 } 235 236 static void gicv5_spi_irq_eoi(struct irq_data *d) 237 { 238 gicv5_hwirq_eoi(d->hwirq, GICV5_HWIRQ_TYPE_SPI); 239 } 240 241 static void gicv5_lpi_irq_eoi(struct irq_data *d) 242 { 243 gicv5_hwirq_eoi(d->hwirq, GICV5_HWIRQ_TYPE_LPI); 244 } 245 246 static int gicv5_iri_irq_set_affinity(struct irq_data *d, 247 const struct cpumask *mask_val, 248 bool force, u8 hwirq_type) 249 { 250 int ret, cpuid; 251 u16 iaffid; 252 u64 cdaff; 253 254 if (force) 255 cpuid = cpumask_first(mask_val); 256 else 257 cpuid = cpumask_any_and(mask_val, cpu_online_mask); 258 259 ret = gicv5_irs_cpu_to_iaffid(cpuid, &iaffid); 260 if (ret) 261 return ret; 262 263 cdaff = FIELD_PREP(GICV5_GIC_CDAFF_IAFFID_MASK, iaffid) | 264 FIELD_PREP(GICV5_GIC_CDAFF_TYPE_MASK, hwirq_type) | 265 FIELD_PREP(GICV5_GIC_CDAFF_ID_MASK, d->hwirq); 266 gic_insn(cdaff, CDAFF); 267 268 irq_data_update_effective_affinity(d, cpumask_of(cpuid)); 269 270 return IRQ_SET_MASK_OK_DONE; 271 } 272 273 static int gicv5_spi_irq_set_affinity(struct irq_data *d, 274 const struct cpumask *mask_val, 275 bool force) 276 { 277 return gicv5_iri_irq_set_affinity(d, mask_val, force, 278 GICV5_HWIRQ_TYPE_SPI); 279 } 280 281 static int gicv5_lpi_irq_set_affinity(struct irq_data *d, 282 const struct cpumask *mask_val, 283 bool force) 284 { 285 return gicv5_iri_irq_set_affinity(d, mask_val, force, 286 GICV5_HWIRQ_TYPE_LPI); 287 } 288 289 enum ppi_reg { 290 PPI_PENDING, 291 PPI_ACTIVE, 292 PPI_HM 293 }; 294 295 static __always_inline u64 read_ppi_sysreg_s(unsigned int irq, 296 const enum ppi_reg which) 297 { 298 switch (which) { 299 case PPI_PENDING: 300 return irq < 64 ? read_sysreg_s(SYS_ICC_PPI_SPENDR0_EL1) : 301 read_sysreg_s(SYS_ICC_PPI_SPENDR1_EL1); 302 case PPI_ACTIVE: 303 return irq < 64 ? read_sysreg_s(SYS_ICC_PPI_SACTIVER0_EL1) : 304 read_sysreg_s(SYS_ICC_PPI_SACTIVER1_EL1); 305 case PPI_HM: 306 return irq < 64 ? read_sysreg_s(SYS_ICC_PPI_HMR0_EL1) : 307 read_sysreg_s(SYS_ICC_PPI_HMR1_EL1); 308 default: 309 BUILD_BUG_ON(1); 310 } 311 } 312 313 static __always_inline void write_ppi_sysreg_s(unsigned int irq, bool set, 314 const enum ppi_reg which) 315 { 316 u64 bit = BIT_ULL(irq % 64); 317 318 switch (which) { 319 case PPI_PENDING: 320 if (set) { 321 if (irq < 64) 322 write_sysreg_s(bit, SYS_ICC_PPI_SPENDR0_EL1); 323 else 324 write_sysreg_s(bit, SYS_ICC_PPI_SPENDR1_EL1); 325 } else { 326 if (irq < 64) 327 write_sysreg_s(bit, SYS_ICC_PPI_CPENDR0_EL1); 328 else 329 write_sysreg_s(bit, SYS_ICC_PPI_CPENDR1_EL1); 330 } 331 return; 332 case PPI_ACTIVE: 333 if (set) { 334 if (irq < 64) 335 write_sysreg_s(bit, SYS_ICC_PPI_SACTIVER0_EL1); 336 else 337 write_sysreg_s(bit, SYS_ICC_PPI_SACTIVER1_EL1); 338 } else { 339 if (irq < 64) 340 write_sysreg_s(bit, SYS_ICC_PPI_CACTIVER0_EL1); 341 else 342 write_sysreg_s(bit, SYS_ICC_PPI_CACTIVER1_EL1); 343 } 344 return; 345 default: 346 BUILD_BUG_ON(1); 347 } 348 } 349 350 static int gicv5_ppi_irq_get_irqchip_state(struct irq_data *d, 351 enum irqchip_irq_state which, 352 bool *state) 353 { 354 u64 hwirq_id_bit = BIT_ULL(d->hwirq % 64); 355 356 switch (which) { 357 case IRQCHIP_STATE_PENDING: 358 *state = !!(read_ppi_sysreg_s(d->hwirq, PPI_PENDING) & hwirq_id_bit); 359 return 0; 360 case IRQCHIP_STATE_ACTIVE: 361 *state = !!(read_ppi_sysreg_s(d->hwirq, PPI_ACTIVE) & hwirq_id_bit); 362 return 0; 363 default: 364 pr_debug("Unexpected PPI irqchip state\n"); 365 return -EINVAL; 366 } 367 } 368 369 static int gicv5_iri_irq_get_irqchip_state(struct irq_data *d, 370 enum irqchip_irq_state which, 371 bool *state, u8 hwirq_type) 372 { 373 u64 icsr, cdrcfg; 374 375 cdrcfg = d->hwirq | FIELD_PREP(GICV5_GIC_CDRCFG_TYPE_MASK, hwirq_type); 376 377 gic_insn(cdrcfg, CDRCFG); 378 isb(); 379 icsr = read_sysreg_s(SYS_ICC_ICSR_EL1); 380 381 if (FIELD_GET(ICC_ICSR_EL1_F, icsr)) { 382 pr_err("ICSR_EL1 is invalid\n"); 383 return -EINVAL; 384 } 385 386 switch (which) { 387 case IRQCHIP_STATE_PENDING: 388 *state = !!(FIELD_GET(ICC_ICSR_EL1_Pending, icsr)); 389 return 0; 390 391 case IRQCHIP_STATE_ACTIVE: 392 *state = !!(FIELD_GET(ICC_ICSR_EL1_Active, icsr)); 393 return 0; 394 395 default: 396 pr_debug("Unexpected irqchip_irq_state\n"); 397 return -EINVAL; 398 } 399 } 400 401 static int gicv5_spi_irq_get_irqchip_state(struct irq_data *d, 402 enum irqchip_irq_state which, 403 bool *state) 404 { 405 return gicv5_iri_irq_get_irqchip_state(d, which, state, 406 GICV5_HWIRQ_TYPE_SPI); 407 } 408 409 static int gicv5_lpi_irq_get_irqchip_state(struct irq_data *d, 410 enum irqchip_irq_state which, 411 bool *state) 412 { 413 return gicv5_iri_irq_get_irqchip_state(d, which, state, 414 GICV5_HWIRQ_TYPE_LPI); 415 } 416 417 static int gicv5_ppi_irq_set_irqchip_state(struct irq_data *d, 418 enum irqchip_irq_state which, 419 bool state) 420 { 421 switch (which) { 422 case IRQCHIP_STATE_PENDING: 423 write_ppi_sysreg_s(d->hwirq, state, PPI_PENDING); 424 return 0; 425 case IRQCHIP_STATE_ACTIVE: 426 write_ppi_sysreg_s(d->hwirq, state, PPI_ACTIVE); 427 return 0; 428 default: 429 pr_debug("Unexpected PPI irqchip state\n"); 430 return -EINVAL; 431 } 432 } 433 434 static void gicv5_iri_irq_write_pending_state(struct irq_data *d, bool state, 435 u8 hwirq_type) 436 { 437 u64 cdpend; 438 439 cdpend = FIELD_PREP(GICV5_GIC_CDPEND_TYPE_MASK, hwirq_type) | 440 FIELD_PREP(GICV5_GIC_CDPEND_ID_MASK, d->hwirq) | 441 FIELD_PREP(GICV5_GIC_CDPEND_PENDING_MASK, state); 442 443 gic_insn(cdpend, CDPEND); 444 } 445 446 static void gicv5_spi_irq_write_pending_state(struct irq_data *d, bool state) 447 { 448 gicv5_iri_irq_write_pending_state(d, state, GICV5_HWIRQ_TYPE_SPI); 449 } 450 451 static void gicv5_lpi_irq_write_pending_state(struct irq_data *d, bool state) 452 { 453 gicv5_iri_irq_write_pending_state(d, state, GICV5_HWIRQ_TYPE_LPI); 454 } 455 456 static int gicv5_spi_irq_set_irqchip_state(struct irq_data *d, 457 enum irqchip_irq_state which, 458 bool state) 459 { 460 switch (which) { 461 case IRQCHIP_STATE_PENDING: 462 gicv5_spi_irq_write_pending_state(d, state); 463 break; 464 default: 465 pr_debug("Unexpected irqchip_irq_state\n"); 466 return -EINVAL; 467 } 468 469 return 0; 470 } 471 472 static int gicv5_lpi_irq_set_irqchip_state(struct irq_data *d, 473 enum irqchip_irq_state which, 474 bool state) 475 { 476 switch (which) { 477 case IRQCHIP_STATE_PENDING: 478 gicv5_lpi_irq_write_pending_state(d, state); 479 break; 480 481 default: 482 pr_debug("Unexpected irqchip_irq_state\n"); 483 return -EINVAL; 484 } 485 486 return 0; 487 } 488 489 static int gicv5_spi_irq_retrigger(struct irq_data *data) 490 { 491 return !gicv5_spi_irq_set_irqchip_state(data, IRQCHIP_STATE_PENDING, 492 true); 493 } 494 495 static int gicv5_lpi_irq_retrigger(struct irq_data *data) 496 { 497 return !gicv5_lpi_irq_set_irqchip_state(data, IRQCHIP_STATE_PENDING, 498 true); 499 } 500 501 static void gicv5_ipi_send_single(struct irq_data *d, unsigned int cpu) 502 { 503 /* Mark the LPI pending */ 504 irq_chip_retrigger_hierarchy(d); 505 } 506 507 static bool gicv5_ppi_irq_is_level(irq_hw_number_t hwirq) 508 { 509 u64 bit = BIT_ULL(hwirq % 64); 510 511 return !!(read_ppi_sysreg_s(hwirq, PPI_HM) & bit); 512 } 513 514 static int gicv5_ppi_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu) 515 { 516 if (vcpu) 517 irqd_set_forwarded_to_vcpu(d); 518 else 519 irqd_clr_forwarded_to_vcpu(d); 520 521 return 0; 522 } 523 524 static const struct irq_chip gicv5_ppi_irq_chip = { 525 .name = "GICv5-PPI", 526 .irq_mask = gicv5_ppi_irq_mask, 527 .irq_unmask = gicv5_ppi_irq_unmask, 528 .irq_eoi = gicv5_ppi_irq_eoi, 529 .irq_get_irqchip_state = gicv5_ppi_irq_get_irqchip_state, 530 .irq_set_irqchip_state = gicv5_ppi_irq_set_irqchip_state, 531 .irq_set_vcpu_affinity = gicv5_ppi_irq_set_vcpu_affinity, 532 .flags = IRQCHIP_SKIP_SET_WAKE | 533 IRQCHIP_MASK_ON_SUSPEND, 534 }; 535 536 static const struct irq_chip gicv5_spi_irq_chip = { 537 .name = "GICv5-SPI", 538 .irq_mask = gicv5_spi_irq_mask, 539 .irq_unmask = gicv5_spi_irq_unmask, 540 .irq_eoi = gicv5_spi_irq_eoi, 541 .irq_set_type = gicv5_spi_irq_set_type, 542 .irq_set_affinity = gicv5_spi_irq_set_affinity, 543 .irq_retrigger = gicv5_spi_irq_retrigger, 544 .irq_get_irqchip_state = gicv5_spi_irq_get_irqchip_state, 545 .irq_set_irqchip_state = gicv5_spi_irq_set_irqchip_state, 546 .flags = IRQCHIP_SET_TYPE_MASKED | 547 IRQCHIP_SKIP_SET_WAKE | 548 IRQCHIP_MASK_ON_SUSPEND, 549 }; 550 551 static const struct irq_chip gicv5_lpi_irq_chip = { 552 .name = "GICv5-LPI", 553 .irq_mask = gicv5_lpi_irq_mask, 554 .irq_unmask = gicv5_lpi_irq_unmask, 555 .irq_eoi = gicv5_lpi_irq_eoi, 556 .irq_set_affinity = gicv5_lpi_irq_set_affinity, 557 .irq_retrigger = gicv5_lpi_irq_retrigger, 558 .irq_get_irqchip_state = gicv5_lpi_irq_get_irqchip_state, 559 .irq_set_irqchip_state = gicv5_lpi_irq_set_irqchip_state, 560 .flags = IRQCHIP_SKIP_SET_WAKE | 561 IRQCHIP_MASK_ON_SUSPEND, 562 }; 563 564 static const struct irq_chip gicv5_ipi_irq_chip = { 565 .name = "GICv5-IPI", 566 .irq_mask = irq_chip_mask_parent, 567 .irq_unmask = irq_chip_unmask_parent, 568 .irq_eoi = irq_chip_eoi_parent, 569 .irq_set_affinity = irq_chip_set_affinity_parent, 570 .irq_get_irqchip_state = irq_chip_get_parent_state, 571 .irq_set_irqchip_state = irq_chip_set_parent_state, 572 .ipi_send_single = gicv5_ipi_send_single, 573 .flags = IRQCHIP_SKIP_SET_WAKE | 574 IRQCHIP_MASK_ON_SUSPEND, 575 }; 576 577 static __always_inline int gicv5_irq_domain_translate(struct irq_domain *d, 578 struct irq_fwspec *fwspec, 579 irq_hw_number_t *hwirq, 580 unsigned int *type, 581 const u8 hwirq_type) 582 { 583 unsigned int hwirq_trigger; 584 u8 fwspec_irq_type; 585 586 if (is_of_node(fwspec->fwnode)) { 587 588 if (fwspec->param_count < 3) 589 return -EINVAL; 590 591 fwspec_irq_type = fwspec->param[0]; 592 593 if (fwspec->param[0] != hwirq_type) 594 return -EINVAL; 595 596 *hwirq = fwspec->param[1]; 597 hwirq_trigger = fwspec->param[2]; 598 } 599 600 if (is_fwnode_irqchip(fwspec->fwnode)) { 601 602 if (fwspec->param_count != 2) 603 return -EINVAL; 604 605 fwspec_irq_type = FIELD_GET(GICV5_HWIRQ_TYPE, fwspec->param[0]); 606 607 if (fwspec_irq_type != hwirq_type) 608 return -EINVAL; 609 610 *hwirq = FIELD_GET(GICV5_HWIRQ_ID, fwspec->param[0]); 611 hwirq_trigger = fwspec->param[1]; 612 } 613 614 switch (hwirq_type) { 615 case GICV5_HWIRQ_TYPE_PPI: 616 /* 617 * Handling mode is hardcoded for PPIs, set the type using 618 * HW reported value. 619 */ 620 *type = gicv5_ppi_irq_is_level(*hwirq) ? IRQ_TYPE_LEVEL_LOW : 621 IRQ_TYPE_EDGE_RISING; 622 break; 623 case GICV5_HWIRQ_TYPE_SPI: 624 *type = hwirq_trigger & IRQ_TYPE_SENSE_MASK; 625 break; 626 default: 627 BUILD_BUG_ON(1); 628 } 629 630 return 0; 631 } 632 633 static int gicv5_irq_ppi_domain_translate(struct irq_domain *d, 634 struct irq_fwspec *fwspec, 635 irq_hw_number_t *hwirq, 636 unsigned int *type) 637 { 638 return gicv5_irq_domain_translate(d, fwspec, hwirq, type, 639 GICV5_HWIRQ_TYPE_PPI); 640 } 641 642 static int gicv5_irq_ppi_domain_alloc(struct irq_domain *domain, unsigned int virq, 643 unsigned int nr_irqs, void *arg) 644 { 645 unsigned int type = IRQ_TYPE_NONE; 646 struct irq_fwspec *fwspec = arg; 647 irq_hw_number_t hwirq; 648 int ret; 649 650 if (WARN_ON_ONCE(nr_irqs != 1)) 651 return -EINVAL; 652 653 ret = gicv5_irq_ppi_domain_translate(domain, fwspec, &hwirq, &type); 654 if (ret) 655 return ret; 656 657 if (type & IRQ_TYPE_LEVEL_MASK) 658 irq_set_status_flags(virq, IRQ_LEVEL); 659 660 irq_set_percpu_devid(virq); 661 irq_domain_set_info(domain, virq, hwirq, &gicv5_ppi_irq_chip, NULL, 662 handle_percpu_devid_irq, NULL, NULL); 663 664 return 0; 665 } 666 667 static void gicv5_irq_domain_free(struct irq_domain *domain, unsigned int virq, 668 unsigned int nr_irqs) 669 { 670 struct irq_data *d; 671 672 if (WARN_ON_ONCE(nr_irqs != 1)) 673 return; 674 675 d = irq_domain_get_irq_data(domain, virq); 676 677 irq_set_handler(virq, NULL); 678 irq_domain_reset_irq_data(d); 679 } 680 681 static int gicv5_irq_ppi_domain_select(struct irq_domain *d, struct irq_fwspec *fwspec, 682 enum irq_domain_bus_token bus_token) 683 { 684 u32 hwirq_type; 685 686 if (fwspec->fwnode != d->fwnode) 687 return 0; 688 689 if (is_of_node(fwspec->fwnode)) 690 hwirq_type = fwspec->param[0]; 691 692 if (is_fwnode_irqchip(fwspec->fwnode)) 693 hwirq_type = FIELD_GET(GICV5_HWIRQ_TYPE, fwspec->param[0]); 694 695 if (hwirq_type != GICV5_HWIRQ_TYPE_PPI) 696 return 0; 697 698 return (d == gicv5_global_data.ppi_domain); 699 } 700 701 static const struct irq_domain_ops gicv5_irq_ppi_domain_ops = { 702 .translate = gicv5_irq_ppi_domain_translate, 703 .alloc = gicv5_irq_ppi_domain_alloc, 704 .free = gicv5_irq_domain_free, 705 .select = gicv5_irq_ppi_domain_select 706 }; 707 708 static int gicv5_irq_spi_domain_translate(struct irq_domain *d, 709 struct irq_fwspec *fwspec, 710 irq_hw_number_t *hwirq, 711 unsigned int *type) 712 { 713 return gicv5_irq_domain_translate(d, fwspec, hwirq, type, 714 GICV5_HWIRQ_TYPE_SPI); 715 } 716 717 static int gicv5_irq_spi_domain_alloc(struct irq_domain *domain, unsigned int virq, 718 unsigned int nr_irqs, void *arg) 719 { 720 struct gicv5_irs_chip_data *chip_data; 721 unsigned int type = IRQ_TYPE_NONE; 722 struct irq_fwspec *fwspec = arg; 723 struct irq_data *irqd; 724 irq_hw_number_t hwirq; 725 int ret; 726 727 if (WARN_ON_ONCE(nr_irqs != 1)) 728 return -EINVAL; 729 730 ret = gicv5_irq_spi_domain_translate(domain, fwspec, &hwirq, &type); 731 if (ret) 732 return ret; 733 734 irqd = irq_desc_get_irq_data(irq_to_desc(virq)); 735 chip_data = gicv5_irs_lookup_by_spi_id(hwirq); 736 737 irq_domain_set_info(domain, virq, hwirq, &gicv5_spi_irq_chip, chip_data, 738 handle_fasteoi_irq, NULL, NULL); 739 irq_set_probe(virq); 740 irqd_set_single_target(irqd); 741 742 gicv5_hwirq_init(hwirq, GICV5_IRQ_PRI_MI, GICV5_HWIRQ_TYPE_SPI); 743 744 return 0; 745 } 746 747 static int gicv5_irq_spi_domain_select(struct irq_domain *d, struct irq_fwspec *fwspec, 748 enum irq_domain_bus_token bus_token) 749 { 750 u32 hwirq_type; 751 752 if (fwspec->fwnode != d->fwnode) 753 return 0; 754 755 if (is_of_node(fwspec->fwnode)) 756 hwirq_type = fwspec->param[0]; 757 758 if (is_fwnode_irqchip(fwspec->fwnode)) 759 hwirq_type = FIELD_GET(GICV5_HWIRQ_TYPE, fwspec->param[0]); 760 761 if (hwirq_type != GICV5_HWIRQ_TYPE_SPI) 762 return 0; 763 764 return (d == gicv5_global_data.spi_domain); 765 } 766 767 static const struct irq_domain_ops gicv5_irq_spi_domain_ops = { 768 .translate = gicv5_irq_spi_domain_translate, 769 .alloc = gicv5_irq_spi_domain_alloc, 770 .free = gicv5_irq_domain_free, 771 .select = gicv5_irq_spi_domain_select 772 }; 773 774 static void gicv5_lpi_config_reset(struct irq_data *d) 775 { 776 u64 cdhm; 777 778 /* 779 * Reset LPIs handling mode to edge by default and clear pending 780 * state to make sure we start the LPI with a clean state from 781 * previous incarnations. 782 */ 783 cdhm = FIELD_PREP(GICV5_GIC_CDHM_HM_MASK, 0) | 784 FIELD_PREP(GICV5_GIC_CDHM_TYPE_MASK, GICV5_HWIRQ_TYPE_LPI) | 785 FIELD_PREP(GICV5_GIC_CDHM_ID_MASK, d->hwirq); 786 gic_insn(cdhm, CDHM); 787 788 gicv5_lpi_irq_write_pending_state(d, false); 789 } 790 791 static int gicv5_irq_lpi_domain_alloc(struct irq_domain *domain, unsigned int virq, 792 unsigned int nr_irqs, void *arg) 793 { 794 irq_hw_number_t hwirq; 795 struct irq_data *irqd; 796 u32 *lpi = arg; 797 int ret; 798 799 if (WARN_ON_ONCE(nr_irqs != 1)) 800 return -EINVAL; 801 802 hwirq = *lpi; 803 804 irqd = irq_domain_get_irq_data(domain, virq); 805 806 irq_domain_set_info(domain, virq, hwirq, &gicv5_lpi_irq_chip, NULL, 807 handle_fasteoi_irq, NULL, NULL); 808 irqd_set_single_target(irqd); 809 810 ret = gicv5_irs_iste_alloc(hwirq); 811 if (ret < 0) 812 return ret; 813 814 gicv5_hwirq_init(hwirq, GICV5_IRQ_PRI_MI, GICV5_HWIRQ_TYPE_LPI); 815 gicv5_lpi_config_reset(irqd); 816 817 return 0; 818 } 819 820 static const struct irq_domain_ops gicv5_irq_lpi_domain_ops = { 821 .alloc = gicv5_irq_lpi_domain_alloc, 822 .free = gicv5_irq_domain_free, 823 }; 824 825 void __init gicv5_init_lpi_domain(void) 826 { 827 struct irq_domain *d; 828 829 d = irq_domain_create_tree(NULL, &gicv5_irq_lpi_domain_ops, NULL); 830 gicv5_global_data.lpi_domain = d; 831 } 832 833 void __init gicv5_free_lpi_domain(void) 834 { 835 irq_domain_remove(gicv5_global_data.lpi_domain); 836 gicv5_global_data.lpi_domain = NULL; 837 } 838 839 static int gicv5_irq_ipi_domain_alloc(struct irq_domain *domain, unsigned int virq, 840 unsigned int nr_irqs, void *arg) 841 { 842 struct irq_data *irqd; 843 int ret, i; 844 u32 lpi; 845 846 for (i = 0; i < nr_irqs; i++) { 847 ret = gicv5_alloc_lpi(); 848 if (ret < 0) 849 return ret; 850 851 lpi = ret; 852 853 ret = irq_domain_alloc_irqs_parent(domain, virq + i, 1, &lpi); 854 if (ret) { 855 gicv5_free_lpi(lpi); 856 return ret; 857 } 858 859 irqd = irq_domain_get_irq_data(domain, virq + i); 860 861 irq_domain_set_hwirq_and_chip(domain, virq + i, i, 862 &gicv5_ipi_irq_chip, NULL); 863 864 irqd_set_single_target(irqd); 865 866 irq_set_handler(virq + i, handle_percpu_irq); 867 } 868 869 return 0; 870 } 871 872 static void gicv5_irq_ipi_domain_free(struct irq_domain *domain, unsigned int virq, 873 unsigned int nr_irqs) 874 { 875 struct irq_data *d; 876 unsigned int i; 877 878 for (i = 0; i < nr_irqs; i++) { 879 d = irq_domain_get_irq_data(domain, virq + i); 880 881 if (!d) 882 return; 883 884 gicv5_free_lpi(d->parent_data->hwirq); 885 886 irq_set_handler(virq + i, NULL); 887 irq_domain_reset_irq_data(d); 888 irq_domain_free_irqs_parent(domain, virq + i, 1); 889 } 890 } 891 892 static const struct irq_domain_ops gicv5_irq_ipi_domain_ops = { 893 .alloc = gicv5_irq_ipi_domain_alloc, 894 .free = gicv5_irq_ipi_domain_free, 895 }; 896 897 static void handle_irq_per_domain(u32 hwirq) 898 { 899 u8 hwirq_type = FIELD_GET(GICV5_HWIRQ_TYPE, hwirq); 900 u32 hwirq_id = FIELD_GET(GICV5_HWIRQ_ID, hwirq); 901 struct irq_domain *domain; 902 903 switch (hwirq_type) { 904 case GICV5_HWIRQ_TYPE_PPI: 905 domain = gicv5_global_data.ppi_domain; 906 break; 907 case GICV5_HWIRQ_TYPE_SPI: 908 domain = gicv5_global_data.spi_domain; 909 break; 910 case GICV5_HWIRQ_TYPE_LPI: 911 domain = gicv5_global_data.lpi_domain; 912 break; 913 default: 914 pr_err_once("Unknown IRQ type, bail out\n"); 915 return; 916 } 917 918 if (generic_handle_domain_irq(domain, hwirq_id)) { 919 pr_err_once("Could not handle, hwirq = 0x%x", hwirq_id); 920 gicv5_hwirq_eoi(hwirq_id, hwirq_type); 921 } 922 } 923 924 static void __exception_irq_entry gicv5_handle_irq(struct pt_regs *regs) 925 { 926 bool valid; 927 u32 hwirq; 928 u64 ia; 929 930 ia = gicr_insn(CDIA); 931 valid = GICV5_GICR_CDIA_VALID(ia); 932 933 if (!valid) 934 return; 935 936 /* 937 * Ensure that the CDIA instruction effects (ie IRQ activation) are 938 * completed before handling the interrupt. 939 */ 940 gsb_ack(); 941 942 /* 943 * Ensure instruction ordering between an acknowledgment and subsequent 944 * instructions in the IRQ handler using an ISB. 945 */ 946 isb(); 947 948 hwirq = FIELD_GET(GICV5_HWIRQ_INTID, ia); 949 950 handle_irq_per_domain(hwirq); 951 } 952 953 static void gicv5_cpu_disable_interrupts(void) 954 { 955 u64 cr0; 956 957 cr0 = FIELD_PREP(ICC_CR0_EL1_EN, 0); 958 write_sysreg_s(cr0, SYS_ICC_CR0_EL1); 959 } 960 961 static void gicv5_cpu_enable_interrupts(void) 962 { 963 u64 cr0, pcr; 964 965 write_sysreg_s(0, SYS_ICC_PPI_ENABLER0_EL1); 966 write_sysreg_s(0, SYS_ICC_PPI_ENABLER1_EL1); 967 968 gicv5_ppi_priority_init(); 969 970 pcr = FIELD_PREP(ICC_PCR_EL1_PRIORITY, GICV5_IRQ_PRI_MI); 971 write_sysreg_s(pcr, SYS_ICC_PCR_EL1); 972 973 cr0 = FIELD_PREP(ICC_CR0_EL1_EN, 1); 974 write_sysreg_s(cr0, SYS_ICC_CR0_EL1); 975 } 976 977 static int base_ipi_virq; 978 979 static int gicv5_starting_cpu(unsigned int cpu) 980 { 981 if (WARN(!gicv5_cpuif_has_gcie(), 982 "GICv5 system components present but CPU does not have FEAT_GCIE")) 983 return -ENODEV; 984 985 gicv5_cpu_enable_interrupts(); 986 987 return gicv5_irs_register_cpu(cpu); 988 } 989 990 static void __init gicv5_smp_init(void) 991 { 992 unsigned int num_ipis = GICV5_IPIS_PER_CPU * nr_cpu_ids; 993 994 cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_GIC_STARTING, 995 "irqchip/arm/gicv5:starting", 996 gicv5_starting_cpu, NULL); 997 998 base_ipi_virq = irq_domain_alloc_irqs(gicv5_global_data.ipi_domain, 999 num_ipis, NUMA_NO_NODE, NULL); 1000 if (WARN(base_ipi_virq <= 0, "IPI IRQ allocation was not successful")) 1001 return; 1002 1003 set_smp_ipi_range_percpu(base_ipi_virq, GICV5_IPIS_PER_CPU, nr_cpu_ids); 1004 } 1005 1006 static void __init gicv5_free_domains(void) 1007 { 1008 if (gicv5_global_data.ppi_domain) 1009 irq_domain_remove(gicv5_global_data.ppi_domain); 1010 if (gicv5_global_data.spi_domain) 1011 irq_domain_remove(gicv5_global_data.spi_domain); 1012 if (gicv5_global_data.ipi_domain) 1013 irq_domain_remove(gicv5_global_data.ipi_domain); 1014 1015 gicv5_global_data.ppi_domain = NULL; 1016 gicv5_global_data.spi_domain = NULL; 1017 gicv5_global_data.ipi_domain = NULL; 1018 } 1019 1020 static int __init gicv5_init_domains(struct fwnode_handle *handle) 1021 { 1022 u32 spi_count = gicv5_global_data.global_spi_count; 1023 struct irq_domain *d; 1024 1025 d = irq_domain_create_linear(handle, PPI_NR, &gicv5_irq_ppi_domain_ops, NULL); 1026 if (!d) 1027 return -ENOMEM; 1028 1029 irq_domain_update_bus_token(d, DOMAIN_BUS_WIRED); 1030 gicv5_global_data.ppi_domain = d; 1031 1032 if (spi_count) { 1033 d = irq_domain_create_linear(handle, spi_count, 1034 &gicv5_irq_spi_domain_ops, NULL); 1035 1036 if (!d) { 1037 gicv5_free_domains(); 1038 return -ENOMEM; 1039 } 1040 1041 gicv5_global_data.spi_domain = d; 1042 irq_domain_update_bus_token(d, DOMAIN_BUS_WIRED); 1043 } 1044 1045 if (!WARN(!gicv5_global_data.lpi_domain, 1046 "LPI domain uninitialized, can't set up IPIs")) { 1047 d = irq_domain_create_hierarchy(gicv5_global_data.lpi_domain, 1048 0, GICV5_IPIS_PER_CPU * nr_cpu_ids, 1049 NULL, &gicv5_irq_ipi_domain_ops, 1050 NULL); 1051 1052 if (!d) { 1053 gicv5_free_domains(); 1054 return -ENOMEM; 1055 } 1056 gicv5_global_data.ipi_domain = d; 1057 } 1058 gicv5_global_data.fwnode = handle; 1059 1060 return 0; 1061 } 1062 1063 static void gicv5_set_cpuif_pribits(void) 1064 { 1065 u64 icc_idr0 = read_sysreg_s(SYS_ICC_IDR0_EL1); 1066 1067 switch (FIELD_GET(ICC_IDR0_EL1_PRI_BITS, icc_idr0)) { 1068 case ICC_IDR0_EL1_PRI_BITS_4BITS: 1069 gicv5_global_data.cpuif_pri_bits = 4; 1070 break; 1071 case ICC_IDR0_EL1_PRI_BITS_5BITS: 1072 gicv5_global_data.cpuif_pri_bits = 5; 1073 break; 1074 default: 1075 pr_err("Unexpected ICC_IDR0_EL1_PRI_BITS value, default to 4"); 1076 gicv5_global_data.cpuif_pri_bits = 4; 1077 break; 1078 } 1079 } 1080 1081 static void gicv5_set_cpuif_idbits(void) 1082 { 1083 u32 icc_idr0 = read_sysreg_s(SYS_ICC_IDR0_EL1); 1084 1085 switch (FIELD_GET(ICC_IDR0_EL1_ID_BITS, icc_idr0)) { 1086 case ICC_IDR0_EL1_ID_BITS_16BITS: 1087 gicv5_global_data.cpuif_id_bits = 16; 1088 break; 1089 case ICC_IDR0_EL1_ID_BITS_24BITS: 1090 gicv5_global_data.cpuif_id_bits = 24; 1091 break; 1092 default: 1093 pr_err("Unexpected ICC_IDR0_EL1_ID_BITS value, default to 16"); 1094 gicv5_global_data.cpuif_id_bits = 16; 1095 break; 1096 } 1097 } 1098 1099 #ifdef CONFIG_KVM 1100 static struct gic_kvm_info gic_v5_kvm_info __initdata; 1101 1102 static void __init gic_of_setup_kvm_info(struct device_node *node) 1103 { 1104 gic_v5_kvm_info.type = GIC_V5; 1105 1106 /* GIC Virtual CPU interface maintenance interrupt */ 1107 gic_v5_kvm_info.no_maint_irq_mask = false; 1108 gic_v5_kvm_info.maint_irq = irq_of_parse_and_map(node, 0); 1109 if (!gic_v5_kvm_info.maint_irq) { 1110 pr_warn("cannot find GICv5 virtual CPU interface maintenance interrupt\n"); 1111 return; 1112 } 1113 1114 vgic_set_kvm_info(&gic_v5_kvm_info); 1115 } 1116 #else 1117 static inline void __init gic_of_setup_kvm_info(struct device_node *node) 1118 { 1119 } 1120 #endif // CONFIG_KVM 1121 1122 static int __init gicv5_init_common(struct fwnode_handle *parent_domain) 1123 { 1124 int ret = gicv5_init_domains(parent_domain); 1125 if (ret) 1126 return ret; 1127 1128 gicv5_set_cpuif_pribits(); 1129 gicv5_set_cpuif_idbits(); 1130 1131 pri_bits = min_not_zero(gicv5_global_data.cpuif_pri_bits, 1132 gicv5_global_data.irs_pri_bits); 1133 1134 ret = gicv5_starting_cpu(smp_processor_id()); 1135 if (ret) 1136 goto out_dom; 1137 1138 ret = set_handle_irq(gicv5_handle_irq); 1139 if (ret) 1140 goto out_int; 1141 1142 ret = gicv5_irs_enable(); 1143 if (ret) 1144 goto out_int; 1145 1146 gicv5_smp_init(); 1147 1148 gicv5_irs_its_probe(); 1149 return 0; 1150 1151 out_int: 1152 gicv5_cpu_disable_interrupts(); 1153 out_dom: 1154 gicv5_free_domains(); 1155 return ret; 1156 } 1157 1158 static int __init gicv5_of_init(struct device_node *node, struct device_node *parent) 1159 { 1160 int ret = gicv5_irs_of_probe(node); 1161 if (ret) 1162 return ret; 1163 1164 ret = gicv5_init_common(of_fwnode_handle(node)); 1165 if (ret) 1166 goto out_irs; 1167 1168 gic_of_setup_kvm_info(node); 1169 1170 return 0; 1171 out_irs: 1172 gicv5_irs_remove(); 1173 1174 return ret; 1175 } 1176 IRQCHIP_DECLARE(gic_v5, "arm,gic-v5", gicv5_of_init); 1177 1178 #ifdef CONFIG_ACPI 1179 static bool __init acpi_validate_gic_table(struct acpi_subtable_header *header, 1180 struct acpi_probe_entry *ape) 1181 { 1182 struct acpi_madt_gicv5_irs *irs = (struct acpi_madt_gicv5_irs *)header; 1183 1184 return (irs->version == ape->driver_data); 1185 } 1186 1187 static struct fwnode_handle *gsi_domain_handle; 1188 1189 static struct fwnode_handle *gic_v5_get_gsi_domain_id(u32 gsi) 1190 { 1191 if (FIELD_GET(GICV5_GSI_IC_TYPE, gsi) == GICV5_GSI_IWB_TYPE) 1192 return iort_iwb_handle(FIELD_GET(GICV5_GSI_IWB_FRAME_ID, gsi)); 1193 1194 return gsi_domain_handle; 1195 } 1196 1197 static int __init gic_acpi_init(union acpi_subtable_headers *header, const unsigned long end) 1198 { 1199 struct acpi_madt_gicv5_irs *irs = (struct acpi_madt_gicv5_irs *)header; 1200 int ret; 1201 1202 if (gsi_domain_handle) 1203 return 0; 1204 1205 gsi_domain_handle = irq_domain_alloc_fwnode(&irs->config_base_address); 1206 if (!gsi_domain_handle) 1207 return -ENOMEM; 1208 1209 ret = gicv5_irs_acpi_probe(); 1210 if (ret) 1211 goto out_fwnode; 1212 1213 ret = gicv5_init_common(gsi_domain_handle); 1214 if (ret) 1215 goto out_irs; 1216 1217 acpi_set_irq_model(ACPI_IRQ_MODEL_GIC_V5, gic_v5_get_gsi_domain_id); 1218 1219 return 0; 1220 1221 out_irs: 1222 gicv5_irs_remove(); 1223 out_fwnode: 1224 irq_domain_free_fwnode(gsi_domain_handle); 1225 return ret; 1226 } 1227 IRQCHIP_ACPI_DECLARE(gic_v5, ACPI_MADT_TYPE_GICV5_IRS, 1228 acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V5, 1229 gic_acpi_init); 1230 #endif 1231