1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved. 4 * Author: Marc Zyngier <marc.zyngier@arm.com> 5 */ 6 7 #define pr_fmt(fmt) "GICv3: " fmt 8 9 #include <linux/acpi.h> 10 #include <linux/cpu.h> 11 #include <linux/cpu_pm.h> 12 #include <linux/delay.h> 13 #include <linux/interrupt.h> 14 #include <linux/irqdomain.h> 15 #include <linux/of.h> 16 #include <linux/of_address.h> 17 #include <linux/of_irq.h> 18 #include <linux/percpu.h> 19 #include <linux/refcount.h> 20 #include <linux/slab.h> 21 22 #include <linux/irqchip.h> 23 #include <linux/irqchip/arm-gic-common.h> 24 #include <linux/irqchip/arm-gic-v3.h> 25 #include <linux/irqchip/irq-partition-percpu.h> 26 27 #include <asm/cputype.h> 28 #include <asm/exception.h> 29 #include <asm/smp_plat.h> 30 #include <asm/virt.h> 31 32 #include "irq-gic-common.h" 33 34 #define GICD_INT_NMI_PRI (GICD_INT_DEF_PRI & ~0x80) 35 36 #define FLAGS_WORKAROUND_GICR_WAKER_MSM8996 (1ULL << 0) 37 #define FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539 (1ULL << 1) 38 39 #define GIC_IRQ_TYPE_PARTITION (GIC_IRQ_TYPE_LPI + 1) 40 41 struct redist_region { 42 void __iomem *redist_base; 43 phys_addr_t phys_base; 44 bool single_redist; 45 }; 46 47 struct gic_chip_data { 48 struct fwnode_handle *fwnode; 49 void __iomem *dist_base; 50 struct redist_region *redist_regions; 51 struct rdists rdists; 52 struct irq_domain *domain; 53 u64 redist_stride; 54 u32 nr_redist_regions; 55 u64 flags; 56 bool has_rss; 57 unsigned int ppi_nr; 58 struct partition_desc **ppi_descs; 59 }; 60 61 static struct gic_chip_data gic_data __read_mostly; 62 static DEFINE_STATIC_KEY_TRUE(supports_deactivate_key); 63 64 #define GIC_ID_NR (1U << GICD_TYPER_ID_BITS(gic_data.rdists.gicd_typer)) 65 #define GIC_LINE_NR min(GICD_TYPER_SPIS(gic_data.rdists.gicd_typer), 1020U) 66 #define GIC_ESPI_NR GICD_TYPER_ESPIS(gic_data.rdists.gicd_typer) 67 68 /* 69 * The behaviours of RPR and PMR registers differ depending on the value of 70 * SCR_EL3.FIQ, and the behaviour of non-secure priority registers of the 71 * distributor and redistributors depends on whether security is enabled in the 72 * GIC. 73 * 74 * When security is enabled, non-secure priority values from the (re)distributor 75 * are presented to the GIC CPUIF as follow: 76 * (GIC_(R)DIST_PRI[irq] >> 1) | 0x80; 77 * 78 * If SCR_EL3.FIQ == 1, the values written to/read from PMR and RPR at non-secure 79 * EL1 are subject to a similar operation thus matching the priorities presented 80 * from the (re)distributor when security is enabled. When SCR_EL3.FIQ == 0, 81 * these values are unchanged by the GIC. 82 * 83 * see GICv3/GICv4 Architecture Specification (IHI0069D): 84 * - section 4.8.1 Non-secure accesses to register fields for Secure interrupt 85 * priorities. 86 * - Figure 4-7 Secure read of the priority field for a Non-secure Group 1 87 * interrupt. 88 */ 89 static DEFINE_STATIC_KEY_FALSE(supports_pseudo_nmis); 90 91 /* 92 * Global static key controlling whether an update to PMR allowing more 93 * interrupts requires to be propagated to the redistributor (DSB SY). 94 * And this needs to be exported for modules to be able to enable 95 * interrupts... 96 */ 97 DEFINE_STATIC_KEY_FALSE(gic_pmr_sync); 98 EXPORT_SYMBOL(gic_pmr_sync); 99 100 DEFINE_STATIC_KEY_FALSE(gic_nonsecure_priorities); 101 EXPORT_SYMBOL(gic_nonsecure_priorities); 102 103 /* 104 * When the Non-secure world has access to group 0 interrupts (as a 105 * consequence of SCR_EL3.FIQ == 0), reading the ICC_RPR_EL1 register will 106 * return the Distributor's view of the interrupt priority. 107 * 108 * When GIC security is enabled (GICD_CTLR.DS == 0), the interrupt priority 109 * written by software is moved to the Non-secure range by the Distributor. 110 * 111 * If both are true (which is when gic_nonsecure_priorities gets enabled), 112 * we need to shift down the priority programmed by software to match it 113 * against the value returned by ICC_RPR_EL1. 114 */ 115 #define GICD_INT_RPR_PRI(priority) \ 116 ({ \ 117 u32 __priority = (priority); \ 118 if (static_branch_unlikely(&gic_nonsecure_priorities)) \ 119 __priority = 0x80 | (__priority >> 1); \ 120 \ 121 __priority; \ 122 }) 123 124 /* ppi_nmi_refs[n] == number of cpus having ppi[n + 16] set as NMI */ 125 static refcount_t *ppi_nmi_refs; 126 127 static struct gic_kvm_info gic_v3_kvm_info __initdata; 128 static DEFINE_PER_CPU(bool, has_rss); 129 130 #define MPIDR_RS(mpidr) (((mpidr) & 0xF0UL) >> 4) 131 #define gic_data_rdist() (this_cpu_ptr(gic_data.rdists.rdist)) 132 #define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base) 133 #define gic_data_rdist_sgi_base() (gic_data_rdist_rd_base() + SZ_64K) 134 135 /* Our default, arbitrary priority value. Linux only uses one anyway. */ 136 #define DEFAULT_PMR_VALUE 0xf0 137 138 enum gic_intid_range { 139 SGI_RANGE, 140 PPI_RANGE, 141 SPI_RANGE, 142 EPPI_RANGE, 143 ESPI_RANGE, 144 LPI_RANGE, 145 __INVALID_RANGE__ 146 }; 147 148 static enum gic_intid_range __get_intid_range(irq_hw_number_t hwirq) 149 { 150 switch (hwirq) { 151 case 0 ... 15: 152 return SGI_RANGE; 153 case 16 ... 31: 154 return PPI_RANGE; 155 case 32 ... 1019: 156 return SPI_RANGE; 157 case EPPI_BASE_INTID ... (EPPI_BASE_INTID + 63): 158 return EPPI_RANGE; 159 case ESPI_BASE_INTID ... (ESPI_BASE_INTID + 1023): 160 return ESPI_RANGE; 161 case 8192 ... GENMASK(23, 0): 162 return LPI_RANGE; 163 default: 164 return __INVALID_RANGE__; 165 } 166 } 167 168 static enum gic_intid_range get_intid_range(struct irq_data *d) 169 { 170 return __get_intid_range(d->hwirq); 171 } 172 173 static inline unsigned int gic_irq(struct irq_data *d) 174 { 175 return d->hwirq; 176 } 177 178 static inline bool gic_irq_in_rdist(struct irq_data *d) 179 { 180 switch (get_intid_range(d)) { 181 case SGI_RANGE: 182 case PPI_RANGE: 183 case EPPI_RANGE: 184 return true; 185 default: 186 return false; 187 } 188 } 189 190 static inline void __iomem *gic_dist_base(struct irq_data *d) 191 { 192 switch (get_intid_range(d)) { 193 case SGI_RANGE: 194 case PPI_RANGE: 195 case EPPI_RANGE: 196 /* SGI+PPI -> SGI_base for this CPU */ 197 return gic_data_rdist_sgi_base(); 198 199 case SPI_RANGE: 200 case ESPI_RANGE: 201 /* SPI -> dist_base */ 202 return gic_data.dist_base; 203 204 default: 205 return NULL; 206 } 207 } 208 209 static void gic_do_wait_for_rwp(void __iomem *base, u32 bit) 210 { 211 u32 count = 1000000; /* 1s! */ 212 213 while (readl_relaxed(base + GICD_CTLR) & bit) { 214 count--; 215 if (!count) { 216 pr_err_ratelimited("RWP timeout, gone fishing\n"); 217 return; 218 } 219 cpu_relax(); 220 udelay(1); 221 } 222 } 223 224 /* Wait for completion of a distributor change */ 225 static void gic_dist_wait_for_rwp(void) 226 { 227 gic_do_wait_for_rwp(gic_data.dist_base, GICD_CTLR_RWP); 228 } 229 230 /* Wait for completion of a redistributor change */ 231 static void gic_redist_wait_for_rwp(void) 232 { 233 gic_do_wait_for_rwp(gic_data_rdist_rd_base(), GICR_CTLR_RWP); 234 } 235 236 #ifdef CONFIG_ARM64 237 238 static u64 __maybe_unused gic_read_iar(void) 239 { 240 if (cpus_have_const_cap(ARM64_WORKAROUND_CAVIUM_23154)) 241 return gic_read_iar_cavium_thunderx(); 242 else 243 return gic_read_iar_common(); 244 } 245 #endif 246 247 static void gic_enable_redist(bool enable) 248 { 249 void __iomem *rbase; 250 u32 count = 1000000; /* 1s! */ 251 u32 val; 252 253 if (gic_data.flags & FLAGS_WORKAROUND_GICR_WAKER_MSM8996) 254 return; 255 256 rbase = gic_data_rdist_rd_base(); 257 258 val = readl_relaxed(rbase + GICR_WAKER); 259 if (enable) 260 /* Wake up this CPU redistributor */ 261 val &= ~GICR_WAKER_ProcessorSleep; 262 else 263 val |= GICR_WAKER_ProcessorSleep; 264 writel_relaxed(val, rbase + GICR_WAKER); 265 266 if (!enable) { /* Check that GICR_WAKER is writeable */ 267 val = readl_relaxed(rbase + GICR_WAKER); 268 if (!(val & GICR_WAKER_ProcessorSleep)) 269 return; /* No PM support in this redistributor */ 270 } 271 272 while (--count) { 273 val = readl_relaxed(rbase + GICR_WAKER); 274 if (enable ^ (bool)(val & GICR_WAKER_ChildrenAsleep)) 275 break; 276 cpu_relax(); 277 udelay(1); 278 } 279 if (!count) 280 pr_err_ratelimited("redistributor failed to %s...\n", 281 enable ? "wakeup" : "sleep"); 282 } 283 284 /* 285 * Routines to disable, enable, EOI and route interrupts 286 */ 287 static u32 convert_offset_index(struct irq_data *d, u32 offset, u32 *index) 288 { 289 switch (get_intid_range(d)) { 290 case SGI_RANGE: 291 case PPI_RANGE: 292 case SPI_RANGE: 293 *index = d->hwirq; 294 return offset; 295 case EPPI_RANGE: 296 /* 297 * Contrary to the ESPI range, the EPPI range is contiguous 298 * to the PPI range in the registers, so let's adjust the 299 * displacement accordingly. Consistency is overrated. 300 */ 301 *index = d->hwirq - EPPI_BASE_INTID + 32; 302 return offset; 303 case ESPI_RANGE: 304 *index = d->hwirq - ESPI_BASE_INTID; 305 switch (offset) { 306 case GICD_ISENABLER: 307 return GICD_ISENABLERnE; 308 case GICD_ICENABLER: 309 return GICD_ICENABLERnE; 310 case GICD_ISPENDR: 311 return GICD_ISPENDRnE; 312 case GICD_ICPENDR: 313 return GICD_ICPENDRnE; 314 case GICD_ISACTIVER: 315 return GICD_ISACTIVERnE; 316 case GICD_ICACTIVER: 317 return GICD_ICACTIVERnE; 318 case GICD_IPRIORITYR: 319 return GICD_IPRIORITYRnE; 320 case GICD_ICFGR: 321 return GICD_ICFGRnE; 322 case GICD_IROUTER: 323 return GICD_IROUTERnE; 324 default: 325 break; 326 } 327 break; 328 default: 329 break; 330 } 331 332 WARN_ON(1); 333 *index = d->hwirq; 334 return offset; 335 } 336 337 static int gic_peek_irq(struct irq_data *d, u32 offset) 338 { 339 void __iomem *base; 340 u32 index, mask; 341 342 offset = convert_offset_index(d, offset, &index); 343 mask = 1 << (index % 32); 344 345 if (gic_irq_in_rdist(d)) 346 base = gic_data_rdist_sgi_base(); 347 else 348 base = gic_data.dist_base; 349 350 return !!(readl_relaxed(base + offset + (index / 32) * 4) & mask); 351 } 352 353 static void gic_poke_irq(struct irq_data *d, u32 offset) 354 { 355 void (*rwp_wait)(void); 356 void __iomem *base; 357 u32 index, mask; 358 359 offset = convert_offset_index(d, offset, &index); 360 mask = 1 << (index % 32); 361 362 if (gic_irq_in_rdist(d)) { 363 base = gic_data_rdist_sgi_base(); 364 rwp_wait = gic_redist_wait_for_rwp; 365 } else { 366 base = gic_data.dist_base; 367 rwp_wait = gic_dist_wait_for_rwp; 368 } 369 370 writel_relaxed(mask, base + offset + (index / 32) * 4); 371 rwp_wait(); 372 } 373 374 static void gic_mask_irq(struct irq_data *d) 375 { 376 gic_poke_irq(d, GICD_ICENABLER); 377 } 378 379 static void gic_eoimode1_mask_irq(struct irq_data *d) 380 { 381 gic_mask_irq(d); 382 /* 383 * When masking a forwarded interrupt, make sure it is 384 * deactivated as well. 385 * 386 * This ensures that an interrupt that is getting 387 * disabled/masked will not get "stuck", because there is 388 * noone to deactivate it (guest is being terminated). 389 */ 390 if (irqd_is_forwarded_to_vcpu(d)) 391 gic_poke_irq(d, GICD_ICACTIVER); 392 } 393 394 static void gic_unmask_irq(struct irq_data *d) 395 { 396 gic_poke_irq(d, GICD_ISENABLER); 397 } 398 399 static inline bool gic_supports_nmi(void) 400 { 401 return IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && 402 static_branch_likely(&supports_pseudo_nmis); 403 } 404 405 static int gic_irq_set_irqchip_state(struct irq_data *d, 406 enum irqchip_irq_state which, bool val) 407 { 408 u32 reg; 409 410 if (d->hwirq >= 8192) /* SGI/PPI/SPI only */ 411 return -EINVAL; 412 413 switch (which) { 414 case IRQCHIP_STATE_PENDING: 415 reg = val ? GICD_ISPENDR : GICD_ICPENDR; 416 break; 417 418 case IRQCHIP_STATE_ACTIVE: 419 reg = val ? GICD_ISACTIVER : GICD_ICACTIVER; 420 break; 421 422 case IRQCHIP_STATE_MASKED: 423 reg = val ? GICD_ICENABLER : GICD_ISENABLER; 424 break; 425 426 default: 427 return -EINVAL; 428 } 429 430 gic_poke_irq(d, reg); 431 return 0; 432 } 433 434 static int gic_irq_get_irqchip_state(struct irq_data *d, 435 enum irqchip_irq_state which, bool *val) 436 { 437 if (d->hwirq >= 8192) /* PPI/SPI only */ 438 return -EINVAL; 439 440 switch (which) { 441 case IRQCHIP_STATE_PENDING: 442 *val = gic_peek_irq(d, GICD_ISPENDR); 443 break; 444 445 case IRQCHIP_STATE_ACTIVE: 446 *val = gic_peek_irq(d, GICD_ISACTIVER); 447 break; 448 449 case IRQCHIP_STATE_MASKED: 450 *val = !gic_peek_irq(d, GICD_ISENABLER); 451 break; 452 453 default: 454 return -EINVAL; 455 } 456 457 return 0; 458 } 459 460 static void gic_irq_set_prio(struct irq_data *d, u8 prio) 461 { 462 void __iomem *base = gic_dist_base(d); 463 u32 offset, index; 464 465 offset = convert_offset_index(d, GICD_IPRIORITYR, &index); 466 467 writeb_relaxed(prio, base + offset + index); 468 } 469 470 static u32 __gic_get_ppi_index(irq_hw_number_t hwirq) 471 { 472 switch (__get_intid_range(hwirq)) { 473 case PPI_RANGE: 474 return hwirq - 16; 475 case EPPI_RANGE: 476 return hwirq - EPPI_BASE_INTID + 16; 477 default: 478 unreachable(); 479 } 480 } 481 482 static u32 gic_get_ppi_index(struct irq_data *d) 483 { 484 return __gic_get_ppi_index(d->hwirq); 485 } 486 487 static int gic_irq_nmi_setup(struct irq_data *d) 488 { 489 struct irq_desc *desc = irq_to_desc(d->irq); 490 491 if (!gic_supports_nmi()) 492 return -EINVAL; 493 494 if (gic_peek_irq(d, GICD_ISENABLER)) { 495 pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq); 496 return -EINVAL; 497 } 498 499 /* 500 * A secondary irq_chip should be in charge of LPI request, 501 * it should not be possible to get there 502 */ 503 if (WARN_ON(gic_irq(d) >= 8192)) 504 return -EINVAL; 505 506 /* desc lock should already be held */ 507 if (gic_irq_in_rdist(d)) { 508 u32 idx = gic_get_ppi_index(d); 509 510 /* Setting up PPI as NMI, only switch handler for first NMI */ 511 if (!refcount_inc_not_zero(&ppi_nmi_refs[idx])) { 512 refcount_set(&ppi_nmi_refs[idx], 1); 513 desc->handle_irq = handle_percpu_devid_fasteoi_nmi; 514 } 515 } else { 516 desc->handle_irq = handle_fasteoi_nmi; 517 } 518 519 gic_irq_set_prio(d, GICD_INT_NMI_PRI); 520 521 return 0; 522 } 523 524 static void gic_irq_nmi_teardown(struct irq_data *d) 525 { 526 struct irq_desc *desc = irq_to_desc(d->irq); 527 528 if (WARN_ON(!gic_supports_nmi())) 529 return; 530 531 if (gic_peek_irq(d, GICD_ISENABLER)) { 532 pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq); 533 return; 534 } 535 536 /* 537 * A secondary irq_chip should be in charge of LPI request, 538 * it should not be possible to get there 539 */ 540 if (WARN_ON(gic_irq(d) >= 8192)) 541 return; 542 543 /* desc lock should already be held */ 544 if (gic_irq_in_rdist(d)) { 545 u32 idx = gic_get_ppi_index(d); 546 547 /* Tearing down NMI, only switch handler for last NMI */ 548 if (refcount_dec_and_test(&ppi_nmi_refs[idx])) 549 desc->handle_irq = handle_percpu_devid_irq; 550 } else { 551 desc->handle_irq = handle_fasteoi_irq; 552 } 553 554 gic_irq_set_prio(d, GICD_INT_DEF_PRI); 555 } 556 557 static void gic_eoi_irq(struct irq_data *d) 558 { 559 gic_write_eoir(gic_irq(d)); 560 } 561 562 static void gic_eoimode1_eoi_irq(struct irq_data *d) 563 { 564 /* 565 * No need to deactivate an LPI, or an interrupt that 566 * is is getting forwarded to a vcpu. 567 */ 568 if (gic_irq(d) >= 8192 || irqd_is_forwarded_to_vcpu(d)) 569 return; 570 gic_write_dir(gic_irq(d)); 571 } 572 573 static int gic_set_type(struct irq_data *d, unsigned int type) 574 { 575 enum gic_intid_range range; 576 unsigned int irq = gic_irq(d); 577 void (*rwp_wait)(void); 578 void __iomem *base; 579 u32 offset, index; 580 int ret; 581 582 range = get_intid_range(d); 583 584 /* Interrupt configuration for SGIs can't be changed */ 585 if (range == SGI_RANGE) 586 return type != IRQ_TYPE_EDGE_RISING ? -EINVAL : 0; 587 588 /* SPIs have restrictions on the supported types */ 589 if ((range == SPI_RANGE || range == ESPI_RANGE) && 590 type != IRQ_TYPE_LEVEL_HIGH && type != IRQ_TYPE_EDGE_RISING) 591 return -EINVAL; 592 593 if (gic_irq_in_rdist(d)) { 594 base = gic_data_rdist_sgi_base(); 595 rwp_wait = gic_redist_wait_for_rwp; 596 } else { 597 base = gic_data.dist_base; 598 rwp_wait = gic_dist_wait_for_rwp; 599 } 600 601 offset = convert_offset_index(d, GICD_ICFGR, &index); 602 603 ret = gic_configure_irq(index, type, base + offset, rwp_wait); 604 if (ret && (range == PPI_RANGE || range == EPPI_RANGE)) { 605 /* Misconfigured PPIs are usually not fatal */ 606 pr_warn("GIC: PPI INTID%d is secure or misconfigured\n", irq); 607 ret = 0; 608 } 609 610 return ret; 611 } 612 613 static int gic_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu) 614 { 615 if (get_intid_range(d) == SGI_RANGE) 616 return -EINVAL; 617 618 if (vcpu) 619 irqd_set_forwarded_to_vcpu(d); 620 else 621 irqd_clr_forwarded_to_vcpu(d); 622 return 0; 623 } 624 625 static u64 gic_mpidr_to_affinity(unsigned long mpidr) 626 { 627 u64 aff; 628 629 aff = ((u64)MPIDR_AFFINITY_LEVEL(mpidr, 3) << 32 | 630 MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 | 631 MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 | 632 MPIDR_AFFINITY_LEVEL(mpidr, 0)); 633 634 return aff; 635 } 636 637 static void gic_deactivate_unhandled(u32 irqnr) 638 { 639 if (static_branch_likely(&supports_deactivate_key)) { 640 if (irqnr < 8192) 641 gic_write_dir(irqnr); 642 } else { 643 gic_write_eoir(irqnr); 644 } 645 } 646 647 static inline void gic_handle_nmi(u32 irqnr, struct pt_regs *regs) 648 { 649 bool irqs_enabled = interrupts_enabled(regs); 650 int err; 651 652 if (irqs_enabled) 653 nmi_enter(); 654 655 if (static_branch_likely(&supports_deactivate_key)) 656 gic_write_eoir(irqnr); 657 /* 658 * Leave the PSR.I bit set to prevent other NMIs to be 659 * received while handling this one. 660 * PSR.I will be restored when we ERET to the 661 * interrupted context. 662 */ 663 err = generic_handle_domain_nmi(gic_data.domain, irqnr); 664 if (err) 665 gic_deactivate_unhandled(irqnr); 666 667 if (irqs_enabled) 668 nmi_exit(); 669 } 670 671 static u32 do_read_iar(struct pt_regs *regs) 672 { 673 u32 iar; 674 675 if (gic_supports_nmi() && unlikely(!interrupts_enabled(regs))) { 676 u64 pmr; 677 678 /* 679 * We were in a context with IRQs disabled. However, the 680 * entry code has set PMR to a value that allows any 681 * interrupt to be acknowledged, and not just NMIs. This can 682 * lead to surprising effects if the NMI has been retired in 683 * the meantime, and that there is an IRQ pending. The IRQ 684 * would then be taken in NMI context, something that nobody 685 * wants to debug twice. 686 * 687 * Until we sort this, drop PMR again to a level that will 688 * actually only allow NMIs before reading IAR, and then 689 * restore it to what it was. 690 */ 691 pmr = gic_read_pmr(); 692 gic_pmr_mask_irqs(); 693 isb(); 694 695 iar = gic_read_iar(); 696 697 gic_write_pmr(pmr); 698 } else { 699 iar = gic_read_iar(); 700 } 701 702 return iar; 703 } 704 705 static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs) 706 { 707 u32 irqnr; 708 709 irqnr = do_read_iar(regs); 710 711 /* Check for special IDs first */ 712 if ((irqnr >= 1020 && irqnr <= 1023)) 713 return; 714 715 if (gic_supports_nmi() && 716 unlikely(gic_read_rpr() == GICD_INT_RPR_PRI(GICD_INT_NMI_PRI))) { 717 gic_handle_nmi(irqnr, regs); 718 return; 719 } 720 721 if (gic_prio_masking_enabled()) { 722 gic_pmr_mask_irqs(); 723 gic_arch_enable_irqs(); 724 } 725 726 if (static_branch_likely(&supports_deactivate_key)) 727 gic_write_eoir(irqnr); 728 else 729 isb(); 730 731 if (generic_handle_domain_irq(gic_data.domain, irqnr)) { 732 WARN_ONCE(true, "Unexpected interrupt received!\n"); 733 gic_deactivate_unhandled(irqnr); 734 } 735 } 736 737 static u32 gic_get_pribits(void) 738 { 739 u32 pribits; 740 741 pribits = gic_read_ctlr(); 742 pribits &= ICC_CTLR_EL1_PRI_BITS_MASK; 743 pribits >>= ICC_CTLR_EL1_PRI_BITS_SHIFT; 744 pribits++; 745 746 return pribits; 747 } 748 749 static bool gic_has_group0(void) 750 { 751 u32 val; 752 u32 old_pmr; 753 754 old_pmr = gic_read_pmr(); 755 756 /* 757 * Let's find out if Group0 is under control of EL3 or not by 758 * setting the highest possible, non-zero priority in PMR. 759 * 760 * If SCR_EL3.FIQ is set, the priority gets shifted down in 761 * order for the CPU interface to set bit 7, and keep the 762 * actual priority in the non-secure range. In the process, it 763 * looses the least significant bit and the actual priority 764 * becomes 0x80. Reading it back returns 0, indicating that 765 * we're don't have access to Group0. 766 */ 767 gic_write_pmr(BIT(8 - gic_get_pribits())); 768 val = gic_read_pmr(); 769 770 gic_write_pmr(old_pmr); 771 772 return val != 0; 773 } 774 775 static void __init gic_dist_init(void) 776 { 777 unsigned int i; 778 u64 affinity; 779 void __iomem *base = gic_data.dist_base; 780 u32 val; 781 782 /* Disable the distributor */ 783 writel_relaxed(0, base + GICD_CTLR); 784 gic_dist_wait_for_rwp(); 785 786 /* 787 * Configure SPIs as non-secure Group-1. This will only matter 788 * if the GIC only has a single security state. This will not 789 * do the right thing if the kernel is running in secure mode, 790 * but that's not the intended use case anyway. 791 */ 792 for (i = 32; i < GIC_LINE_NR; i += 32) 793 writel_relaxed(~0, base + GICD_IGROUPR + i / 8); 794 795 /* Extended SPI range, not handled by the GICv2/GICv3 common code */ 796 for (i = 0; i < GIC_ESPI_NR; i += 32) { 797 writel_relaxed(~0U, base + GICD_ICENABLERnE + i / 8); 798 writel_relaxed(~0U, base + GICD_ICACTIVERnE + i / 8); 799 } 800 801 for (i = 0; i < GIC_ESPI_NR; i += 32) 802 writel_relaxed(~0U, base + GICD_IGROUPRnE + i / 8); 803 804 for (i = 0; i < GIC_ESPI_NR; i += 16) 805 writel_relaxed(0, base + GICD_ICFGRnE + i / 4); 806 807 for (i = 0; i < GIC_ESPI_NR; i += 4) 808 writel_relaxed(GICD_INT_DEF_PRI_X4, base + GICD_IPRIORITYRnE + i); 809 810 /* Now do the common stuff, and wait for the distributor to drain */ 811 gic_dist_config(base, GIC_LINE_NR, gic_dist_wait_for_rwp); 812 813 val = GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1; 814 if (gic_data.rdists.gicd_typer2 & GICD_TYPER2_nASSGIcap) { 815 pr_info("Enabling SGIs without active state\n"); 816 val |= GICD_CTLR_nASSGIreq; 817 } 818 819 /* Enable distributor with ARE, Group1 */ 820 writel_relaxed(val, base + GICD_CTLR); 821 822 /* 823 * Set all global interrupts to the boot CPU only. ARE must be 824 * enabled. 825 */ 826 affinity = gic_mpidr_to_affinity(cpu_logical_map(smp_processor_id())); 827 for (i = 32; i < GIC_LINE_NR; i++) 828 gic_write_irouter(affinity, base + GICD_IROUTER + i * 8); 829 830 for (i = 0; i < GIC_ESPI_NR; i++) 831 gic_write_irouter(affinity, base + GICD_IROUTERnE + i * 8); 832 } 833 834 static int gic_iterate_rdists(int (*fn)(struct redist_region *, void __iomem *)) 835 { 836 int ret = -ENODEV; 837 int i; 838 839 for (i = 0; i < gic_data.nr_redist_regions; i++) { 840 void __iomem *ptr = gic_data.redist_regions[i].redist_base; 841 u64 typer; 842 u32 reg; 843 844 reg = readl_relaxed(ptr + GICR_PIDR2) & GIC_PIDR2_ARCH_MASK; 845 if (reg != GIC_PIDR2_ARCH_GICv3 && 846 reg != GIC_PIDR2_ARCH_GICv4) { /* We're in trouble... */ 847 pr_warn("No redistributor present @%p\n", ptr); 848 break; 849 } 850 851 do { 852 typer = gic_read_typer(ptr + GICR_TYPER); 853 ret = fn(gic_data.redist_regions + i, ptr); 854 if (!ret) 855 return 0; 856 857 if (gic_data.redist_regions[i].single_redist) 858 break; 859 860 if (gic_data.redist_stride) { 861 ptr += gic_data.redist_stride; 862 } else { 863 ptr += SZ_64K * 2; /* Skip RD_base + SGI_base */ 864 if (typer & GICR_TYPER_VLPIS) 865 ptr += SZ_64K * 2; /* Skip VLPI_base + reserved page */ 866 } 867 } while (!(typer & GICR_TYPER_LAST)); 868 } 869 870 return ret ? -ENODEV : 0; 871 } 872 873 static int __gic_populate_rdist(struct redist_region *region, void __iomem *ptr) 874 { 875 unsigned long mpidr = cpu_logical_map(smp_processor_id()); 876 u64 typer; 877 u32 aff; 878 879 /* 880 * Convert affinity to a 32bit value that can be matched to 881 * GICR_TYPER bits [63:32]. 882 */ 883 aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 | 884 MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 | 885 MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 | 886 MPIDR_AFFINITY_LEVEL(mpidr, 0)); 887 888 typer = gic_read_typer(ptr + GICR_TYPER); 889 if ((typer >> 32) == aff) { 890 u64 offset = ptr - region->redist_base; 891 raw_spin_lock_init(&gic_data_rdist()->rd_lock); 892 gic_data_rdist_rd_base() = ptr; 893 gic_data_rdist()->phys_base = region->phys_base + offset; 894 895 pr_info("CPU%d: found redistributor %lx region %d:%pa\n", 896 smp_processor_id(), mpidr, 897 (int)(region - gic_data.redist_regions), 898 &gic_data_rdist()->phys_base); 899 return 0; 900 } 901 902 /* Try next one */ 903 return 1; 904 } 905 906 static int gic_populate_rdist(void) 907 { 908 if (gic_iterate_rdists(__gic_populate_rdist) == 0) 909 return 0; 910 911 /* We couldn't even deal with ourselves... */ 912 WARN(true, "CPU%d: mpidr %lx has no re-distributor!\n", 913 smp_processor_id(), 914 (unsigned long)cpu_logical_map(smp_processor_id())); 915 return -ENODEV; 916 } 917 918 static int __gic_update_rdist_properties(struct redist_region *region, 919 void __iomem *ptr) 920 { 921 u64 typer = gic_read_typer(ptr + GICR_TYPER); 922 923 /* Boot-time cleanip */ 924 if ((typer & GICR_TYPER_VLPIS) && (typer & GICR_TYPER_RVPEID)) { 925 u64 val; 926 927 /* Deactivate any present vPE */ 928 val = gicr_read_vpendbaser(ptr + SZ_128K + GICR_VPENDBASER); 929 if (val & GICR_VPENDBASER_Valid) 930 gicr_write_vpendbaser(GICR_VPENDBASER_PendingLast, 931 ptr + SZ_128K + GICR_VPENDBASER); 932 933 /* Mark the VPE table as invalid */ 934 val = gicr_read_vpropbaser(ptr + SZ_128K + GICR_VPROPBASER); 935 val &= ~GICR_VPROPBASER_4_1_VALID; 936 gicr_write_vpropbaser(val, ptr + SZ_128K + GICR_VPROPBASER); 937 } 938 939 gic_data.rdists.has_vlpis &= !!(typer & GICR_TYPER_VLPIS); 940 941 /* RVPEID implies some form of DirectLPI, no matter what the doc says... :-/ */ 942 gic_data.rdists.has_rvpeid &= !!(typer & GICR_TYPER_RVPEID); 943 gic_data.rdists.has_direct_lpi &= (!!(typer & GICR_TYPER_DirectLPIS) | 944 gic_data.rdists.has_rvpeid); 945 gic_data.rdists.has_vpend_valid_dirty &= !!(typer & GICR_TYPER_DIRTY); 946 947 /* Detect non-sensical configurations */ 948 if (WARN_ON_ONCE(gic_data.rdists.has_rvpeid && !gic_data.rdists.has_vlpis)) { 949 gic_data.rdists.has_direct_lpi = false; 950 gic_data.rdists.has_vlpis = false; 951 gic_data.rdists.has_rvpeid = false; 952 } 953 954 gic_data.ppi_nr = min(GICR_TYPER_NR_PPIS(typer), gic_data.ppi_nr); 955 956 return 1; 957 } 958 959 static void gic_update_rdist_properties(void) 960 { 961 gic_data.ppi_nr = UINT_MAX; 962 gic_iterate_rdists(__gic_update_rdist_properties); 963 if (WARN_ON(gic_data.ppi_nr == UINT_MAX)) 964 gic_data.ppi_nr = 0; 965 pr_info("%d PPIs implemented\n", gic_data.ppi_nr); 966 if (gic_data.rdists.has_vlpis) 967 pr_info("GICv4 features: %s%s%s\n", 968 gic_data.rdists.has_direct_lpi ? "DirectLPI " : "", 969 gic_data.rdists.has_rvpeid ? "RVPEID " : "", 970 gic_data.rdists.has_vpend_valid_dirty ? "Valid+Dirty " : ""); 971 } 972 973 /* Check whether it's single security state view */ 974 static inline bool gic_dist_security_disabled(void) 975 { 976 return readl_relaxed(gic_data.dist_base + GICD_CTLR) & GICD_CTLR_DS; 977 } 978 979 static void gic_cpu_sys_reg_init(void) 980 { 981 int i, cpu = smp_processor_id(); 982 u64 mpidr = cpu_logical_map(cpu); 983 u64 need_rss = MPIDR_RS(mpidr); 984 bool group0; 985 u32 pribits; 986 987 /* 988 * Need to check that the SRE bit has actually been set. If 989 * not, it means that SRE is disabled at EL2. We're going to 990 * die painfully, and there is nothing we can do about it. 991 * 992 * Kindly inform the luser. 993 */ 994 if (!gic_enable_sre()) 995 pr_err("GIC: unable to set SRE (disabled at EL2), panic ahead\n"); 996 997 pribits = gic_get_pribits(); 998 999 group0 = gic_has_group0(); 1000 1001 /* Set priority mask register */ 1002 if (!gic_prio_masking_enabled()) { 1003 write_gicreg(DEFAULT_PMR_VALUE, ICC_PMR_EL1); 1004 } else if (gic_supports_nmi()) { 1005 /* 1006 * Mismatch configuration with boot CPU, the system is likely 1007 * to die as interrupt masking will not work properly on all 1008 * CPUs 1009 * 1010 * The boot CPU calls this function before enabling NMI support, 1011 * and as a result we'll never see this warning in the boot path 1012 * for that CPU. 1013 */ 1014 if (static_branch_unlikely(&gic_nonsecure_priorities)) 1015 WARN_ON(!group0 || gic_dist_security_disabled()); 1016 else 1017 WARN_ON(group0 && !gic_dist_security_disabled()); 1018 } 1019 1020 /* 1021 * Some firmwares hand over to the kernel with the BPR changed from 1022 * its reset value (and with a value large enough to prevent 1023 * any pre-emptive interrupts from working at all). Writing a zero 1024 * to BPR restores is reset value. 1025 */ 1026 gic_write_bpr1(0); 1027 1028 if (static_branch_likely(&supports_deactivate_key)) { 1029 /* EOI drops priority only (mode 1) */ 1030 gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop); 1031 } else { 1032 /* EOI deactivates interrupt too (mode 0) */ 1033 gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop_dir); 1034 } 1035 1036 /* Always whack Group0 before Group1 */ 1037 if (group0) { 1038 switch(pribits) { 1039 case 8: 1040 case 7: 1041 write_gicreg(0, ICC_AP0R3_EL1); 1042 write_gicreg(0, ICC_AP0R2_EL1); 1043 fallthrough; 1044 case 6: 1045 write_gicreg(0, ICC_AP0R1_EL1); 1046 fallthrough; 1047 case 5: 1048 case 4: 1049 write_gicreg(0, ICC_AP0R0_EL1); 1050 } 1051 1052 isb(); 1053 } 1054 1055 switch(pribits) { 1056 case 8: 1057 case 7: 1058 write_gicreg(0, ICC_AP1R3_EL1); 1059 write_gicreg(0, ICC_AP1R2_EL1); 1060 fallthrough; 1061 case 6: 1062 write_gicreg(0, ICC_AP1R1_EL1); 1063 fallthrough; 1064 case 5: 1065 case 4: 1066 write_gicreg(0, ICC_AP1R0_EL1); 1067 } 1068 1069 isb(); 1070 1071 /* ... and let's hit the road... */ 1072 gic_write_grpen1(1); 1073 1074 /* Keep the RSS capability status in per_cpu variable */ 1075 per_cpu(has_rss, cpu) = !!(gic_read_ctlr() & ICC_CTLR_EL1_RSS); 1076 1077 /* Check all the CPUs have capable of sending SGIs to other CPUs */ 1078 for_each_online_cpu(i) { 1079 bool have_rss = per_cpu(has_rss, i) && per_cpu(has_rss, cpu); 1080 1081 need_rss |= MPIDR_RS(cpu_logical_map(i)); 1082 if (need_rss && (!have_rss)) 1083 pr_crit("CPU%d (%lx) can't SGI CPU%d (%lx), no RSS\n", 1084 cpu, (unsigned long)mpidr, 1085 i, (unsigned long)cpu_logical_map(i)); 1086 } 1087 1088 /** 1089 * GIC spec says, when ICC_CTLR_EL1.RSS==1 and GICD_TYPER.RSS==0, 1090 * writing ICC_ASGI1R_EL1 register with RS != 0 is a CONSTRAINED 1091 * UNPREDICTABLE choice of : 1092 * - The write is ignored. 1093 * - The RS field is treated as 0. 1094 */ 1095 if (need_rss && (!gic_data.has_rss)) 1096 pr_crit_once("RSS is required but GICD doesn't support it\n"); 1097 } 1098 1099 static bool gicv3_nolpi; 1100 1101 static int __init gicv3_nolpi_cfg(char *buf) 1102 { 1103 return strtobool(buf, &gicv3_nolpi); 1104 } 1105 early_param("irqchip.gicv3_nolpi", gicv3_nolpi_cfg); 1106 1107 static int gic_dist_supports_lpis(void) 1108 { 1109 return (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) && 1110 !!(readl_relaxed(gic_data.dist_base + GICD_TYPER) & GICD_TYPER_LPIS) && 1111 !gicv3_nolpi); 1112 } 1113 1114 static void gic_cpu_init(void) 1115 { 1116 void __iomem *rbase; 1117 int i; 1118 1119 /* Register ourselves with the rest of the world */ 1120 if (gic_populate_rdist()) 1121 return; 1122 1123 gic_enable_redist(true); 1124 1125 WARN((gic_data.ppi_nr > 16 || GIC_ESPI_NR != 0) && 1126 !(gic_read_ctlr() & ICC_CTLR_EL1_ExtRange), 1127 "Distributor has extended ranges, but CPU%d doesn't\n", 1128 smp_processor_id()); 1129 1130 rbase = gic_data_rdist_sgi_base(); 1131 1132 /* Configure SGIs/PPIs as non-secure Group-1 */ 1133 for (i = 0; i < gic_data.ppi_nr + 16; i += 32) 1134 writel_relaxed(~0, rbase + GICR_IGROUPR0 + i / 8); 1135 1136 gic_cpu_config(rbase, gic_data.ppi_nr + 16, gic_redist_wait_for_rwp); 1137 1138 /* initialise system registers */ 1139 gic_cpu_sys_reg_init(); 1140 } 1141 1142 #ifdef CONFIG_SMP 1143 1144 #define MPIDR_TO_SGI_RS(mpidr) (MPIDR_RS(mpidr) << ICC_SGI1R_RS_SHIFT) 1145 #define MPIDR_TO_SGI_CLUSTER_ID(mpidr) ((mpidr) & ~0xFUL) 1146 1147 static int gic_starting_cpu(unsigned int cpu) 1148 { 1149 gic_cpu_init(); 1150 1151 if (gic_dist_supports_lpis()) 1152 its_cpu_init(); 1153 1154 return 0; 1155 } 1156 1157 static u16 gic_compute_target_list(int *base_cpu, const struct cpumask *mask, 1158 unsigned long cluster_id) 1159 { 1160 int next_cpu, cpu = *base_cpu; 1161 unsigned long mpidr = cpu_logical_map(cpu); 1162 u16 tlist = 0; 1163 1164 while (cpu < nr_cpu_ids) { 1165 tlist |= 1 << (mpidr & 0xf); 1166 1167 next_cpu = cpumask_next(cpu, mask); 1168 if (next_cpu >= nr_cpu_ids) 1169 goto out; 1170 cpu = next_cpu; 1171 1172 mpidr = cpu_logical_map(cpu); 1173 1174 if (cluster_id != MPIDR_TO_SGI_CLUSTER_ID(mpidr)) { 1175 cpu--; 1176 goto out; 1177 } 1178 } 1179 out: 1180 *base_cpu = cpu; 1181 return tlist; 1182 } 1183 1184 #define MPIDR_TO_SGI_AFFINITY(cluster_id, level) \ 1185 (MPIDR_AFFINITY_LEVEL(cluster_id, level) \ 1186 << ICC_SGI1R_AFFINITY_## level ##_SHIFT) 1187 1188 static void gic_send_sgi(u64 cluster_id, u16 tlist, unsigned int irq) 1189 { 1190 u64 val; 1191 1192 val = (MPIDR_TO_SGI_AFFINITY(cluster_id, 3) | 1193 MPIDR_TO_SGI_AFFINITY(cluster_id, 2) | 1194 irq << ICC_SGI1R_SGI_ID_SHIFT | 1195 MPIDR_TO_SGI_AFFINITY(cluster_id, 1) | 1196 MPIDR_TO_SGI_RS(cluster_id) | 1197 tlist << ICC_SGI1R_TARGET_LIST_SHIFT); 1198 1199 pr_devel("CPU%d: ICC_SGI1R_EL1 %llx\n", smp_processor_id(), val); 1200 gic_write_sgi1r(val); 1201 } 1202 1203 static void gic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask) 1204 { 1205 int cpu; 1206 1207 if (WARN_ON(d->hwirq >= 16)) 1208 return; 1209 1210 /* 1211 * Ensure that stores to Normal memory are visible to the 1212 * other CPUs before issuing the IPI. 1213 */ 1214 dsb(ishst); 1215 1216 for_each_cpu(cpu, mask) { 1217 u64 cluster_id = MPIDR_TO_SGI_CLUSTER_ID(cpu_logical_map(cpu)); 1218 u16 tlist; 1219 1220 tlist = gic_compute_target_list(&cpu, mask, cluster_id); 1221 gic_send_sgi(cluster_id, tlist, d->hwirq); 1222 } 1223 1224 /* Force the above writes to ICC_SGI1R_EL1 to be executed */ 1225 isb(); 1226 } 1227 1228 static void __init gic_smp_init(void) 1229 { 1230 struct irq_fwspec sgi_fwspec = { 1231 .fwnode = gic_data.fwnode, 1232 .param_count = 1, 1233 }; 1234 int base_sgi; 1235 1236 cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_GIC_STARTING, 1237 "irqchip/arm/gicv3:starting", 1238 gic_starting_cpu, NULL); 1239 1240 /* Register all 8 non-secure SGIs */ 1241 base_sgi = __irq_domain_alloc_irqs(gic_data.domain, -1, 8, 1242 NUMA_NO_NODE, &sgi_fwspec, 1243 false, NULL); 1244 if (WARN_ON(base_sgi <= 0)) 1245 return; 1246 1247 set_smp_ipi_range(base_sgi, 8); 1248 } 1249 1250 static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val, 1251 bool force) 1252 { 1253 unsigned int cpu; 1254 u32 offset, index; 1255 void __iomem *reg; 1256 int enabled; 1257 u64 val; 1258 1259 if (force) 1260 cpu = cpumask_first(mask_val); 1261 else 1262 cpu = cpumask_any_and(mask_val, cpu_online_mask); 1263 1264 if (cpu >= nr_cpu_ids) 1265 return -EINVAL; 1266 1267 if (gic_irq_in_rdist(d)) 1268 return -EINVAL; 1269 1270 /* If interrupt was enabled, disable it first */ 1271 enabled = gic_peek_irq(d, GICD_ISENABLER); 1272 if (enabled) 1273 gic_mask_irq(d); 1274 1275 offset = convert_offset_index(d, GICD_IROUTER, &index); 1276 reg = gic_dist_base(d) + offset + (index * 8); 1277 val = gic_mpidr_to_affinity(cpu_logical_map(cpu)); 1278 1279 gic_write_irouter(val, reg); 1280 1281 /* 1282 * If the interrupt was enabled, enabled it again. Otherwise, 1283 * just wait for the distributor to have digested our changes. 1284 */ 1285 if (enabled) 1286 gic_unmask_irq(d); 1287 else 1288 gic_dist_wait_for_rwp(); 1289 1290 irq_data_update_effective_affinity(d, cpumask_of(cpu)); 1291 1292 return IRQ_SET_MASK_OK_DONE; 1293 } 1294 #else 1295 #define gic_set_affinity NULL 1296 #define gic_ipi_send_mask NULL 1297 #define gic_smp_init() do { } while(0) 1298 #endif 1299 1300 static int gic_retrigger(struct irq_data *data) 1301 { 1302 return !gic_irq_set_irqchip_state(data, IRQCHIP_STATE_PENDING, true); 1303 } 1304 1305 #ifdef CONFIG_CPU_PM 1306 static int gic_cpu_pm_notifier(struct notifier_block *self, 1307 unsigned long cmd, void *v) 1308 { 1309 if (cmd == CPU_PM_EXIT) { 1310 if (gic_dist_security_disabled()) 1311 gic_enable_redist(true); 1312 gic_cpu_sys_reg_init(); 1313 } else if (cmd == CPU_PM_ENTER && gic_dist_security_disabled()) { 1314 gic_write_grpen1(0); 1315 gic_enable_redist(false); 1316 } 1317 return NOTIFY_OK; 1318 } 1319 1320 static struct notifier_block gic_cpu_pm_notifier_block = { 1321 .notifier_call = gic_cpu_pm_notifier, 1322 }; 1323 1324 static void gic_cpu_pm_init(void) 1325 { 1326 cpu_pm_register_notifier(&gic_cpu_pm_notifier_block); 1327 } 1328 1329 #else 1330 static inline void gic_cpu_pm_init(void) { } 1331 #endif /* CONFIG_CPU_PM */ 1332 1333 static struct irq_chip gic_chip = { 1334 .name = "GICv3", 1335 .irq_mask = gic_mask_irq, 1336 .irq_unmask = gic_unmask_irq, 1337 .irq_eoi = gic_eoi_irq, 1338 .irq_set_type = gic_set_type, 1339 .irq_set_affinity = gic_set_affinity, 1340 .irq_retrigger = gic_retrigger, 1341 .irq_get_irqchip_state = gic_irq_get_irqchip_state, 1342 .irq_set_irqchip_state = gic_irq_set_irqchip_state, 1343 .irq_nmi_setup = gic_irq_nmi_setup, 1344 .irq_nmi_teardown = gic_irq_nmi_teardown, 1345 .ipi_send_mask = gic_ipi_send_mask, 1346 .flags = IRQCHIP_SET_TYPE_MASKED | 1347 IRQCHIP_SKIP_SET_WAKE | 1348 IRQCHIP_MASK_ON_SUSPEND, 1349 }; 1350 1351 static struct irq_chip gic_eoimode1_chip = { 1352 .name = "GICv3", 1353 .irq_mask = gic_eoimode1_mask_irq, 1354 .irq_unmask = gic_unmask_irq, 1355 .irq_eoi = gic_eoimode1_eoi_irq, 1356 .irq_set_type = gic_set_type, 1357 .irq_set_affinity = gic_set_affinity, 1358 .irq_retrigger = gic_retrigger, 1359 .irq_get_irqchip_state = gic_irq_get_irqchip_state, 1360 .irq_set_irqchip_state = gic_irq_set_irqchip_state, 1361 .irq_set_vcpu_affinity = gic_irq_set_vcpu_affinity, 1362 .irq_nmi_setup = gic_irq_nmi_setup, 1363 .irq_nmi_teardown = gic_irq_nmi_teardown, 1364 .ipi_send_mask = gic_ipi_send_mask, 1365 .flags = IRQCHIP_SET_TYPE_MASKED | 1366 IRQCHIP_SKIP_SET_WAKE | 1367 IRQCHIP_MASK_ON_SUSPEND, 1368 }; 1369 1370 static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq, 1371 irq_hw_number_t hw) 1372 { 1373 struct irq_chip *chip = &gic_chip; 1374 struct irq_data *irqd = irq_desc_get_irq_data(irq_to_desc(irq)); 1375 1376 if (static_branch_likely(&supports_deactivate_key)) 1377 chip = &gic_eoimode1_chip; 1378 1379 switch (__get_intid_range(hw)) { 1380 case SGI_RANGE: 1381 case PPI_RANGE: 1382 case EPPI_RANGE: 1383 irq_set_percpu_devid(irq); 1384 irq_domain_set_info(d, irq, hw, chip, d->host_data, 1385 handle_percpu_devid_irq, NULL, NULL); 1386 break; 1387 1388 case SPI_RANGE: 1389 case ESPI_RANGE: 1390 irq_domain_set_info(d, irq, hw, chip, d->host_data, 1391 handle_fasteoi_irq, NULL, NULL); 1392 irq_set_probe(irq); 1393 irqd_set_single_target(irqd); 1394 break; 1395 1396 case LPI_RANGE: 1397 if (!gic_dist_supports_lpis()) 1398 return -EPERM; 1399 irq_domain_set_info(d, irq, hw, chip, d->host_data, 1400 handle_fasteoi_irq, NULL, NULL); 1401 break; 1402 1403 default: 1404 return -EPERM; 1405 } 1406 1407 /* Prevents SW retriggers which mess up the ACK/EOI ordering */ 1408 irqd_set_handle_enforce_irqctx(irqd); 1409 return 0; 1410 } 1411 1412 static int gic_irq_domain_translate(struct irq_domain *d, 1413 struct irq_fwspec *fwspec, 1414 unsigned long *hwirq, 1415 unsigned int *type) 1416 { 1417 if (fwspec->param_count == 1 && fwspec->param[0] < 16) { 1418 *hwirq = fwspec->param[0]; 1419 *type = IRQ_TYPE_EDGE_RISING; 1420 return 0; 1421 } 1422 1423 if (is_of_node(fwspec->fwnode)) { 1424 if (fwspec->param_count < 3) 1425 return -EINVAL; 1426 1427 switch (fwspec->param[0]) { 1428 case 0: /* SPI */ 1429 *hwirq = fwspec->param[1] + 32; 1430 break; 1431 case 1: /* PPI */ 1432 *hwirq = fwspec->param[1] + 16; 1433 break; 1434 case 2: /* ESPI */ 1435 *hwirq = fwspec->param[1] + ESPI_BASE_INTID; 1436 break; 1437 case 3: /* EPPI */ 1438 *hwirq = fwspec->param[1] + EPPI_BASE_INTID; 1439 break; 1440 case GIC_IRQ_TYPE_LPI: /* LPI */ 1441 *hwirq = fwspec->param[1]; 1442 break; 1443 case GIC_IRQ_TYPE_PARTITION: 1444 *hwirq = fwspec->param[1]; 1445 if (fwspec->param[1] >= 16) 1446 *hwirq += EPPI_BASE_INTID - 16; 1447 else 1448 *hwirq += 16; 1449 break; 1450 default: 1451 return -EINVAL; 1452 } 1453 1454 *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK; 1455 1456 /* 1457 * Make it clear that broken DTs are... broken. 1458 * Partitioned PPIs are an unfortunate exception. 1459 */ 1460 WARN_ON(*type == IRQ_TYPE_NONE && 1461 fwspec->param[0] != GIC_IRQ_TYPE_PARTITION); 1462 return 0; 1463 } 1464 1465 if (is_fwnode_irqchip(fwspec->fwnode)) { 1466 if(fwspec->param_count != 2) 1467 return -EINVAL; 1468 1469 if (fwspec->param[0] < 16) { 1470 pr_err(FW_BUG "Illegal GSI%d translation request\n", 1471 fwspec->param[0]); 1472 return -EINVAL; 1473 } 1474 1475 *hwirq = fwspec->param[0]; 1476 *type = fwspec->param[1]; 1477 1478 WARN_ON(*type == IRQ_TYPE_NONE); 1479 return 0; 1480 } 1481 1482 return -EINVAL; 1483 } 1484 1485 static int gic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, 1486 unsigned int nr_irqs, void *arg) 1487 { 1488 int i, ret; 1489 irq_hw_number_t hwirq; 1490 unsigned int type = IRQ_TYPE_NONE; 1491 struct irq_fwspec *fwspec = arg; 1492 1493 ret = gic_irq_domain_translate(domain, fwspec, &hwirq, &type); 1494 if (ret) 1495 return ret; 1496 1497 for (i = 0; i < nr_irqs; i++) { 1498 ret = gic_irq_domain_map(domain, virq + i, hwirq + i); 1499 if (ret) 1500 return ret; 1501 } 1502 1503 return 0; 1504 } 1505 1506 static void gic_irq_domain_free(struct irq_domain *domain, unsigned int virq, 1507 unsigned int nr_irqs) 1508 { 1509 int i; 1510 1511 for (i = 0; i < nr_irqs; i++) { 1512 struct irq_data *d = irq_domain_get_irq_data(domain, virq + i); 1513 irq_set_handler(virq + i, NULL); 1514 irq_domain_reset_irq_data(d); 1515 } 1516 } 1517 1518 static bool fwspec_is_partitioned_ppi(struct irq_fwspec *fwspec, 1519 irq_hw_number_t hwirq) 1520 { 1521 enum gic_intid_range range; 1522 1523 if (!gic_data.ppi_descs) 1524 return false; 1525 1526 if (!is_of_node(fwspec->fwnode)) 1527 return false; 1528 1529 if (fwspec->param_count < 4 || !fwspec->param[3]) 1530 return false; 1531 1532 range = __get_intid_range(hwirq); 1533 if (range != PPI_RANGE && range != EPPI_RANGE) 1534 return false; 1535 1536 return true; 1537 } 1538 1539 static int gic_irq_domain_select(struct irq_domain *d, 1540 struct irq_fwspec *fwspec, 1541 enum irq_domain_bus_token bus_token) 1542 { 1543 unsigned int type, ret, ppi_idx; 1544 irq_hw_number_t hwirq; 1545 1546 /* Not for us */ 1547 if (fwspec->fwnode != d->fwnode) 1548 return 0; 1549 1550 /* If this is not DT, then we have a single domain */ 1551 if (!is_of_node(fwspec->fwnode)) 1552 return 1; 1553 1554 ret = gic_irq_domain_translate(d, fwspec, &hwirq, &type); 1555 if (WARN_ON_ONCE(ret)) 1556 return 0; 1557 1558 if (!fwspec_is_partitioned_ppi(fwspec, hwirq)) 1559 return d == gic_data.domain; 1560 1561 /* 1562 * If this is a PPI and we have a 4th (non-null) parameter, 1563 * then we need to match the partition domain. 1564 */ 1565 ppi_idx = __gic_get_ppi_index(hwirq); 1566 return d == partition_get_domain(gic_data.ppi_descs[ppi_idx]); 1567 } 1568 1569 static const struct irq_domain_ops gic_irq_domain_ops = { 1570 .translate = gic_irq_domain_translate, 1571 .alloc = gic_irq_domain_alloc, 1572 .free = gic_irq_domain_free, 1573 .select = gic_irq_domain_select, 1574 }; 1575 1576 static int partition_domain_translate(struct irq_domain *d, 1577 struct irq_fwspec *fwspec, 1578 unsigned long *hwirq, 1579 unsigned int *type) 1580 { 1581 unsigned long ppi_intid; 1582 struct device_node *np; 1583 unsigned int ppi_idx; 1584 int ret; 1585 1586 if (!gic_data.ppi_descs) 1587 return -ENOMEM; 1588 1589 np = of_find_node_by_phandle(fwspec->param[3]); 1590 if (WARN_ON(!np)) 1591 return -EINVAL; 1592 1593 ret = gic_irq_domain_translate(d, fwspec, &ppi_intid, type); 1594 if (WARN_ON_ONCE(ret)) 1595 return 0; 1596 1597 ppi_idx = __gic_get_ppi_index(ppi_intid); 1598 ret = partition_translate_id(gic_data.ppi_descs[ppi_idx], 1599 of_node_to_fwnode(np)); 1600 if (ret < 0) 1601 return ret; 1602 1603 *hwirq = ret; 1604 *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK; 1605 1606 return 0; 1607 } 1608 1609 static const struct irq_domain_ops partition_domain_ops = { 1610 .translate = partition_domain_translate, 1611 .select = gic_irq_domain_select, 1612 }; 1613 1614 static bool gic_enable_quirk_msm8996(void *data) 1615 { 1616 struct gic_chip_data *d = data; 1617 1618 d->flags |= FLAGS_WORKAROUND_GICR_WAKER_MSM8996; 1619 1620 return true; 1621 } 1622 1623 static bool gic_enable_quirk_cavium_38539(void *data) 1624 { 1625 struct gic_chip_data *d = data; 1626 1627 d->flags |= FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539; 1628 1629 return true; 1630 } 1631 1632 static bool gic_enable_quirk_hip06_07(void *data) 1633 { 1634 struct gic_chip_data *d = data; 1635 1636 /* 1637 * HIP06 GICD_IIDR clashes with GIC-600 product number (despite 1638 * not being an actual ARM implementation). The saving grace is 1639 * that GIC-600 doesn't have ESPI, so nothing to do in that case. 1640 * HIP07 doesn't even have a proper IIDR, and still pretends to 1641 * have ESPI. In both cases, put them right. 1642 */ 1643 if (d->rdists.gicd_typer & GICD_TYPER_ESPI) { 1644 /* Zero both ESPI and the RES0 field next to it... */ 1645 d->rdists.gicd_typer &= ~GENMASK(9, 8); 1646 return true; 1647 } 1648 1649 return false; 1650 } 1651 1652 static const struct gic_quirk gic_quirks[] = { 1653 { 1654 .desc = "GICv3: Qualcomm MSM8996 broken firmware", 1655 .compatible = "qcom,msm8996-gic-v3", 1656 .init = gic_enable_quirk_msm8996, 1657 }, 1658 { 1659 .desc = "GICv3: HIP06 erratum 161010803", 1660 .iidr = 0x0204043b, 1661 .mask = 0xffffffff, 1662 .init = gic_enable_quirk_hip06_07, 1663 }, 1664 { 1665 .desc = "GICv3: HIP07 erratum 161010803", 1666 .iidr = 0x00000000, 1667 .mask = 0xffffffff, 1668 .init = gic_enable_quirk_hip06_07, 1669 }, 1670 { 1671 /* 1672 * Reserved register accesses generate a Synchronous 1673 * External Abort. This erratum applies to: 1674 * - ThunderX: CN88xx 1675 * - OCTEON TX: CN83xx, CN81xx 1676 * - OCTEON TX2: CN93xx, CN96xx, CN98xx, CNF95xx* 1677 */ 1678 .desc = "GICv3: Cavium erratum 38539", 1679 .iidr = 0xa000034c, 1680 .mask = 0xe8f00fff, 1681 .init = gic_enable_quirk_cavium_38539, 1682 }, 1683 { 1684 } 1685 }; 1686 1687 static void gic_enable_nmi_support(void) 1688 { 1689 int i; 1690 1691 if (!gic_prio_masking_enabled()) 1692 return; 1693 1694 ppi_nmi_refs = kcalloc(gic_data.ppi_nr, sizeof(*ppi_nmi_refs), GFP_KERNEL); 1695 if (!ppi_nmi_refs) 1696 return; 1697 1698 for (i = 0; i < gic_data.ppi_nr; i++) 1699 refcount_set(&ppi_nmi_refs[i], 0); 1700 1701 /* 1702 * Linux itself doesn't use 1:N distribution, so has no need to 1703 * set PMHE. The only reason to have it set is if EL3 requires it 1704 * (and we can't change it). 1705 */ 1706 if (gic_read_ctlr() & ICC_CTLR_EL1_PMHE_MASK) 1707 static_branch_enable(&gic_pmr_sync); 1708 1709 pr_info("Pseudo-NMIs enabled using %s ICC_PMR_EL1 synchronisation\n", 1710 static_branch_unlikely(&gic_pmr_sync) ? "forced" : "relaxed"); 1711 1712 /* 1713 * How priority values are used by the GIC depends on two things: 1714 * the security state of the GIC (controlled by the GICD_CTRL.DS bit) 1715 * and if Group 0 interrupts can be delivered to Linux in the non-secure 1716 * world as FIQs (controlled by the SCR_EL3.FIQ bit). These affect the 1717 * the ICC_PMR_EL1 register and the priority that software assigns to 1718 * interrupts: 1719 * 1720 * GICD_CTRL.DS | SCR_EL3.FIQ | ICC_PMR_EL1 | Group 1 priority 1721 * ----------------------------------------------------------- 1722 * 1 | - | unchanged | unchanged 1723 * ----------------------------------------------------------- 1724 * 0 | 1 | non-secure | non-secure 1725 * ----------------------------------------------------------- 1726 * 0 | 0 | unchanged | non-secure 1727 * 1728 * where non-secure means that the value is right-shifted by one and the 1729 * MSB bit set, to make it fit in the non-secure priority range. 1730 * 1731 * In the first two cases, where ICC_PMR_EL1 and the interrupt priority 1732 * are both either modified or unchanged, we can use the same set of 1733 * priorities. 1734 * 1735 * In the last case, where only the interrupt priorities are modified to 1736 * be in the non-secure range, we use a different PMR value to mask IRQs 1737 * and the rest of the values that we use remain unchanged. 1738 */ 1739 if (gic_has_group0() && !gic_dist_security_disabled()) 1740 static_branch_enable(&gic_nonsecure_priorities); 1741 1742 static_branch_enable(&supports_pseudo_nmis); 1743 1744 if (static_branch_likely(&supports_deactivate_key)) 1745 gic_eoimode1_chip.flags |= IRQCHIP_SUPPORTS_NMI; 1746 else 1747 gic_chip.flags |= IRQCHIP_SUPPORTS_NMI; 1748 } 1749 1750 static int __init gic_init_bases(void __iomem *dist_base, 1751 struct redist_region *rdist_regs, 1752 u32 nr_redist_regions, 1753 u64 redist_stride, 1754 struct fwnode_handle *handle) 1755 { 1756 u32 typer; 1757 int err; 1758 1759 if (!is_hyp_mode_available()) 1760 static_branch_disable(&supports_deactivate_key); 1761 1762 if (static_branch_likely(&supports_deactivate_key)) 1763 pr_info("GIC: Using split EOI/Deactivate mode\n"); 1764 1765 gic_data.fwnode = handle; 1766 gic_data.dist_base = dist_base; 1767 gic_data.redist_regions = rdist_regs; 1768 gic_data.nr_redist_regions = nr_redist_regions; 1769 gic_data.redist_stride = redist_stride; 1770 1771 /* 1772 * Find out how many interrupts are supported. 1773 */ 1774 typer = readl_relaxed(gic_data.dist_base + GICD_TYPER); 1775 gic_data.rdists.gicd_typer = typer; 1776 1777 gic_enable_quirks(readl_relaxed(gic_data.dist_base + GICD_IIDR), 1778 gic_quirks, &gic_data); 1779 1780 pr_info("%d SPIs implemented\n", GIC_LINE_NR - 32); 1781 pr_info("%d Extended SPIs implemented\n", GIC_ESPI_NR); 1782 1783 /* 1784 * ThunderX1 explodes on reading GICD_TYPER2, in violation of the 1785 * architecture spec (which says that reserved registers are RES0). 1786 */ 1787 if (!(gic_data.flags & FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539)) 1788 gic_data.rdists.gicd_typer2 = readl_relaxed(gic_data.dist_base + GICD_TYPER2); 1789 1790 gic_data.domain = irq_domain_create_tree(handle, &gic_irq_domain_ops, 1791 &gic_data); 1792 gic_data.rdists.rdist = alloc_percpu(typeof(*gic_data.rdists.rdist)); 1793 gic_data.rdists.has_rvpeid = true; 1794 gic_data.rdists.has_vlpis = true; 1795 gic_data.rdists.has_direct_lpi = true; 1796 gic_data.rdists.has_vpend_valid_dirty = true; 1797 1798 if (WARN_ON(!gic_data.domain) || WARN_ON(!gic_data.rdists.rdist)) { 1799 err = -ENOMEM; 1800 goto out_free; 1801 } 1802 1803 irq_domain_update_bus_token(gic_data.domain, DOMAIN_BUS_WIRED); 1804 1805 gic_data.has_rss = !!(typer & GICD_TYPER_RSS); 1806 pr_info("Distributor has %sRange Selector support\n", 1807 gic_data.has_rss ? "" : "no "); 1808 1809 if (typer & GICD_TYPER_MBIS) { 1810 err = mbi_init(handle, gic_data.domain); 1811 if (err) 1812 pr_err("Failed to initialize MBIs\n"); 1813 } 1814 1815 set_handle_irq(gic_handle_irq); 1816 1817 gic_update_rdist_properties(); 1818 1819 gic_dist_init(); 1820 gic_cpu_init(); 1821 gic_smp_init(); 1822 gic_cpu_pm_init(); 1823 1824 if (gic_dist_supports_lpis()) { 1825 its_init(handle, &gic_data.rdists, gic_data.domain); 1826 its_cpu_init(); 1827 its_lpi_memreserve_init(); 1828 } else { 1829 if (IS_ENABLED(CONFIG_ARM_GIC_V2M)) 1830 gicv2m_init(handle, gic_data.domain); 1831 } 1832 1833 gic_enable_nmi_support(); 1834 1835 return 0; 1836 1837 out_free: 1838 if (gic_data.domain) 1839 irq_domain_remove(gic_data.domain); 1840 free_percpu(gic_data.rdists.rdist); 1841 return err; 1842 } 1843 1844 static int __init gic_validate_dist_version(void __iomem *dist_base) 1845 { 1846 u32 reg = readl_relaxed(dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK; 1847 1848 if (reg != GIC_PIDR2_ARCH_GICv3 && reg != GIC_PIDR2_ARCH_GICv4) 1849 return -ENODEV; 1850 1851 return 0; 1852 } 1853 1854 /* Create all possible partitions at boot time */ 1855 static void __init gic_populate_ppi_partitions(struct device_node *gic_node) 1856 { 1857 struct device_node *parts_node, *child_part; 1858 int part_idx = 0, i; 1859 int nr_parts; 1860 struct partition_affinity *parts; 1861 1862 parts_node = of_get_child_by_name(gic_node, "ppi-partitions"); 1863 if (!parts_node) 1864 return; 1865 1866 gic_data.ppi_descs = kcalloc(gic_data.ppi_nr, sizeof(*gic_data.ppi_descs), GFP_KERNEL); 1867 if (!gic_data.ppi_descs) 1868 return; 1869 1870 nr_parts = of_get_child_count(parts_node); 1871 1872 if (!nr_parts) 1873 goto out_put_node; 1874 1875 parts = kcalloc(nr_parts, sizeof(*parts), GFP_KERNEL); 1876 if (WARN_ON(!parts)) 1877 goto out_put_node; 1878 1879 for_each_child_of_node(parts_node, child_part) { 1880 struct partition_affinity *part; 1881 int n; 1882 1883 part = &parts[part_idx]; 1884 1885 part->partition_id = of_node_to_fwnode(child_part); 1886 1887 pr_info("GIC: PPI partition %pOFn[%d] { ", 1888 child_part, part_idx); 1889 1890 n = of_property_count_elems_of_size(child_part, "affinity", 1891 sizeof(u32)); 1892 WARN_ON(n <= 0); 1893 1894 for (i = 0; i < n; i++) { 1895 int err, cpu; 1896 u32 cpu_phandle; 1897 struct device_node *cpu_node; 1898 1899 err = of_property_read_u32_index(child_part, "affinity", 1900 i, &cpu_phandle); 1901 if (WARN_ON(err)) 1902 continue; 1903 1904 cpu_node = of_find_node_by_phandle(cpu_phandle); 1905 if (WARN_ON(!cpu_node)) 1906 continue; 1907 1908 cpu = of_cpu_node_to_id(cpu_node); 1909 if (WARN_ON(cpu < 0)) 1910 continue; 1911 1912 pr_cont("%pOF[%d] ", cpu_node, cpu); 1913 1914 cpumask_set_cpu(cpu, &part->mask); 1915 } 1916 1917 pr_cont("}\n"); 1918 part_idx++; 1919 } 1920 1921 for (i = 0; i < gic_data.ppi_nr; i++) { 1922 unsigned int irq; 1923 struct partition_desc *desc; 1924 struct irq_fwspec ppi_fwspec = { 1925 .fwnode = gic_data.fwnode, 1926 .param_count = 3, 1927 .param = { 1928 [0] = GIC_IRQ_TYPE_PARTITION, 1929 [1] = i, 1930 [2] = IRQ_TYPE_NONE, 1931 }, 1932 }; 1933 1934 irq = irq_create_fwspec_mapping(&ppi_fwspec); 1935 if (WARN_ON(!irq)) 1936 continue; 1937 desc = partition_create_desc(gic_data.fwnode, parts, nr_parts, 1938 irq, &partition_domain_ops); 1939 if (WARN_ON(!desc)) 1940 continue; 1941 1942 gic_data.ppi_descs[i] = desc; 1943 } 1944 1945 out_put_node: 1946 of_node_put(parts_node); 1947 } 1948 1949 static void __init gic_of_setup_kvm_info(struct device_node *node) 1950 { 1951 int ret; 1952 struct resource r; 1953 u32 gicv_idx; 1954 1955 gic_v3_kvm_info.type = GIC_V3; 1956 1957 gic_v3_kvm_info.maint_irq = irq_of_parse_and_map(node, 0); 1958 if (!gic_v3_kvm_info.maint_irq) 1959 return; 1960 1961 if (of_property_read_u32(node, "#redistributor-regions", 1962 &gicv_idx)) 1963 gicv_idx = 1; 1964 1965 gicv_idx += 3; /* Also skip GICD, GICC, GICH */ 1966 ret = of_address_to_resource(node, gicv_idx, &r); 1967 if (!ret) 1968 gic_v3_kvm_info.vcpu = r; 1969 1970 gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis; 1971 gic_v3_kvm_info.has_v4_1 = gic_data.rdists.has_rvpeid; 1972 vgic_set_kvm_info(&gic_v3_kvm_info); 1973 } 1974 1975 static int __init gic_of_init(struct device_node *node, struct device_node *parent) 1976 { 1977 void __iomem *dist_base; 1978 struct redist_region *rdist_regs; 1979 u64 redist_stride; 1980 u32 nr_redist_regions; 1981 int err, i; 1982 1983 dist_base = of_iomap(node, 0); 1984 if (!dist_base) { 1985 pr_err("%pOF: unable to map gic dist registers\n", node); 1986 return -ENXIO; 1987 } 1988 1989 err = gic_validate_dist_version(dist_base); 1990 if (err) { 1991 pr_err("%pOF: no distributor detected, giving up\n", node); 1992 goto out_unmap_dist; 1993 } 1994 1995 if (of_property_read_u32(node, "#redistributor-regions", &nr_redist_regions)) 1996 nr_redist_regions = 1; 1997 1998 rdist_regs = kcalloc(nr_redist_regions, sizeof(*rdist_regs), 1999 GFP_KERNEL); 2000 if (!rdist_regs) { 2001 err = -ENOMEM; 2002 goto out_unmap_dist; 2003 } 2004 2005 for (i = 0; i < nr_redist_regions; i++) { 2006 struct resource res; 2007 int ret; 2008 2009 ret = of_address_to_resource(node, 1 + i, &res); 2010 rdist_regs[i].redist_base = of_iomap(node, 1 + i); 2011 if (ret || !rdist_regs[i].redist_base) { 2012 pr_err("%pOF: couldn't map region %d\n", node, i); 2013 err = -ENODEV; 2014 goto out_unmap_rdist; 2015 } 2016 rdist_regs[i].phys_base = res.start; 2017 } 2018 2019 if (of_property_read_u64(node, "redistributor-stride", &redist_stride)) 2020 redist_stride = 0; 2021 2022 gic_enable_of_quirks(node, gic_quirks, &gic_data); 2023 2024 err = gic_init_bases(dist_base, rdist_regs, nr_redist_regions, 2025 redist_stride, &node->fwnode); 2026 if (err) 2027 goto out_unmap_rdist; 2028 2029 gic_populate_ppi_partitions(node); 2030 2031 if (static_branch_likely(&supports_deactivate_key)) 2032 gic_of_setup_kvm_info(node); 2033 return 0; 2034 2035 out_unmap_rdist: 2036 for (i = 0; i < nr_redist_regions; i++) 2037 if (rdist_regs[i].redist_base) 2038 iounmap(rdist_regs[i].redist_base); 2039 kfree(rdist_regs); 2040 out_unmap_dist: 2041 iounmap(dist_base); 2042 return err; 2043 } 2044 2045 IRQCHIP_DECLARE(gic_v3, "arm,gic-v3", gic_of_init); 2046 2047 #ifdef CONFIG_ACPI 2048 static struct 2049 { 2050 void __iomem *dist_base; 2051 struct redist_region *redist_regs; 2052 u32 nr_redist_regions; 2053 bool single_redist; 2054 int enabled_rdists; 2055 u32 maint_irq; 2056 int maint_irq_mode; 2057 phys_addr_t vcpu_base; 2058 } acpi_data __initdata; 2059 2060 static void __init 2061 gic_acpi_register_redist(phys_addr_t phys_base, void __iomem *redist_base) 2062 { 2063 static int count = 0; 2064 2065 acpi_data.redist_regs[count].phys_base = phys_base; 2066 acpi_data.redist_regs[count].redist_base = redist_base; 2067 acpi_data.redist_regs[count].single_redist = acpi_data.single_redist; 2068 count++; 2069 } 2070 2071 static int __init 2072 gic_acpi_parse_madt_redist(union acpi_subtable_headers *header, 2073 const unsigned long end) 2074 { 2075 struct acpi_madt_generic_redistributor *redist = 2076 (struct acpi_madt_generic_redistributor *)header; 2077 void __iomem *redist_base; 2078 2079 redist_base = ioremap(redist->base_address, redist->length); 2080 if (!redist_base) { 2081 pr_err("Couldn't map GICR region @%llx\n", redist->base_address); 2082 return -ENOMEM; 2083 } 2084 2085 gic_acpi_register_redist(redist->base_address, redist_base); 2086 return 0; 2087 } 2088 2089 static int __init 2090 gic_acpi_parse_madt_gicc(union acpi_subtable_headers *header, 2091 const unsigned long end) 2092 { 2093 struct acpi_madt_generic_interrupt *gicc = 2094 (struct acpi_madt_generic_interrupt *)header; 2095 u32 reg = readl_relaxed(acpi_data.dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK; 2096 u32 size = reg == GIC_PIDR2_ARCH_GICv4 ? SZ_64K * 4 : SZ_64K * 2; 2097 void __iomem *redist_base; 2098 2099 /* GICC entry which has !ACPI_MADT_ENABLED is not unusable so skip */ 2100 if (!(gicc->flags & ACPI_MADT_ENABLED)) 2101 return 0; 2102 2103 redist_base = ioremap(gicc->gicr_base_address, size); 2104 if (!redist_base) 2105 return -ENOMEM; 2106 2107 gic_acpi_register_redist(gicc->gicr_base_address, redist_base); 2108 return 0; 2109 } 2110 2111 static int __init gic_acpi_collect_gicr_base(void) 2112 { 2113 acpi_tbl_entry_handler redist_parser; 2114 enum acpi_madt_type type; 2115 2116 if (acpi_data.single_redist) { 2117 type = ACPI_MADT_TYPE_GENERIC_INTERRUPT; 2118 redist_parser = gic_acpi_parse_madt_gicc; 2119 } else { 2120 type = ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR; 2121 redist_parser = gic_acpi_parse_madt_redist; 2122 } 2123 2124 /* Collect redistributor base addresses in GICR entries */ 2125 if (acpi_table_parse_madt(type, redist_parser, 0) > 0) 2126 return 0; 2127 2128 pr_info("No valid GICR entries exist\n"); 2129 return -ENODEV; 2130 } 2131 2132 static int __init gic_acpi_match_gicr(union acpi_subtable_headers *header, 2133 const unsigned long end) 2134 { 2135 /* Subtable presence means that redist exists, that's it */ 2136 return 0; 2137 } 2138 2139 static int __init gic_acpi_match_gicc(union acpi_subtable_headers *header, 2140 const unsigned long end) 2141 { 2142 struct acpi_madt_generic_interrupt *gicc = 2143 (struct acpi_madt_generic_interrupt *)header; 2144 2145 /* 2146 * If GICC is enabled and has valid gicr base address, then it means 2147 * GICR base is presented via GICC 2148 */ 2149 if ((gicc->flags & ACPI_MADT_ENABLED) && gicc->gicr_base_address) { 2150 acpi_data.enabled_rdists++; 2151 return 0; 2152 } 2153 2154 /* 2155 * It's perfectly valid firmware can pass disabled GICC entry, driver 2156 * should not treat as errors, skip the entry instead of probe fail. 2157 */ 2158 if (!(gicc->flags & ACPI_MADT_ENABLED)) 2159 return 0; 2160 2161 return -ENODEV; 2162 } 2163 2164 static int __init gic_acpi_count_gicr_regions(void) 2165 { 2166 int count; 2167 2168 /* 2169 * Count how many redistributor regions we have. It is not allowed 2170 * to mix redistributor description, GICR and GICC subtables have to be 2171 * mutually exclusive. 2172 */ 2173 count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR, 2174 gic_acpi_match_gicr, 0); 2175 if (count > 0) { 2176 acpi_data.single_redist = false; 2177 return count; 2178 } 2179 2180 count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT, 2181 gic_acpi_match_gicc, 0); 2182 if (count > 0) { 2183 acpi_data.single_redist = true; 2184 count = acpi_data.enabled_rdists; 2185 } 2186 2187 return count; 2188 } 2189 2190 static bool __init acpi_validate_gic_table(struct acpi_subtable_header *header, 2191 struct acpi_probe_entry *ape) 2192 { 2193 struct acpi_madt_generic_distributor *dist; 2194 int count; 2195 2196 dist = (struct acpi_madt_generic_distributor *)header; 2197 if (dist->version != ape->driver_data) 2198 return false; 2199 2200 /* We need to do that exercise anyway, the sooner the better */ 2201 count = gic_acpi_count_gicr_regions(); 2202 if (count <= 0) 2203 return false; 2204 2205 acpi_data.nr_redist_regions = count; 2206 return true; 2207 } 2208 2209 static int __init gic_acpi_parse_virt_madt_gicc(union acpi_subtable_headers *header, 2210 const unsigned long end) 2211 { 2212 struct acpi_madt_generic_interrupt *gicc = 2213 (struct acpi_madt_generic_interrupt *)header; 2214 int maint_irq_mode; 2215 static int first_madt = true; 2216 2217 /* Skip unusable CPUs */ 2218 if (!(gicc->flags & ACPI_MADT_ENABLED)) 2219 return 0; 2220 2221 maint_irq_mode = (gicc->flags & ACPI_MADT_VGIC_IRQ_MODE) ? 2222 ACPI_EDGE_SENSITIVE : ACPI_LEVEL_SENSITIVE; 2223 2224 if (first_madt) { 2225 first_madt = false; 2226 2227 acpi_data.maint_irq = gicc->vgic_interrupt; 2228 acpi_data.maint_irq_mode = maint_irq_mode; 2229 acpi_data.vcpu_base = gicc->gicv_base_address; 2230 2231 return 0; 2232 } 2233 2234 /* 2235 * The maintenance interrupt and GICV should be the same for every CPU 2236 */ 2237 if ((acpi_data.maint_irq != gicc->vgic_interrupt) || 2238 (acpi_data.maint_irq_mode != maint_irq_mode) || 2239 (acpi_data.vcpu_base != gicc->gicv_base_address)) 2240 return -EINVAL; 2241 2242 return 0; 2243 } 2244 2245 static bool __init gic_acpi_collect_virt_info(void) 2246 { 2247 int count; 2248 2249 count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT, 2250 gic_acpi_parse_virt_madt_gicc, 0); 2251 2252 return (count > 0); 2253 } 2254 2255 #define ACPI_GICV3_DIST_MEM_SIZE (SZ_64K) 2256 #define ACPI_GICV2_VCTRL_MEM_SIZE (SZ_4K) 2257 #define ACPI_GICV2_VCPU_MEM_SIZE (SZ_8K) 2258 2259 static void __init gic_acpi_setup_kvm_info(void) 2260 { 2261 int irq; 2262 2263 if (!gic_acpi_collect_virt_info()) { 2264 pr_warn("Unable to get hardware information used for virtualization\n"); 2265 return; 2266 } 2267 2268 gic_v3_kvm_info.type = GIC_V3; 2269 2270 irq = acpi_register_gsi(NULL, acpi_data.maint_irq, 2271 acpi_data.maint_irq_mode, 2272 ACPI_ACTIVE_HIGH); 2273 if (irq <= 0) 2274 return; 2275 2276 gic_v3_kvm_info.maint_irq = irq; 2277 2278 if (acpi_data.vcpu_base) { 2279 struct resource *vcpu = &gic_v3_kvm_info.vcpu; 2280 2281 vcpu->flags = IORESOURCE_MEM; 2282 vcpu->start = acpi_data.vcpu_base; 2283 vcpu->end = vcpu->start + ACPI_GICV2_VCPU_MEM_SIZE - 1; 2284 } 2285 2286 gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis; 2287 gic_v3_kvm_info.has_v4_1 = gic_data.rdists.has_rvpeid; 2288 vgic_set_kvm_info(&gic_v3_kvm_info); 2289 } 2290 2291 static int __init 2292 gic_acpi_init(union acpi_subtable_headers *header, const unsigned long end) 2293 { 2294 struct acpi_madt_generic_distributor *dist; 2295 struct fwnode_handle *domain_handle; 2296 size_t size; 2297 int i, err; 2298 2299 /* Get distributor base address */ 2300 dist = (struct acpi_madt_generic_distributor *)header; 2301 acpi_data.dist_base = ioremap(dist->base_address, 2302 ACPI_GICV3_DIST_MEM_SIZE); 2303 if (!acpi_data.dist_base) { 2304 pr_err("Unable to map GICD registers\n"); 2305 return -ENOMEM; 2306 } 2307 2308 err = gic_validate_dist_version(acpi_data.dist_base); 2309 if (err) { 2310 pr_err("No distributor detected at @%p, giving up\n", 2311 acpi_data.dist_base); 2312 goto out_dist_unmap; 2313 } 2314 2315 size = sizeof(*acpi_data.redist_regs) * acpi_data.nr_redist_regions; 2316 acpi_data.redist_regs = kzalloc(size, GFP_KERNEL); 2317 if (!acpi_data.redist_regs) { 2318 err = -ENOMEM; 2319 goto out_dist_unmap; 2320 } 2321 2322 err = gic_acpi_collect_gicr_base(); 2323 if (err) 2324 goto out_redist_unmap; 2325 2326 domain_handle = irq_domain_alloc_fwnode(&dist->base_address); 2327 if (!domain_handle) { 2328 err = -ENOMEM; 2329 goto out_redist_unmap; 2330 } 2331 2332 err = gic_init_bases(acpi_data.dist_base, acpi_data.redist_regs, 2333 acpi_data.nr_redist_regions, 0, domain_handle); 2334 if (err) 2335 goto out_fwhandle_free; 2336 2337 acpi_set_irq_model(ACPI_IRQ_MODEL_GIC, domain_handle); 2338 2339 if (static_branch_likely(&supports_deactivate_key)) 2340 gic_acpi_setup_kvm_info(); 2341 2342 return 0; 2343 2344 out_fwhandle_free: 2345 irq_domain_free_fwnode(domain_handle); 2346 out_redist_unmap: 2347 for (i = 0; i < acpi_data.nr_redist_regions; i++) 2348 if (acpi_data.redist_regs[i].redist_base) 2349 iounmap(acpi_data.redist_regs[i].redist_base); 2350 kfree(acpi_data.redist_regs); 2351 out_dist_unmap: 2352 iounmap(acpi_data.dist_base); 2353 return err; 2354 } 2355 IRQCHIP_ACPI_DECLARE(gic_v3, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR, 2356 acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V3, 2357 gic_acpi_init); 2358 IRQCHIP_ACPI_DECLARE(gic_v4, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR, 2359 acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V4, 2360 gic_acpi_init); 2361 IRQCHIP_ACPI_DECLARE(gic_v3_or_v4, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR, 2362 acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_NONE, 2363 gic_acpi_init); 2364 #endif 2365