1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (C) 2008 Ralf Baechle (ralf@linux-mips.org) 7 * Copyright (C) 2012 MIPS Technologies, Inc. All rights reserved. 8 */ 9 10 #define pr_fmt(fmt) "irq-mips-gic: " fmt 11 12 #include <linux/bitfield.h> 13 #include <linux/bitmap.h> 14 #include <linux/clocksource.h> 15 #include <linux/cpuhotplug.h> 16 #include <linux/init.h> 17 #include <linux/interrupt.h> 18 #include <linux/irq.h> 19 #include <linux/irqchip.h> 20 #include <linux/irqdomain.h> 21 #include <linux/of_address.h> 22 #include <linux/percpu.h> 23 #include <linux/sched.h> 24 #include <linux/smp.h> 25 26 #include <asm/mips-cps.h> 27 #include <asm/setup.h> 28 #include <asm/traps.h> 29 30 #include <dt-bindings/interrupt-controller/mips-gic.h> 31 32 #define GIC_MAX_INTRS 256 33 #define GIC_MAX_LONGS BITS_TO_LONGS(GIC_MAX_INTRS) 34 35 /* Add 2 to convert GIC CPU pin to core interrupt */ 36 #define GIC_CPU_PIN_OFFSET 2 37 38 /* Mapped interrupt to pin X, then GIC will generate the vector (X+1). */ 39 #define GIC_PIN_TO_VEC_OFFSET 1 40 41 /* Convert between local/shared IRQ number and GIC HW IRQ number. */ 42 #define GIC_LOCAL_HWIRQ_BASE 0 43 #define GIC_LOCAL_TO_HWIRQ(x) (GIC_LOCAL_HWIRQ_BASE + (x)) 44 #define GIC_HWIRQ_TO_LOCAL(x) ((x) - GIC_LOCAL_HWIRQ_BASE) 45 #define GIC_SHARED_HWIRQ_BASE GIC_NUM_LOCAL_INTRS 46 #define GIC_SHARED_TO_HWIRQ(x) (GIC_SHARED_HWIRQ_BASE + (x)) 47 #define GIC_HWIRQ_TO_SHARED(x) ((x) - GIC_SHARED_HWIRQ_BASE) 48 49 void __iomem *mips_gic_base; 50 51 static DEFINE_PER_CPU_READ_MOSTLY(unsigned long[GIC_MAX_LONGS], pcpu_masks); 52 53 static DEFINE_RAW_SPINLOCK(gic_lock); 54 static struct irq_domain *gic_irq_domain; 55 static int gic_shared_intrs; 56 static unsigned int gic_cpu_pin; 57 static struct irq_chip gic_level_irq_controller, gic_edge_irq_controller; 58 59 #ifdef CONFIG_GENERIC_IRQ_IPI 60 static DECLARE_BITMAP(ipi_resrv, GIC_MAX_INTRS); 61 static DECLARE_BITMAP(ipi_available, GIC_MAX_INTRS); 62 #endif /* CONFIG_GENERIC_IRQ_IPI */ 63 64 static struct gic_all_vpes_chip_data { 65 u32 map; 66 bool mask; 67 } gic_all_vpes_chip_data[GIC_NUM_LOCAL_INTRS]; 68 69 static int __gic_with_next_online_cpu(int prev) 70 { 71 unsigned int cpu; 72 73 /* Release the redirect/other region lock to the previous CPU, if any. */ 74 if (prev >= 0) 75 mips_cm_unlock_other(); 76 77 /* Discover the next online CPU */ 78 cpu = cpumask_next(prev, cpu_online_mask); 79 80 /* If there isn't one, we're done */ 81 if (cpu >= nr_cpu_ids) 82 return cpu; 83 84 /* Lock access to redirect/other region to the next CPU */ 85 mips_cm_lock_other_cpu(cpu, CM_GCR_Cx_OTHER_BLOCK_LOCAL); 86 87 return cpu; 88 } 89 90 /** 91 * for_each_online_cpu_gic() - Iterate over online CPUs, access local registers 92 * @cpu: An integer variable to hold the current CPU number 93 * @gic_lock: A pointer to raw spin lock used as a guard 94 * 95 * Iterate over online CPUs & configure the other/redirect register region to 96 * access each CPUs GIC local register block, which can be accessed from the 97 * loop body using read_gic_vo_*() or write_gic_vo_*() accessor functions or 98 * their derivatives. 99 */ 100 #define for_each_online_cpu_gic(cpu, gic_lock) \ 101 guard(raw_spinlock_irqsave)(gic_lock); \ 102 for ((cpu) = __gic_with_next_online_cpu(-1); \ 103 (cpu) < nr_cpu_ids; \ 104 (cpu) = __gic_with_next_online_cpu(cpu)) 105 106 /** 107 * gic_irq_lock_cluster() - Lock redirect block access to IRQ's cluster 108 * @d: struct irq_data corresponding to the interrupt we're interested in 109 * 110 * Locks redirect register block access to the global register block of the GIC 111 * within the remote cluster that the IRQ corresponding to @d is affine to, 112 * returning true when this redirect block setup & locking has been performed. 113 * 114 * If @d is affine to the local cluster then no locking is performed and this 115 * function will return false, indicating to the caller that it should access 116 * the local clusters registers without the overhead of indirection through the 117 * redirect block. 118 * 119 * In summary, if this function returns true then the caller should access GIC 120 * registers using redirect register block accessors & then call 121 * mips_cm_unlock_other() when done. If this function returns false then the 122 * caller should trivially access GIC registers in the local cluster. 123 * 124 * Returns true if locking performed, else false. 125 */ 126 static bool gic_irq_lock_cluster(struct irq_data *d) 127 { 128 unsigned int cpu, cl; 129 130 cpu = cpumask_first(irq_data_get_effective_affinity_mask(d)); 131 BUG_ON(cpu >= NR_CPUS); 132 133 cl = cpu_cluster(&cpu_data[cpu]); 134 if (cl == cpu_cluster(¤t_cpu_data)) 135 return false; 136 if (mips_cps_numcores(cl) == 0) 137 return false; 138 mips_cm_lock_other(cl, 0, 0, CM_GCR_Cx_OTHER_BLOCK_GLOBAL); 139 return true; 140 } 141 142 static void gic_clear_pcpu_masks(unsigned int intr) 143 { 144 unsigned int i; 145 146 /* Clear the interrupt's bit in all pcpu_masks */ 147 for_each_possible_cpu(i) 148 clear_bit(intr, per_cpu_ptr(pcpu_masks, i)); 149 } 150 151 static bool gic_local_irq_is_routable(int intr) 152 { 153 u32 vpe_ctl; 154 155 /* All local interrupts are routable in EIC mode. */ 156 if (cpu_has_veic) 157 return true; 158 159 vpe_ctl = read_gic_vl_ctl(); 160 switch (intr) { 161 case GIC_LOCAL_INT_TIMER: 162 return vpe_ctl & GIC_VX_CTL_TIMER_ROUTABLE; 163 case GIC_LOCAL_INT_PERFCTR: 164 return vpe_ctl & GIC_VX_CTL_PERFCNT_ROUTABLE; 165 case GIC_LOCAL_INT_FDC: 166 return vpe_ctl & GIC_VX_CTL_FDC_ROUTABLE; 167 case GIC_LOCAL_INT_SWINT0: 168 case GIC_LOCAL_INT_SWINT1: 169 return vpe_ctl & GIC_VX_CTL_SWINT_ROUTABLE; 170 default: 171 return true; 172 } 173 } 174 175 static void gic_bind_eic_interrupt(int irq, int set) 176 { 177 /* Convert irq vector # to hw int # */ 178 irq -= GIC_PIN_TO_VEC_OFFSET; 179 180 /* Set irq to use shadow set */ 181 write_gic_vl_eic_shadow_set(irq, set); 182 } 183 184 static void gic_send_ipi(struct irq_data *d, unsigned int cpu) 185 { 186 irq_hw_number_t hwirq = GIC_HWIRQ_TO_SHARED(irqd_to_hwirq(d)); 187 188 if (gic_irq_lock_cluster(d)) { 189 write_gic_redir_wedge(GIC_WEDGE_RW | hwirq); 190 mips_cm_unlock_other(); 191 } else { 192 write_gic_wedge(GIC_WEDGE_RW | hwirq); 193 } 194 } 195 196 int gic_get_c0_compare_int(void) 197 { 198 if (!gic_local_irq_is_routable(GIC_LOCAL_INT_TIMER)) 199 return MIPS_CPU_IRQ_BASE + cp0_compare_irq; 200 return irq_create_mapping(gic_irq_domain, 201 GIC_LOCAL_TO_HWIRQ(GIC_LOCAL_INT_TIMER)); 202 } 203 204 int gic_get_c0_perfcount_int(void) 205 { 206 if (!gic_local_irq_is_routable(GIC_LOCAL_INT_PERFCTR)) { 207 /* Is the performance counter shared with the timer? */ 208 if (cp0_perfcount_irq < 0) 209 return -1; 210 return MIPS_CPU_IRQ_BASE + cp0_perfcount_irq; 211 } 212 return irq_create_mapping(gic_irq_domain, 213 GIC_LOCAL_TO_HWIRQ(GIC_LOCAL_INT_PERFCTR)); 214 } 215 216 int gic_get_c0_fdc_int(void) 217 { 218 if (!gic_local_irq_is_routable(GIC_LOCAL_INT_FDC)) { 219 /* Is the FDC IRQ even present? */ 220 if (cp0_fdc_irq < 0) 221 return -1; 222 return MIPS_CPU_IRQ_BASE + cp0_fdc_irq; 223 } 224 225 return irq_create_mapping(gic_irq_domain, 226 GIC_LOCAL_TO_HWIRQ(GIC_LOCAL_INT_FDC)); 227 } 228 229 static void gic_handle_shared_int(bool chained) 230 { 231 unsigned int intr; 232 unsigned long *pcpu_mask; 233 DECLARE_BITMAP(pending, GIC_MAX_INTRS); 234 235 /* Get per-cpu bitmaps */ 236 pcpu_mask = this_cpu_ptr(pcpu_masks); 237 238 if (mips_cm_is64) 239 __ioread64_copy(pending, addr_gic_pend(), 240 DIV_ROUND_UP(gic_shared_intrs, 64)); 241 else 242 __ioread32_copy(pending, addr_gic_pend(), 243 DIV_ROUND_UP(gic_shared_intrs, 32)); 244 245 bitmap_and(pending, pending, pcpu_mask, gic_shared_intrs); 246 247 for_each_set_bit(intr, pending, gic_shared_intrs) { 248 if (chained) 249 generic_handle_domain_irq(gic_irq_domain, 250 GIC_SHARED_TO_HWIRQ(intr)); 251 else 252 do_domain_IRQ(gic_irq_domain, 253 GIC_SHARED_TO_HWIRQ(intr)); 254 } 255 } 256 257 static void gic_mask_irq(struct irq_data *d) 258 { 259 unsigned int intr = GIC_HWIRQ_TO_SHARED(d->hwirq); 260 261 if (gic_irq_lock_cluster(d)) { 262 write_gic_redir_rmask(intr); 263 mips_cm_unlock_other(); 264 } else { 265 write_gic_rmask(intr); 266 } 267 268 gic_clear_pcpu_masks(intr); 269 } 270 271 static void gic_unmask_irq(struct irq_data *d) 272 { 273 unsigned int intr = GIC_HWIRQ_TO_SHARED(d->hwirq); 274 unsigned int cpu; 275 276 if (gic_irq_lock_cluster(d)) { 277 write_gic_redir_smask(intr); 278 mips_cm_unlock_other(); 279 } else { 280 write_gic_smask(intr); 281 } 282 283 gic_clear_pcpu_masks(intr); 284 cpu = cpumask_first(irq_data_get_effective_affinity_mask(d)); 285 set_bit(intr, per_cpu_ptr(pcpu_masks, cpu)); 286 } 287 288 static void gic_ack_irq(struct irq_data *d) 289 { 290 unsigned int irq = GIC_HWIRQ_TO_SHARED(d->hwirq); 291 292 if (gic_irq_lock_cluster(d)) { 293 write_gic_redir_wedge(irq); 294 mips_cm_unlock_other(); 295 } else { 296 write_gic_wedge(irq); 297 } 298 } 299 300 static int gic_set_type_locked(struct irq_data *d, unsigned int type) 301 { 302 unsigned int irq, pol, trig, dual; 303 304 lockdep_assert_held(&gic_lock); 305 306 irq = GIC_HWIRQ_TO_SHARED(d->hwirq); 307 308 switch (type & IRQ_TYPE_SENSE_MASK) { 309 case IRQ_TYPE_EDGE_FALLING: 310 pol = GIC_POL_FALLING_EDGE; 311 trig = GIC_TRIG_EDGE; 312 dual = GIC_DUAL_SINGLE; 313 break; 314 case IRQ_TYPE_EDGE_RISING: 315 pol = GIC_POL_RISING_EDGE; 316 trig = GIC_TRIG_EDGE; 317 dual = GIC_DUAL_SINGLE; 318 break; 319 case IRQ_TYPE_EDGE_BOTH: 320 pol = 0; /* Doesn't matter */ 321 trig = GIC_TRIG_EDGE; 322 dual = GIC_DUAL_DUAL; 323 break; 324 case IRQ_TYPE_LEVEL_LOW: 325 pol = GIC_POL_ACTIVE_LOW; 326 trig = GIC_TRIG_LEVEL; 327 dual = GIC_DUAL_SINGLE; 328 break; 329 case IRQ_TYPE_LEVEL_HIGH: 330 default: 331 pol = GIC_POL_ACTIVE_HIGH; 332 trig = GIC_TRIG_LEVEL; 333 dual = GIC_DUAL_SINGLE; 334 break; 335 } 336 337 if (gic_irq_lock_cluster(d)) { 338 change_gic_redir_pol(irq, pol); 339 change_gic_redir_trig(irq, trig); 340 change_gic_redir_dual(irq, dual); 341 mips_cm_unlock_other(); 342 } else { 343 change_gic_pol(irq, pol); 344 change_gic_trig(irq, trig); 345 change_gic_dual(irq, dual); 346 } 347 348 if (trig == GIC_TRIG_EDGE) 349 irq_set_chip_handler_name_locked(d, &gic_edge_irq_controller, 350 handle_edge_irq, NULL); 351 else 352 irq_set_chip_handler_name_locked(d, &gic_level_irq_controller, 353 handle_level_irq, NULL); 354 355 return 0; 356 } 357 358 static int gic_set_type(struct irq_data *d, unsigned int type) 359 { 360 guard(raw_spinlock_irqsave)(&gic_lock); 361 return gic_set_type_locked(d, type); 362 } 363 364 #ifdef CONFIG_SMP 365 static int gic_set_affinity(struct irq_data *d, const struct cpumask *cpumask, 366 bool force) 367 { 368 unsigned int irq = GIC_HWIRQ_TO_SHARED(d->hwirq); 369 unsigned int cpu, cl, old_cpu, old_cl; 370 unsigned long flags; 371 bool enabled; 372 373 /* 374 * The GIC specifies that we can only route an interrupt to one VP(E), 375 * ie. CPU in Linux parlance, at a time. Therefore we always route to 376 * the first forced or online CPU in the mask. 377 */ 378 if (force) 379 cpu = cpumask_first(cpumask); 380 else 381 cpu = cpumask_first_and(cpumask, cpu_online_mask); 382 383 if (cpu >= NR_CPUS) 384 return -EINVAL; 385 386 old_cpu = cpumask_first(irq_data_get_effective_affinity_mask(d)); 387 old_cl = cpu_cluster(&cpu_data[old_cpu]); 388 cl = cpu_cluster(&cpu_data[cpu]); 389 390 raw_spin_lock_irqsave(&gic_lock, flags); 391 392 /* 393 * If we're moving affinity between clusters, save the interrupt's 394 * mask state, stop routing it to any VP(E) in the old cluster and 395 * disable it there so it is no longer delivered. 396 */ 397 if (cl != old_cl) { 398 if (gic_irq_lock_cluster(d)) { 399 enabled = read_gic_redir_mask(irq); 400 write_gic_redir_map_vp(irq, 0); 401 write_gic_redir_rmask(irq); 402 mips_cm_unlock_other(); 403 } else { 404 enabled = read_gic_mask(irq); 405 write_gic_map_vp(irq, 0); 406 write_gic_rmask(irq); 407 } 408 } 409 410 /* 411 * Update effective affinity - after this gic_irq_lock_cluster() will 412 * begin operating on the new cluster. 413 */ 414 irq_data_update_effective_affinity(d, cpumask_of(cpu)); 415 416 /* 417 * If we're moving affinity between clusters, configure the interrupt 418 * trigger type and, if it was enabled in the old cluster, enable it 419 * in the new one. 420 */ 421 if (cl != old_cl) { 422 gic_set_type_locked(d, irqd_get_trigger_type(d)); 423 if (enabled) { 424 if (gic_irq_lock_cluster(d)) { 425 write_gic_redir_smask(irq); 426 mips_cm_unlock_other(); 427 } else { 428 write_gic_smask(irq); 429 } 430 } 431 } 432 433 /* Route the interrupt to its new VP(E) */ 434 if (gic_irq_lock_cluster(d)) { 435 write_gic_redir_map_pin(irq, 436 GIC_MAP_PIN_MAP_TO_PIN | gic_cpu_pin); 437 write_gic_redir_map_vp(irq, BIT(mips_cm_vp_id(cpu))); 438 439 /* Update the pcpu_masks */ 440 gic_clear_pcpu_masks(irq); 441 if (read_gic_redir_mask(irq)) 442 set_bit(irq, per_cpu_ptr(pcpu_masks, cpu)); 443 444 mips_cm_unlock_other(); 445 } else { 446 write_gic_map_pin(irq, GIC_MAP_PIN_MAP_TO_PIN | gic_cpu_pin); 447 write_gic_map_vp(irq, BIT(mips_cm_vp_id(cpu))); 448 449 /* Update the pcpu_masks */ 450 gic_clear_pcpu_masks(irq); 451 if (read_gic_mask(irq)) 452 set_bit(irq, per_cpu_ptr(pcpu_masks, cpu)); 453 } 454 455 raw_spin_unlock_irqrestore(&gic_lock, flags); 456 457 return IRQ_SET_MASK_OK; 458 } 459 #endif 460 461 static struct irq_chip gic_level_irq_controller = { 462 .name = "MIPS GIC", 463 .irq_mask = gic_mask_irq, 464 .irq_unmask = gic_unmask_irq, 465 .irq_set_type = gic_set_type, 466 #ifdef CONFIG_SMP 467 .irq_set_affinity = gic_set_affinity, 468 #endif 469 }; 470 471 static struct irq_chip gic_edge_irq_controller = { 472 .name = "MIPS GIC", 473 .irq_ack = gic_ack_irq, 474 .irq_mask = gic_mask_irq, 475 .irq_unmask = gic_unmask_irq, 476 .irq_set_type = gic_set_type, 477 #ifdef CONFIG_SMP 478 .irq_set_affinity = gic_set_affinity, 479 #endif 480 .ipi_send_single = gic_send_ipi, 481 }; 482 483 static void gic_handle_local_int(bool chained) 484 { 485 unsigned long pending, masked; 486 unsigned int intr; 487 488 pending = read_gic_vl_pend(); 489 masked = read_gic_vl_mask(); 490 491 bitmap_and(&pending, &pending, &masked, GIC_NUM_LOCAL_INTRS); 492 493 for_each_set_bit(intr, &pending, GIC_NUM_LOCAL_INTRS) { 494 if (chained) 495 generic_handle_domain_irq(gic_irq_domain, 496 GIC_LOCAL_TO_HWIRQ(intr)); 497 else 498 do_domain_IRQ(gic_irq_domain, 499 GIC_LOCAL_TO_HWIRQ(intr)); 500 } 501 } 502 503 static void gic_mask_local_irq(struct irq_data *d) 504 { 505 int intr = GIC_HWIRQ_TO_LOCAL(d->hwirq); 506 507 write_gic_vl_rmask(BIT(intr)); 508 } 509 510 static void gic_unmask_local_irq(struct irq_data *d) 511 { 512 int intr = GIC_HWIRQ_TO_LOCAL(d->hwirq); 513 514 write_gic_vl_smask(BIT(intr)); 515 } 516 517 static struct irq_chip gic_local_irq_controller = { 518 .name = "MIPS GIC Local", 519 .irq_mask = gic_mask_local_irq, 520 .irq_unmask = gic_unmask_local_irq, 521 }; 522 523 static void gic_mask_local_irq_all_vpes(struct irq_data *d) 524 { 525 struct gic_all_vpes_chip_data *cd; 526 int intr, cpu; 527 528 if (!mips_cps_multicluster_cpus()) 529 return; 530 531 intr = GIC_HWIRQ_TO_LOCAL(d->hwirq); 532 cd = irq_data_get_irq_chip_data(d); 533 cd->mask = false; 534 535 for_each_online_cpu_gic(cpu, &gic_lock) 536 write_gic_vo_rmask(BIT(intr)); 537 } 538 539 static void gic_unmask_local_irq_all_vpes(struct irq_data *d) 540 { 541 struct gic_all_vpes_chip_data *cd; 542 int intr, cpu; 543 544 if (!mips_cps_multicluster_cpus()) 545 return; 546 547 intr = GIC_HWIRQ_TO_LOCAL(d->hwirq); 548 cd = irq_data_get_irq_chip_data(d); 549 cd->mask = true; 550 551 for_each_online_cpu_gic(cpu, &gic_lock) 552 write_gic_vo_smask(BIT(intr)); 553 } 554 555 static void gic_all_vpes_irq_cpu_online(void) 556 { 557 static const unsigned int local_intrs[] = { 558 GIC_LOCAL_INT_TIMER, 559 GIC_LOCAL_INT_PERFCTR, 560 GIC_LOCAL_INT_FDC, 561 }; 562 unsigned long flags; 563 int i; 564 565 raw_spin_lock_irqsave(&gic_lock, flags); 566 567 for (i = 0; i < ARRAY_SIZE(local_intrs); i++) { 568 unsigned int intr = local_intrs[i]; 569 struct gic_all_vpes_chip_data *cd; 570 571 if (!gic_local_irq_is_routable(intr)) 572 continue; 573 cd = &gic_all_vpes_chip_data[intr]; 574 write_gic_vl_map(mips_gic_vx_map_reg(intr), cd->map); 575 if (cd->mask) 576 write_gic_vl_smask(BIT(intr)); 577 } 578 579 raw_spin_unlock_irqrestore(&gic_lock, flags); 580 } 581 582 static struct irq_chip gic_all_vpes_local_irq_controller = { 583 .name = "MIPS GIC Local", 584 .irq_mask = gic_mask_local_irq_all_vpes, 585 .irq_unmask = gic_unmask_local_irq_all_vpes, 586 }; 587 588 static void __gic_irq_dispatch(void) 589 { 590 gic_handle_local_int(false); 591 gic_handle_shared_int(false); 592 } 593 594 static void gic_irq_dispatch(struct irq_desc *desc) 595 { 596 gic_handle_local_int(true); 597 gic_handle_shared_int(true); 598 } 599 600 static int gic_shared_irq_domain_map(struct irq_domain *d, unsigned int virq, 601 irq_hw_number_t hw, unsigned int cpu) 602 { 603 int intr = GIC_HWIRQ_TO_SHARED(hw); 604 struct irq_data *data; 605 unsigned long flags; 606 607 data = irq_get_irq_data(virq); 608 irq_data_update_effective_affinity(data, cpumask_of(cpu)); 609 610 raw_spin_lock_irqsave(&gic_lock, flags); 611 612 /* Route the interrupt to its VP(E) */ 613 if (gic_irq_lock_cluster(data)) { 614 write_gic_redir_map_pin(intr, 615 GIC_MAP_PIN_MAP_TO_PIN | gic_cpu_pin); 616 write_gic_redir_map_vp(intr, BIT(mips_cm_vp_id(cpu))); 617 mips_cm_unlock_other(); 618 } else { 619 write_gic_map_pin(intr, GIC_MAP_PIN_MAP_TO_PIN | gic_cpu_pin); 620 write_gic_map_vp(intr, BIT(mips_cm_vp_id(cpu))); 621 } 622 623 raw_spin_unlock_irqrestore(&gic_lock, flags); 624 625 return 0; 626 } 627 628 static int gic_irq_domain_xlate(struct irq_domain *d, struct device_node *ctrlr, 629 const u32 *intspec, unsigned int intsize, 630 irq_hw_number_t *out_hwirq, 631 unsigned int *out_type) 632 { 633 if (intsize != 3) 634 return -EINVAL; 635 636 if (intspec[0] == GIC_SHARED) 637 *out_hwirq = GIC_SHARED_TO_HWIRQ(intspec[1]); 638 else if (intspec[0] == GIC_LOCAL) 639 *out_hwirq = GIC_LOCAL_TO_HWIRQ(intspec[1]); 640 else 641 return -EINVAL; 642 *out_type = intspec[2] & IRQ_TYPE_SENSE_MASK; 643 644 return 0; 645 } 646 647 static int gic_irq_domain_map(struct irq_domain *d, unsigned int virq, 648 irq_hw_number_t hwirq) 649 { 650 struct gic_all_vpes_chip_data *cd; 651 unsigned int intr; 652 int err, cpu; 653 u32 map; 654 655 if (hwirq >= GIC_SHARED_HWIRQ_BASE) { 656 #ifdef CONFIG_GENERIC_IRQ_IPI 657 /* verify that shared irqs don't conflict with an IPI irq */ 658 if (test_bit(GIC_HWIRQ_TO_SHARED(hwirq), ipi_resrv)) 659 return -EBUSY; 660 #endif /* CONFIG_GENERIC_IRQ_IPI */ 661 662 err = irq_domain_set_hwirq_and_chip(d, virq, hwirq, 663 &gic_level_irq_controller, 664 NULL); 665 if (err) 666 return err; 667 668 irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(virq))); 669 return gic_shared_irq_domain_map(d, virq, hwirq, 0); 670 } 671 672 intr = GIC_HWIRQ_TO_LOCAL(hwirq); 673 map = GIC_MAP_PIN_MAP_TO_PIN | gic_cpu_pin; 674 675 /* 676 * If adding support for more per-cpu interrupts, keep the 677 * array in gic_all_vpes_irq_cpu_online() in sync. 678 */ 679 switch (intr) { 680 case GIC_LOCAL_INT_TIMER: 681 case GIC_LOCAL_INT_PERFCTR: 682 case GIC_LOCAL_INT_FDC: 683 /* 684 * HACK: These are all really percpu interrupts, but 685 * the rest of the MIPS kernel code does not use the 686 * percpu IRQ API for them. 687 */ 688 cd = &gic_all_vpes_chip_data[intr]; 689 cd->map = map; 690 err = irq_domain_set_hwirq_and_chip(d, virq, hwirq, 691 &gic_all_vpes_local_irq_controller, 692 cd); 693 if (err) 694 return err; 695 696 irq_set_handler(virq, handle_percpu_irq); 697 break; 698 699 default: 700 err = irq_domain_set_hwirq_and_chip(d, virq, hwirq, 701 &gic_local_irq_controller, 702 NULL); 703 if (err) 704 return err; 705 706 irq_set_handler(virq, handle_percpu_devid_irq); 707 irq_set_percpu_devid(virq); 708 break; 709 } 710 711 if (!gic_local_irq_is_routable(intr)) 712 return -EPERM; 713 714 if (mips_cps_multicluster_cpus()) { 715 for_each_online_cpu_gic(cpu, &gic_lock) 716 write_gic_vo_map(mips_gic_vx_map_reg(intr), map); 717 } 718 719 return 0; 720 } 721 722 static int gic_irq_domain_alloc(struct irq_domain *d, unsigned int virq, 723 unsigned int nr_irqs, void *arg) 724 { 725 struct irq_fwspec *fwspec = arg; 726 irq_hw_number_t hwirq; 727 728 if (fwspec->param[0] == GIC_SHARED) 729 hwirq = GIC_SHARED_TO_HWIRQ(fwspec->param[1]); 730 else 731 hwirq = GIC_LOCAL_TO_HWIRQ(fwspec->param[1]); 732 733 return gic_irq_domain_map(d, virq, hwirq); 734 } 735 736 static void gic_irq_domain_free(struct irq_domain *d, unsigned int virq, 737 unsigned int nr_irqs) 738 { 739 } 740 741 static const struct irq_domain_ops gic_irq_domain_ops = { 742 .xlate = gic_irq_domain_xlate, 743 .alloc = gic_irq_domain_alloc, 744 .free = gic_irq_domain_free, 745 .map = gic_irq_domain_map, 746 }; 747 748 #ifdef CONFIG_GENERIC_IRQ_IPI 749 750 static int gic_ipi_domain_xlate(struct irq_domain *d, struct device_node *ctrlr, 751 const u32 *intspec, unsigned int intsize, 752 irq_hw_number_t *out_hwirq, 753 unsigned int *out_type) 754 { 755 /* 756 * There's nothing to translate here. hwirq is dynamically allocated and 757 * the irq type is always edge triggered. 758 * */ 759 *out_hwirq = 0; 760 *out_type = IRQ_TYPE_EDGE_RISING; 761 762 return 0; 763 } 764 765 static int gic_ipi_domain_alloc(struct irq_domain *d, unsigned int virq, 766 unsigned int nr_irqs, void *arg) 767 { 768 struct cpumask *ipimask = arg; 769 irq_hw_number_t hwirq, base_hwirq; 770 int cpu, ret, i; 771 772 base_hwirq = find_first_bit(ipi_available, gic_shared_intrs); 773 if (base_hwirq == gic_shared_intrs) 774 return -ENOMEM; 775 776 /* check that we have enough space */ 777 for (i = base_hwirq; i < nr_irqs; i++) { 778 if (!test_bit(i, ipi_available)) 779 return -EBUSY; 780 } 781 bitmap_clear(ipi_available, base_hwirq, nr_irqs); 782 783 /* map the hwirq for each cpu consecutively */ 784 i = 0; 785 for_each_cpu(cpu, ipimask) { 786 hwirq = GIC_SHARED_TO_HWIRQ(base_hwirq + i); 787 788 ret = irq_domain_set_hwirq_and_chip(d, virq + i, hwirq, 789 &gic_edge_irq_controller, 790 NULL); 791 if (ret) 792 goto error; 793 794 ret = irq_domain_set_hwirq_and_chip(d->parent, virq + i, hwirq, 795 &gic_edge_irq_controller, 796 NULL); 797 if (ret) 798 goto error; 799 800 /* Set affinity to cpu. */ 801 irq_data_update_effective_affinity(irq_get_irq_data(virq + i), 802 cpumask_of(cpu)); 803 ret = irq_set_irq_type(virq + i, IRQ_TYPE_EDGE_RISING); 804 if (ret) 805 goto error; 806 807 ret = gic_shared_irq_domain_map(d, virq + i, hwirq, cpu); 808 if (ret) 809 goto error; 810 811 i++; 812 } 813 814 return 0; 815 error: 816 bitmap_set(ipi_available, base_hwirq, nr_irqs); 817 return ret; 818 } 819 820 static void gic_ipi_domain_free(struct irq_domain *d, unsigned int virq, 821 unsigned int nr_irqs) 822 { 823 irq_hw_number_t base_hwirq; 824 struct irq_data *data; 825 826 data = irq_get_irq_data(virq); 827 if (!data) 828 return; 829 830 base_hwirq = GIC_HWIRQ_TO_SHARED(irqd_to_hwirq(data)); 831 bitmap_set(ipi_available, base_hwirq, nr_irqs); 832 } 833 834 static int gic_ipi_domain_match(struct irq_domain *d, struct device_node *node, 835 enum irq_domain_bus_token bus_token) 836 { 837 bool is_ipi; 838 839 switch (bus_token) { 840 case DOMAIN_BUS_IPI: 841 is_ipi = d->bus_token == bus_token; 842 return (!node || to_of_node(d->fwnode) == node) && is_ipi; 843 break; 844 default: 845 return 0; 846 } 847 } 848 849 static const struct irq_domain_ops gic_ipi_domain_ops = { 850 .xlate = gic_ipi_domain_xlate, 851 .alloc = gic_ipi_domain_alloc, 852 .free = gic_ipi_domain_free, 853 .match = gic_ipi_domain_match, 854 }; 855 856 static int gic_register_ipi_domain(struct device_node *node) 857 { 858 struct irq_domain *gic_ipi_domain; 859 unsigned int v[2], num_ipis; 860 861 gic_ipi_domain = irq_domain_create_hierarchy(gic_irq_domain, IRQ_DOMAIN_FLAG_IPI_PER_CPU, 862 GIC_NUM_LOCAL_INTRS + gic_shared_intrs, 863 of_fwnode_handle(node), &gic_ipi_domain_ops, 864 NULL); 865 if (!gic_ipi_domain) { 866 pr_err("Failed to add IPI domain"); 867 return -ENXIO; 868 } 869 870 irq_domain_update_bus_token(gic_ipi_domain, DOMAIN_BUS_IPI); 871 872 if (node && 873 !of_property_read_u32_array(node, "mti,reserved-ipi-vectors", v, 2)) { 874 bitmap_set(ipi_resrv, v[0], v[1]); 875 } else { 876 /* 877 * Reserve 2 interrupts per possible CPU/VP for use as IPIs, 878 * meeting the requirements of arch/mips SMP. 879 */ 880 num_ipis = 2 * num_possible_cpus(); 881 bitmap_set(ipi_resrv, gic_shared_intrs - num_ipis, num_ipis); 882 } 883 884 bitmap_copy(ipi_available, ipi_resrv, GIC_MAX_INTRS); 885 886 return 0; 887 } 888 889 #else /* !CONFIG_GENERIC_IRQ_IPI */ 890 891 static inline int gic_register_ipi_domain(struct device_node *node) 892 { 893 return 0; 894 } 895 896 #endif /* !CONFIG_GENERIC_IRQ_IPI */ 897 898 static int gic_cpu_startup(unsigned int cpu) 899 { 900 /* Enable or disable EIC */ 901 change_gic_vl_ctl(GIC_VX_CTL_EIC, 902 cpu_has_veic ? GIC_VX_CTL_EIC : 0); 903 904 /* Clear all local IRQ masks (ie. disable all local interrupts) */ 905 write_gic_vl_rmask(~0); 906 907 /* Enable desired interrupts */ 908 gic_all_vpes_irq_cpu_online(); 909 910 return 0; 911 } 912 913 static int __init gic_of_init(struct device_node *node, 914 struct device_node *parent) 915 { 916 unsigned int cpu_vec, i, gicconfig, cl, nclusters; 917 unsigned long reserved; 918 phys_addr_t gic_base; 919 struct resource res; 920 size_t gic_len; 921 int ret; 922 923 /* Find the first available CPU vector. */ 924 i = 0; 925 reserved = (C_SW0 | C_SW1) >> __ffs(C_SW0); 926 while (!of_property_read_u32_index(node, "mti,reserved-cpu-vectors", 927 i++, &cpu_vec)) 928 reserved |= BIT(cpu_vec); 929 930 cpu_vec = find_first_zero_bit(&reserved, hweight_long(ST0_IM)); 931 if (cpu_vec == hweight_long(ST0_IM)) { 932 pr_err("No CPU vectors available\n"); 933 return -ENODEV; 934 } 935 936 if (of_address_to_resource(node, 0, &res)) { 937 /* 938 * Probe the CM for the GIC base address if not specified 939 * in the device-tree. 940 */ 941 if (mips_cm_present()) { 942 gic_base = read_gcr_gic_base() & 943 ~CM_GCR_GIC_BASE_GICEN; 944 gic_len = 0x20000; 945 pr_warn("Using inherited base address %pa\n", 946 &gic_base); 947 } else { 948 pr_err("Failed to get memory range\n"); 949 return -ENODEV; 950 } 951 } else { 952 gic_base = res.start; 953 gic_len = resource_size(&res); 954 } 955 956 if (mips_cm_present()) { 957 write_gcr_gic_base(gic_base | CM_GCR_GIC_BASE_GICEN); 958 /* Ensure GIC region is enabled before trying to access it */ 959 __sync(); 960 } 961 962 mips_gic_base = ioremap(gic_base, gic_len); 963 if (!mips_gic_base) { 964 pr_err("Failed to ioremap gic_base\n"); 965 return -ENOMEM; 966 } 967 968 gicconfig = read_gic_config(); 969 gic_shared_intrs = FIELD_GET(GIC_CONFIG_NUMINTERRUPTS, gicconfig); 970 gic_shared_intrs = (gic_shared_intrs + 1) * 8; 971 972 if (cpu_has_veic) { 973 /* Always use vector 1 in EIC mode */ 974 gic_cpu_pin = 0; 975 set_vi_handler(gic_cpu_pin + GIC_PIN_TO_VEC_OFFSET, 976 __gic_irq_dispatch); 977 } else { 978 gic_cpu_pin = cpu_vec - GIC_CPU_PIN_OFFSET; 979 irq_set_chained_handler(MIPS_CPU_IRQ_BASE + cpu_vec, 980 gic_irq_dispatch); 981 } 982 983 gic_irq_domain = irq_domain_create_simple(of_fwnode_handle(node), 984 GIC_NUM_LOCAL_INTRS + 985 gic_shared_intrs, 0, 986 &gic_irq_domain_ops, NULL); 987 if (!gic_irq_domain) { 988 pr_err("Failed to add IRQ domain"); 989 return -ENXIO; 990 } 991 992 ret = gic_register_ipi_domain(node); 993 if (ret) 994 return ret; 995 996 board_bind_eic_interrupt = &gic_bind_eic_interrupt; 997 998 /* 999 * Initialise each cluster's GIC shared registers to sane default 1000 * values. 1001 * Otherwise, the IPI set up will be erased if we move code 1002 * to gic_cpu_startup for each cpu. 1003 */ 1004 nclusters = mips_cps_numclusters(); 1005 for (cl = 0; cl < nclusters; cl++) { 1006 if (cl == cpu_cluster(¤t_cpu_data)) { 1007 for (i = 0; i < gic_shared_intrs; i++) { 1008 change_gic_pol(i, GIC_POL_ACTIVE_HIGH); 1009 change_gic_trig(i, GIC_TRIG_LEVEL); 1010 write_gic_rmask(i); 1011 } 1012 } else if (mips_cps_numcores(cl) != 0) { 1013 mips_cm_lock_other(cl, 0, 0, CM_GCR_Cx_OTHER_BLOCK_GLOBAL); 1014 for (i = 0; i < gic_shared_intrs; i++) { 1015 change_gic_redir_pol(i, GIC_POL_ACTIVE_HIGH); 1016 change_gic_redir_trig(i, GIC_TRIG_LEVEL); 1017 write_gic_redir_rmask(i); 1018 } 1019 mips_cm_unlock_other(); 1020 1021 } else { 1022 pr_warn("No CPU cores on the cluster %d skip it\n", cl); 1023 } 1024 } 1025 1026 return cpuhp_setup_state(CPUHP_AP_IRQ_MIPS_GIC_STARTING, 1027 "irqchip/mips/gic:starting", 1028 gic_cpu_startup, NULL); 1029 } 1030 IRQCHIP_DECLARE(mips_gic, "mti,gic", gic_of_init); 1031