irq-gic.c (4dccc93f1e7f4e2fd92a48c40a7ac9211a029352) irq-gic.c (a27d21e03eb14a63dae12467a7655be3334ac80c)
1/*
2 * Copyright (C) 2002 ARM Limited, All Rights Reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * Interrupt architecture for the GIC:

--- 55 unchanged lines hidden (view full) ---

64#endif
65
66union gic_base {
67 void __iomem *common_base;
68 void __percpu * __iomem *percpu_base;
69};
70
71struct gic_chip_data {
1/*
2 * Copyright (C) 2002 ARM Limited, All Rights Reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * Interrupt architecture for the GIC:

--- 55 unchanged lines hidden (view full) ---

64#endif
65
66union gic_base {
67 void __iomem *common_base;
68 void __percpu * __iomem *percpu_base;
69};
70
71struct gic_chip_data {
72 struct irq_chip chip;
72 union gic_base dist_base;
73 union gic_base cpu_base;
74#ifdef CONFIG_CPU_PM
75 u32 saved_spi_enable[DIV_ROUND_UP(1020, 32)];
76 u32 saved_spi_active[DIV_ROUND_UP(1020, 32)];
77 u32 saved_spi_conf[DIV_ROUND_UP(1020, 16)];
78 u32 saved_spi_target[DIV_ROUND_UP(1020, 4)];
79 u32 __percpu *saved_ppi_enable;

--- 14 unchanged lines hidden (view full) ---

94 * the logical CPU numbering. Let's use a mapping as returned
95 * by the GIC itself.
96 */
97#define NR_GIC_CPU_IF 8
98static u8 gic_cpu_map[NR_GIC_CPU_IF] __read_mostly;
99
100static struct static_key supports_deactivate = STATIC_KEY_INIT_TRUE;
101
73 union gic_base dist_base;
74 union gic_base cpu_base;
75#ifdef CONFIG_CPU_PM
76 u32 saved_spi_enable[DIV_ROUND_UP(1020, 32)];
77 u32 saved_spi_active[DIV_ROUND_UP(1020, 32)];
78 u32 saved_spi_conf[DIV_ROUND_UP(1020, 16)];
79 u32 saved_spi_target[DIV_ROUND_UP(1020, 4)];
80 u32 __percpu *saved_ppi_enable;

--- 14 unchanged lines hidden (view full) ---

95 * the logical CPU numbering. Let's use a mapping as returned
96 * by the GIC itself.
97 */
98#define NR_GIC_CPU_IF 8
99static u8 gic_cpu_map[NR_GIC_CPU_IF] __read_mostly;
100
101static struct static_key supports_deactivate = STATIC_KEY_INIT_TRUE;
102
102#ifndef MAX_GIC_NR
103#define MAX_GIC_NR 1
104#endif
103static struct gic_chip_data gic_data[CONFIG_ARM_GIC_MAX_NR] __read_mostly;
105
104
106static struct gic_chip_data gic_data[MAX_GIC_NR] __read_mostly;
107
108#ifdef CONFIG_GIC_NON_BANKED
109static void __iomem *gic_get_percpu_base(union gic_base *base)
110{
111 return raw_cpu_read(*base->percpu_base);
112}
113
114static void __iomem *gic_get_common_base(union gic_base *base)
115{

--- 215 unchanged lines hidden (view full) ---

331 u32 irqstat, irqnr;
332 struct gic_chip_data *gic = &gic_data[0];
333 void __iomem *cpu_base = gic_data_cpu_base(gic);
334
335 do {
336 irqstat = readl_relaxed(cpu_base + GIC_CPU_INTACK);
337 irqnr = irqstat & GICC_IAR_INT_ID_MASK;
338
105#ifdef CONFIG_GIC_NON_BANKED
106static void __iomem *gic_get_percpu_base(union gic_base *base)
107{
108 return raw_cpu_read(*base->percpu_base);
109}
110
111static void __iomem *gic_get_common_base(union gic_base *base)
112{

--- 215 unchanged lines hidden (view full) ---

328 u32 irqstat, irqnr;
329 struct gic_chip_data *gic = &gic_data[0];
330 void __iomem *cpu_base = gic_data_cpu_base(gic);
331
332 do {
333 irqstat = readl_relaxed(cpu_base + GIC_CPU_INTACK);
334 irqnr = irqstat & GICC_IAR_INT_ID_MASK;
335
339 if (likely(irqnr > 15 && irqnr < 1021)) {
336 if (likely(irqnr > 15 && irqnr < 1020)) {
340 if (static_key_true(&supports_deactivate))
341 writel_relaxed(irqstat, cpu_base + GIC_CPU_EOI);
342 handle_domain_irq(gic->domain, irqnr, regs);
343 continue;
344 }
345 if (irqnr < 16) {
346 writel_relaxed(irqstat, cpu_base + GIC_CPU_EOI);
347 if (static_key_true(&supports_deactivate))

--- 30 unchanged lines hidden (view full) ---

378 else
379 generic_handle_irq(cascade_irq);
380
381 out:
382 chained_irq_exit(chip, desc);
383}
384
385static struct irq_chip gic_chip = {
337 if (static_key_true(&supports_deactivate))
338 writel_relaxed(irqstat, cpu_base + GIC_CPU_EOI);
339 handle_domain_irq(gic->domain, irqnr, regs);
340 continue;
341 }
342 if (irqnr < 16) {
343 writel_relaxed(irqstat, cpu_base + GIC_CPU_EOI);
344 if (static_key_true(&supports_deactivate))

--- 30 unchanged lines hidden (view full) ---

375 else
376 generic_handle_irq(cascade_irq);
377
378 out:
379 chained_irq_exit(chip, desc);
380}
381
382static struct irq_chip gic_chip = {
386 .name = "GIC",
387 .irq_mask = gic_mask_irq,
388 .irq_unmask = gic_unmask_irq,
389 .irq_eoi = gic_eoi_irq,
390 .irq_set_type = gic_set_type,
391#ifdef CONFIG_SMP
392 .irq_set_affinity = gic_set_affinity,
393#endif
394 .irq_get_irqchip_state = gic_irq_get_irqchip_state,

--- 17 unchanged lines hidden (view full) ---

412 .irq_set_vcpu_affinity = gic_irq_set_vcpu_affinity,
413 .flags = IRQCHIP_SET_TYPE_MASKED |
414 IRQCHIP_SKIP_SET_WAKE |
415 IRQCHIP_MASK_ON_SUSPEND,
416};
417
418void __init gic_cascade_irq(unsigned int gic_nr, unsigned int irq)
419{
383 .irq_mask = gic_mask_irq,
384 .irq_unmask = gic_unmask_irq,
385 .irq_eoi = gic_eoi_irq,
386 .irq_set_type = gic_set_type,
387#ifdef CONFIG_SMP
388 .irq_set_affinity = gic_set_affinity,
389#endif
390 .irq_get_irqchip_state = gic_irq_get_irqchip_state,

--- 17 unchanged lines hidden (view full) ---

408 .irq_set_vcpu_affinity = gic_irq_set_vcpu_affinity,
409 .flags = IRQCHIP_SET_TYPE_MASKED |
410 IRQCHIP_SKIP_SET_WAKE |
411 IRQCHIP_MASK_ON_SUSPEND,
412};
413
414void __init gic_cascade_irq(unsigned int gic_nr, unsigned int irq)
415{
420 if (gic_nr >= MAX_GIC_NR)
421 BUG();
416 BUG_ON(gic_nr >= CONFIG_ARM_GIC_MAX_NR);
422 irq_set_chained_handler_and_data(irq, gic_handle_cascade_irq,
423 &gic_data[gic_nr]);
424}
425
426static u8 gic_get_cpumask(struct gic_chip_data *gic)
427{
428 void __iomem *base = gic_data_dist_base(gic);
429 u32 mask, i;

--- 89 unchanged lines hidden (view full) ---

519 gic_cpu_if_up(gic);
520}
521
522int gic_cpu_if_down(unsigned int gic_nr)
523{
524 void __iomem *cpu_base;
525 u32 val = 0;
526
417 irq_set_chained_handler_and_data(irq, gic_handle_cascade_irq,
418 &gic_data[gic_nr]);
419}
420
421static u8 gic_get_cpumask(struct gic_chip_data *gic)
422{
423 void __iomem *base = gic_data_dist_base(gic);
424 u32 mask, i;

--- 89 unchanged lines hidden (view full) ---

514 gic_cpu_if_up(gic);
515}
516
517int gic_cpu_if_down(unsigned int gic_nr)
518{
519 void __iomem *cpu_base;
520 u32 val = 0;
521
527 if (gic_nr >= MAX_GIC_NR)
522 if (gic_nr >= CONFIG_ARM_GIC_MAX_NR)
528 return -EINVAL;
529
530 cpu_base = gic_data_cpu_base(&gic_data[gic_nr]);
531 val = readl(cpu_base + GIC_CPU_CTRL);
532 val &= ~GICC_ENABLE;
533 writel_relaxed(val, cpu_base + GIC_CPU_CTRL);
534
535 return 0;

--- 7 unchanged lines hidden (view full) ---

543 * platform-specific wakeup source must be enabled.
544 */
545static void gic_dist_save(unsigned int gic_nr)
546{
547 unsigned int gic_irqs;
548 void __iomem *dist_base;
549 int i;
550
523 return -EINVAL;
524
525 cpu_base = gic_data_cpu_base(&gic_data[gic_nr]);
526 val = readl(cpu_base + GIC_CPU_CTRL);
527 val &= ~GICC_ENABLE;
528 writel_relaxed(val, cpu_base + GIC_CPU_CTRL);
529
530 return 0;

--- 7 unchanged lines hidden (view full) ---

538 * platform-specific wakeup source must be enabled.
539 */
540static void gic_dist_save(unsigned int gic_nr)
541{
542 unsigned int gic_irqs;
543 void __iomem *dist_base;
544 int i;
545
551 if (gic_nr >= MAX_GIC_NR)
552 BUG();
546 BUG_ON(gic_nr >= CONFIG_ARM_GIC_MAX_NR);
553
554 gic_irqs = gic_data[gic_nr].gic_irqs;
555 dist_base = gic_data_dist_base(&gic_data[gic_nr]);
556
557 if (!dist_base)
558 return;
559
560 for (i = 0; i < DIV_ROUND_UP(gic_irqs, 16); i++)

--- 21 unchanged lines hidden (view full) ---

582 * the GIC and need to be handled by the platform-specific wakeup source.
583 */
584static void gic_dist_restore(unsigned int gic_nr)
585{
586 unsigned int gic_irqs;
587 unsigned int i;
588 void __iomem *dist_base;
589
547
548 gic_irqs = gic_data[gic_nr].gic_irqs;
549 dist_base = gic_data_dist_base(&gic_data[gic_nr]);
550
551 if (!dist_base)
552 return;
553
554 for (i = 0; i < DIV_ROUND_UP(gic_irqs, 16); i++)

--- 21 unchanged lines hidden (view full) ---

576 * the GIC and need to be handled by the platform-specific wakeup source.
577 */
578static void gic_dist_restore(unsigned int gic_nr)
579{
580 unsigned int gic_irqs;
581 unsigned int i;
582 void __iomem *dist_base;
583
590 if (gic_nr >= MAX_GIC_NR)
591 BUG();
584 BUG_ON(gic_nr >= CONFIG_ARM_GIC_MAX_NR);
592
593 gic_irqs = gic_data[gic_nr].gic_irqs;
594 dist_base = gic_data_dist_base(&gic_data[gic_nr]);
595
596 if (!dist_base)
597 return;
598
599 writel_relaxed(GICD_DISABLE, dist_base + GIC_DIST_CTRL);

--- 29 unchanged lines hidden (view full) ---

629
630static void gic_cpu_save(unsigned int gic_nr)
631{
632 int i;
633 u32 *ptr;
634 void __iomem *dist_base;
635 void __iomem *cpu_base;
636
585
586 gic_irqs = gic_data[gic_nr].gic_irqs;
587 dist_base = gic_data_dist_base(&gic_data[gic_nr]);
588
589 if (!dist_base)
590 return;
591
592 writel_relaxed(GICD_DISABLE, dist_base + GIC_DIST_CTRL);

--- 29 unchanged lines hidden (view full) ---

622
623static void gic_cpu_save(unsigned int gic_nr)
624{
625 int i;
626 u32 *ptr;
627 void __iomem *dist_base;
628 void __iomem *cpu_base;
629
637 if (gic_nr >= MAX_GIC_NR)
638 BUG();
630 BUG_ON(gic_nr >= CONFIG_ARM_GIC_MAX_NR);
639
640 dist_base = gic_data_dist_base(&gic_data[gic_nr]);
641 cpu_base = gic_data_cpu_base(&gic_data[gic_nr]);
642
643 if (!dist_base || !cpu_base)
644 return;
645
646 ptr = raw_cpu_ptr(gic_data[gic_nr].saved_ppi_enable);

--- 12 unchanged lines hidden (view full) ---

659
660static void gic_cpu_restore(unsigned int gic_nr)
661{
662 int i;
663 u32 *ptr;
664 void __iomem *dist_base;
665 void __iomem *cpu_base;
666
631
632 dist_base = gic_data_dist_base(&gic_data[gic_nr]);
633 cpu_base = gic_data_cpu_base(&gic_data[gic_nr]);
634
635 if (!dist_base || !cpu_base)
636 return;
637
638 ptr = raw_cpu_ptr(gic_data[gic_nr].saved_ppi_enable);

--- 12 unchanged lines hidden (view full) ---

651
652static void gic_cpu_restore(unsigned int gic_nr)
653{
654 int i;
655 u32 *ptr;
656 void __iomem *dist_base;
657 void __iomem *cpu_base;
658
667 if (gic_nr >= MAX_GIC_NR)
668 BUG();
659 BUG_ON(gic_nr >= CONFIG_ARM_GIC_MAX_NR);
669
670 dist_base = gic_data_dist_base(&gic_data[gic_nr]);
671 cpu_base = gic_data_cpu_base(&gic_data[gic_nr]);
672
673 if (!dist_base || !cpu_base)
674 return;
675
676 ptr = raw_cpu_ptr(gic_data[gic_nr].saved_ppi_enable);

--- 21 unchanged lines hidden (view full) ---

698 writel_relaxed(GICC_INT_PRI_THRESHOLD, cpu_base + GIC_CPU_PRIMASK);
699 gic_cpu_if_up(&gic_data[gic_nr]);
700}
701
702static int gic_notifier(struct notifier_block *self, unsigned long cmd, void *v)
703{
704 int i;
705
660
661 dist_base = gic_data_dist_base(&gic_data[gic_nr]);
662 cpu_base = gic_data_cpu_base(&gic_data[gic_nr]);
663
664 if (!dist_base || !cpu_base)
665 return;
666
667 ptr = raw_cpu_ptr(gic_data[gic_nr].saved_ppi_enable);

--- 21 unchanged lines hidden (view full) ---

689 writel_relaxed(GICC_INT_PRI_THRESHOLD, cpu_base + GIC_CPU_PRIMASK);
690 gic_cpu_if_up(&gic_data[gic_nr]);
691}
692
693static int gic_notifier(struct notifier_block *self, unsigned long cmd, void *v)
694{
695 int i;
696
706 for (i = 0; i < MAX_GIC_NR; i++) {
697 for (i = 0; i < CONFIG_ARM_GIC_MAX_NR; i++) {
707#ifdef CONFIG_GIC_NON_BANKED
708 /* Skip over unused GICs */
709 if (!gic_data[i].get_base)
710 continue;
711#endif
712 switch (cmd) {
713 case CPU_PM_ENTER:
714 gic_cpu_save(i);

--- 115 unchanged lines hidden (view full) ---

830 */
831void gic_migrate_target(unsigned int new_cpu_id)
832{
833 unsigned int cur_cpu_id, gic_irqs, gic_nr = 0;
834 void __iomem *dist_base;
835 int i, ror_val, cpu = smp_processor_id();
836 u32 val, cur_target_mask, active_mask;
837
698#ifdef CONFIG_GIC_NON_BANKED
699 /* Skip over unused GICs */
700 if (!gic_data[i].get_base)
701 continue;
702#endif
703 switch (cmd) {
704 case CPU_PM_ENTER:
705 gic_cpu_save(i);

--- 115 unchanged lines hidden (view full) ---

821 */
822void gic_migrate_target(unsigned int new_cpu_id)
823{
824 unsigned int cur_cpu_id, gic_irqs, gic_nr = 0;
825 void __iomem *dist_base;
826 int i, ror_val, cpu = smp_processor_id();
827 u32 val, cur_target_mask, active_mask;
828
838 if (gic_nr >= MAX_GIC_NR)
839 BUG();
829 BUG_ON(gic_nr >= CONFIG_ARM_GIC_MAX_NR);
840
841 dist_base = gic_data_dist_base(&gic_data[gic_nr]);
842 if (!dist_base)
843 return;
844 gic_irqs = gic_data[gic_nr].gic_irqs;
845
846 cur_cpu_id = __ffs(gic_cpu_map[cpu]);
847 cur_target_mask = 0x01010101 << cur_cpu_id;

--- 72 unchanged lines hidden (view full) ---

920
921#else
922#define gic_init_physaddr(node) do { } while (0)
923#endif
924
925static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq,
926 irq_hw_number_t hw)
927{
830
831 dist_base = gic_data_dist_base(&gic_data[gic_nr]);
832 if (!dist_base)
833 return;
834 gic_irqs = gic_data[gic_nr].gic_irqs;
835
836 cur_cpu_id = __ffs(gic_cpu_map[cpu]);
837 cur_target_mask = 0x01010101 << cur_cpu_id;

--- 72 unchanged lines hidden (view full) ---

910
911#else
912#define gic_init_physaddr(node) do { } while (0)
913#endif
914
915static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq,
916 irq_hw_number_t hw)
917{
928 struct irq_chip *chip = &gic_chip;
918 struct gic_chip_data *gic = d->host_data;
929
919
930 if (static_key_true(&supports_deactivate)) {
931 if (d->host_data == (void *)&gic_data[0])
932 chip = &gic_eoimode1_chip;
933 }
934
935 if (hw < 32) {
936 irq_set_percpu_devid(irq);
920 if (hw < 32) {
921 irq_set_percpu_devid(irq);
937 irq_domain_set_info(d, irq, hw, chip, d->host_data,
922 irq_domain_set_info(d, irq, hw, &gic->chip, d->host_data,
938 handle_percpu_devid_irq, NULL, NULL);
939 irq_set_status_flags(irq, IRQ_NOAUTOEN);
940 } else {
923 handle_percpu_devid_irq, NULL, NULL);
924 irq_set_status_flags(irq, IRQ_NOAUTOEN);
925 } else {
941 irq_domain_set_info(d, irq, hw, chip, d->host_data,
926 irq_domain_set_info(d, irq, hw, &gic->chip, d->host_data,
942 handle_fasteoi_irq, NULL, NULL);
943 irq_set_probe(irq);
944 }
945 return 0;
946}
947
948static void gic_irq_domain_unmap(struct irq_domain *d, unsigned int irq)
949{

--- 85 unchanged lines hidden (view full) ---

1035static void __init __gic_init_bases(unsigned int gic_nr, int irq_start,
1036 void __iomem *dist_base, void __iomem *cpu_base,
1037 u32 percpu_offset, struct fwnode_handle *handle)
1038{
1039 irq_hw_number_t hwirq_base;
1040 struct gic_chip_data *gic;
1041 int gic_irqs, irq_base, i;
1042
927 handle_fasteoi_irq, NULL, NULL);
928 irq_set_probe(irq);
929 }
930 return 0;
931}
932
933static void gic_irq_domain_unmap(struct irq_domain *d, unsigned int irq)
934{

--- 85 unchanged lines hidden (view full) ---

1020static void __init __gic_init_bases(unsigned int gic_nr, int irq_start,
1021 void __iomem *dist_base, void __iomem *cpu_base,
1022 u32 percpu_offset, struct fwnode_handle *handle)
1023{
1024 irq_hw_number_t hwirq_base;
1025 struct gic_chip_data *gic;
1026 int gic_irqs, irq_base, i;
1027
1043 BUG_ON(gic_nr >= MAX_GIC_NR);
1028 BUG_ON(gic_nr >= CONFIG_ARM_GIC_MAX_NR);
1044
1045 gic_check_cpu_features();
1046
1047 gic = &gic_data[gic_nr];
1029
1030 gic_check_cpu_features();
1031
1032 gic = &gic_data[gic_nr];
1033
1034 /* Initialize irq_chip */
1035 if (static_key_true(&supports_deactivate) && gic_nr == 0) {
1036 gic->chip = gic_eoimode1_chip;
1037 } else {
1038 gic->chip = gic_chip;
1039 gic->chip.name = kasprintf(GFP_KERNEL, "GIC-%d", gic_nr);
1040 }
1041
1048#ifdef CONFIG_GIC_NON_BANKED
1049 if (percpu_offset) { /* Frankein-GIC without banked registers... */
1050 unsigned int cpu;
1051
1052 gic->dist_base.percpu_base = alloc_percpu(void __iomem *);
1053 gic->cpu_base.percpu_base = alloc_percpu(void __iomem *);
1054 if (WARN_ON(!gic->dist_base.percpu_base ||
1055 !gic->cpu_base.percpu_base)) {

--- 135 unchanged lines hidden (view full) ---

1191 cpuif_res.start += 0xf000;
1192 pr_warn("GIC: Adjusting CPU interface base to %pa",
1193 &cpuif_res.start);
1194 }
1195
1196 return true;
1197}
1198
1042#ifdef CONFIG_GIC_NON_BANKED
1043 if (percpu_offset) { /* Frankein-GIC without banked registers... */
1044 unsigned int cpu;
1045
1046 gic->dist_base.percpu_base = alloc_percpu(void __iomem *);
1047 gic->cpu_base.percpu_base = alloc_percpu(void __iomem *);
1048 if (WARN_ON(!gic->dist_base.percpu_base ||
1049 !gic->cpu_base.percpu_base)) {

--- 135 unchanged lines hidden (view full) ---

1185 cpuif_res.start += 0xf000;
1186 pr_warn("GIC: Adjusting CPU interface base to %pa",
1187 &cpuif_res.start);
1188 }
1189
1190 return true;
1191}
1192
1199static int __init
1193int __init
1200gic_of_init(struct device_node *node, struct device_node *parent)
1201{
1202 void __iomem *cpu_base;
1203 void __iomem *dist_base;
1204 u32 percpu_offset;
1205 int irq;
1206
1207 if (WARN_ON(!node))

--- 163 unchanged lines hidden --- 		1194gic_of_init(struct device_node *node, struct device_node *parent)
1195{
1196 void __iomem *cpu_base;
1197 void __iomem *dist_base;
1198 u32 percpu_offset;
1199 int irq;
1200
1201 if (WARN_ON(!node))

--- 163 unchanged lines hidden ---