1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2021 Western Digital Corporation or its affiliates. 4 * Copyright (C) 2022 Ventana Micro Systems Inc. 5 */ 6 7 #define pr_fmt(fmt) "riscv-imsic: " fmt 8 #include <linux/cpu.h> 9 #include <linux/interrupt.h> 10 #include <linux/io.h> 11 #include <linux/irq.h> 12 #include <linux/irqchip.h> 13 #include <linux/irqchip/chained_irq.h> 14 #include <linux/module.h> 15 #include <linux/spinlock.h> 16 #include <linux/smp.h> 17 18 #include "irq-riscv-imsic-state.h" 19 20 static int imsic_parent_irq; 21 22 #ifdef CONFIG_SMP 23 static void imsic_ipi_send(unsigned int cpu) 24 { 25 struct imsic_local_config *local = per_cpu_ptr(imsic->global.local, cpu); 26 27 writel_relaxed(IMSIC_IPI_ID, local->msi_va); 28 } 29 30 static void imsic_ipi_starting_cpu(void) 31 { 32 /* Enable IPIs for current CPU. */ 33 __imsic_id_set_enable(IMSIC_IPI_ID); 34 } 35 36 static void imsic_ipi_dying_cpu(void) 37 { 38 /* Disable IPIs for current CPU. */ 39 __imsic_id_clear_enable(IMSIC_IPI_ID); 40 } 41 42 static int __init imsic_ipi_domain_init(void) 43 { 44 int virq; 45 46 /* Create IMSIC IPI multiplexing */ 47 virq = ipi_mux_create(IMSIC_NR_IPI, imsic_ipi_send); 48 if (virq <= 0) 49 return virq < 0 ? virq : -ENOMEM; 50 51 /* Set vIRQ range */ 52 riscv_ipi_set_virq_range(virq, IMSIC_NR_IPI); 53 54 /* Announce that IMSIC is providing IPIs */ 55 pr_info("%pfwP: providing IPIs using interrupt %d\n", imsic->fwnode, IMSIC_IPI_ID); 56 57 return 0; 58 } 59 #else 60 static void imsic_ipi_starting_cpu(void) { } 61 static void imsic_ipi_dying_cpu(void) { } 62 static int __init imsic_ipi_domain_init(void) { return 0; } 63 #endif 64 65 /* 66 * To handle an interrupt, we read the TOPEI CSR and write zero in one 67 * instruction. If TOPEI CSR is non-zero then we translate TOPEI.ID to 68 * Linux interrupt number and let Linux IRQ subsystem handle it. 69 */ 70 static void imsic_handle_irq(struct irq_desc *desc) 71 { 72 struct irq_chip *chip = irq_desc_get_chip(desc); 73 int err, cpu = smp_processor_id(); 74 struct imsic_vector *vec; 75 unsigned long local_id; 76 77 chained_irq_enter(chip, desc); 78 79 while ((local_id = csr_swap(CSR_TOPEI, 0))) { 80 local_id >>= TOPEI_ID_SHIFT; 81 82 if (local_id == IMSIC_IPI_ID) { 83 if (IS_ENABLED(CONFIG_SMP)) 84 ipi_mux_process(); 85 continue; 86 } 87 88 if (unlikely(!imsic->base_domain)) 89 continue; 90 91 vec = imsic_vector_from_local_id(cpu, local_id); 92 if (!vec) { 93 pr_warn_ratelimited("vector not found for local ID 0x%lx\n", local_id); 94 continue; 95 } 96 97 err = generic_handle_domain_irq(imsic->base_domain, vec->hwirq); 98 if (unlikely(err)) 99 pr_warn_ratelimited("hwirq 0x%x mapping not found\n", vec->hwirq); 100 } 101 102 chained_irq_exit(chip, desc); 103 } 104 105 static int imsic_starting_cpu(unsigned int cpu) 106 { 107 /* Mark per-CPU IMSIC state as online */ 108 imsic_state_online(); 109 110 /* Enable per-CPU parent interrupt */ 111 enable_percpu_irq(imsic_parent_irq, irq_get_trigger_type(imsic_parent_irq)); 112 113 /* Setup IPIs */ 114 imsic_ipi_starting_cpu(); 115 116 /* 117 * Interrupts identities might have been enabled/disabled while 118 * this CPU was not running so sync-up local enable/disable state. 119 */ 120 imsic_local_sync_all(); 121 122 /* Enable local interrupt delivery */ 123 imsic_local_delivery(true); 124 125 return 0; 126 } 127 128 static int imsic_dying_cpu(unsigned int cpu) 129 { 130 /* Cleanup IPIs */ 131 imsic_ipi_dying_cpu(); 132 133 /* Mark per-CPU IMSIC state as offline */ 134 imsic_state_offline(); 135 136 return 0; 137 } 138 139 static int __init imsic_early_probe(struct fwnode_handle *fwnode) 140 { 141 struct irq_domain *domain; 142 int rc; 143 144 /* Find parent domain and register chained handler */ 145 domain = irq_find_matching_fwnode(riscv_get_intc_hwnode(), DOMAIN_BUS_ANY); 146 if (!domain) { 147 pr_err("%pfwP: Failed to find INTC domain\n", fwnode); 148 return -ENOENT; 149 } 150 imsic_parent_irq = irq_create_mapping(domain, RV_IRQ_EXT); 151 if (!imsic_parent_irq) { 152 pr_err("%pfwP: Failed to create INTC mapping\n", fwnode); 153 return -ENOENT; 154 } 155 156 /* Initialize IPI domain */ 157 rc = imsic_ipi_domain_init(); 158 if (rc) { 159 pr_err("%pfwP: Failed to initialize IPI domain\n", fwnode); 160 return rc; 161 } 162 163 /* Setup chained handler to the parent domain interrupt */ 164 irq_set_chained_handler(imsic_parent_irq, imsic_handle_irq); 165 166 /* 167 * Setup cpuhp state (must be done after setting imsic_parent_irq) 168 * 169 * Don't disable per-CPU IMSIC file when CPU goes offline 170 * because this affects IPI and the masking/unmasking of 171 * virtual IPIs is done via generic IPI-Mux 172 */ 173 cpuhp_setup_state(CPUHP_AP_IRQ_RISCV_IMSIC_STARTING, "irqchip/riscv/imsic:starting", 174 imsic_starting_cpu, imsic_dying_cpu); 175 176 return 0; 177 } 178 179 static int __init imsic_early_dt_init(struct device_node *node, struct device_node *parent) 180 { 181 struct fwnode_handle *fwnode = &node->fwnode; 182 int rc; 183 184 /* Setup IMSIC state */ 185 rc = imsic_setup_state(fwnode); 186 if (rc) { 187 pr_err("%pfwP: failed to setup state (error %d)\n", fwnode, rc); 188 return rc; 189 } 190 191 /* Do early setup of IPIs */ 192 rc = imsic_early_probe(fwnode); 193 if (rc) 194 return rc; 195 196 /* Ensure that OF platform device gets probed */ 197 of_node_clear_flag(node, OF_POPULATED); 198 return 0; 199 } 200 201 IRQCHIP_DECLARE(riscv_imsic, "riscv,imsics", imsic_early_dt_init); 202