1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2021 Western Digital Corporation or its affiliates. 4 * Copyright (C) 2022 Ventana Micro Systems Inc. 5 */ 6 7 #define pr_fmt(fmt) "riscv-imsic: " fmt 8 #include <linux/acpi.h> 9 #include <linux/cpu.h> 10 #include <linux/interrupt.h> 11 #include <linux/io.h> 12 #include <linux/irq.h> 13 #include <linux/irqchip.h> 14 #include <linux/irqchip/chained_irq.h> 15 #include <linux/irqchip/riscv-imsic.h> 16 #include <linux/module.h> 17 #include <linux/pci.h> 18 #include <linux/spinlock.h> 19 #include <linux/smp.h> 20 21 #include "irq-riscv-imsic-state.h" 22 23 static int imsic_parent_irq; 24 25 #ifdef CONFIG_SMP 26 static void imsic_ipi_send(unsigned int cpu) 27 { 28 struct imsic_local_config *local = per_cpu_ptr(imsic->global.local, cpu); 29 30 writel_relaxed(IMSIC_IPI_ID, local->msi_va); 31 } 32 33 static void imsic_ipi_starting_cpu(void) 34 { 35 /* Enable IPIs for current CPU. */ 36 __imsic_id_set_enable(IMSIC_IPI_ID); 37 } 38 39 static void imsic_ipi_dying_cpu(void) 40 { 41 /* Disable IPIs for current CPU. */ 42 __imsic_id_clear_enable(IMSIC_IPI_ID); 43 } 44 45 static int __init imsic_ipi_domain_init(void) 46 { 47 int virq; 48 49 /* Create IMSIC IPI multiplexing */ 50 virq = ipi_mux_create(IMSIC_NR_IPI, imsic_ipi_send); 51 if (virq <= 0) 52 return virq < 0 ? virq : -ENOMEM; 53 54 /* Set vIRQ range */ 55 riscv_ipi_set_virq_range(virq, IMSIC_NR_IPI); 56 57 /* Announce that IMSIC is providing IPIs */ 58 pr_info("%pfwP: providing IPIs using interrupt %d\n", imsic->fwnode, IMSIC_IPI_ID); 59 60 return 0; 61 } 62 #else 63 static void imsic_ipi_starting_cpu(void) { } 64 static void imsic_ipi_dying_cpu(void) { } 65 static int __init imsic_ipi_domain_init(void) { return 0; } 66 #endif 67 68 /* 69 * To handle an interrupt, we read the TOPEI CSR and write zero in one 70 * instruction. If TOPEI CSR is non-zero then we translate TOPEI.ID to 71 * Linux interrupt number and let Linux IRQ subsystem handle it. 72 */ 73 static void imsic_handle_irq(struct irq_desc *desc) 74 { 75 struct irq_chip *chip = irq_desc_get_chip(desc); 76 int err, cpu = smp_processor_id(); 77 struct imsic_vector *vec; 78 unsigned long local_id; 79 80 chained_irq_enter(chip, desc); 81 82 while ((local_id = csr_swap(CSR_TOPEI, 0))) { 83 local_id >>= TOPEI_ID_SHIFT; 84 85 if (local_id == IMSIC_IPI_ID) { 86 if (IS_ENABLED(CONFIG_SMP)) 87 ipi_mux_process(); 88 continue; 89 } 90 91 if (unlikely(!imsic->base_domain)) 92 continue; 93 94 vec = imsic_vector_from_local_id(cpu, local_id); 95 if (!vec) { 96 pr_warn_ratelimited("vector not found for local ID 0x%lx\n", local_id); 97 continue; 98 } 99 100 err = generic_handle_domain_irq(imsic->base_domain, vec->hwirq); 101 if (unlikely(err)) 102 pr_warn_ratelimited("hwirq 0x%x mapping not found\n", vec->hwirq); 103 } 104 105 chained_irq_exit(chip, desc); 106 } 107 108 static int imsic_starting_cpu(unsigned int cpu) 109 { 110 /* Mark per-CPU IMSIC state as online */ 111 imsic_state_online(); 112 113 /* Enable per-CPU parent interrupt */ 114 enable_percpu_irq(imsic_parent_irq, irq_get_trigger_type(imsic_parent_irq)); 115 116 /* Setup IPIs */ 117 imsic_ipi_starting_cpu(); 118 119 /* 120 * Interrupts identities might have been enabled/disabled while 121 * this CPU was not running so sync-up local enable/disable state. 122 */ 123 imsic_local_sync_all(); 124 125 /* Enable local interrupt delivery */ 126 imsic_local_delivery(true); 127 128 return 0; 129 } 130 131 static int imsic_dying_cpu(unsigned int cpu) 132 { 133 /* Cleanup IPIs */ 134 imsic_ipi_dying_cpu(); 135 136 /* Mark per-CPU IMSIC state as offline */ 137 imsic_state_offline(); 138 139 return 0; 140 } 141 142 static int __init imsic_early_probe(struct fwnode_handle *fwnode) 143 { 144 struct irq_domain *domain; 145 int rc; 146 147 /* Find parent domain and register chained handler */ 148 domain = irq_find_matching_fwnode(riscv_get_intc_hwnode(), DOMAIN_BUS_ANY); 149 if (!domain) { 150 pr_err("%pfwP: Failed to find INTC domain\n", fwnode); 151 return -ENOENT; 152 } 153 imsic_parent_irq = irq_create_mapping(domain, RV_IRQ_EXT); 154 if (!imsic_parent_irq) { 155 pr_err("%pfwP: Failed to create INTC mapping\n", fwnode); 156 return -ENOENT; 157 } 158 159 /* Initialize IPI domain */ 160 rc = imsic_ipi_domain_init(); 161 if (rc) { 162 pr_err("%pfwP: Failed to initialize IPI domain\n", fwnode); 163 return rc; 164 } 165 166 /* Setup chained handler to the parent domain interrupt */ 167 irq_set_chained_handler(imsic_parent_irq, imsic_handle_irq); 168 169 /* 170 * Setup cpuhp state (must be done after setting imsic_parent_irq) 171 * 172 * Don't disable per-CPU IMSIC file when CPU goes offline 173 * because this affects IPI and the masking/unmasking of 174 * virtual IPIs is done via generic IPI-Mux 175 */ 176 cpuhp_setup_state(CPUHP_AP_IRQ_RISCV_IMSIC_STARTING, "irqchip/riscv/imsic:starting", 177 imsic_starting_cpu, imsic_dying_cpu); 178 179 return 0; 180 } 181 182 static int __init imsic_early_dt_init(struct device_node *node, struct device_node *parent) 183 { 184 struct fwnode_handle *fwnode = &node->fwnode; 185 int rc; 186 187 /* Setup IMSIC state */ 188 rc = imsic_setup_state(fwnode, NULL); 189 if (rc) { 190 pr_err("%pfwP: failed to setup state (error %d)\n", fwnode, rc); 191 return rc; 192 } 193 194 /* Do early setup of IPIs */ 195 rc = imsic_early_probe(fwnode); 196 if (rc) 197 return rc; 198 199 /* Ensure that OF platform device gets probed */ 200 of_node_clear_flag(node, OF_POPULATED); 201 return 0; 202 } 203 204 IRQCHIP_DECLARE(riscv_imsic, "riscv,imsics", imsic_early_dt_init); 205 206 #ifdef CONFIG_ACPI 207 208 static struct fwnode_handle *imsic_acpi_fwnode; 209 210 struct fwnode_handle *imsic_acpi_get_fwnode(struct device *dev) 211 { 212 return imsic_acpi_fwnode; 213 } 214 215 static int __init imsic_early_acpi_init(union acpi_subtable_headers *header, 216 const unsigned long end) 217 { 218 struct acpi_madt_imsic *imsic = (struct acpi_madt_imsic *)header; 219 int rc; 220 221 imsic_acpi_fwnode = irq_domain_alloc_named_fwnode("imsic"); 222 if (!imsic_acpi_fwnode) { 223 pr_err("unable to allocate IMSIC FW node\n"); 224 return -ENOMEM; 225 } 226 227 /* Setup IMSIC state */ 228 rc = imsic_setup_state(imsic_acpi_fwnode, imsic); 229 if (rc) { 230 pr_err("%pfwP: failed to setup state (error %d)\n", imsic_acpi_fwnode, rc); 231 return rc; 232 } 233 234 /* Do early setup of IMSIC state and IPIs */ 235 rc = imsic_early_probe(imsic_acpi_fwnode); 236 if (rc) { 237 irq_domain_free_fwnode(imsic_acpi_fwnode); 238 imsic_acpi_fwnode = NULL; 239 return rc; 240 } 241 242 rc = imsic_platform_acpi_probe(imsic_acpi_fwnode); 243 244 #ifdef CONFIG_PCI 245 if (!rc) 246 pci_msi_register_fwnode_provider(&imsic_acpi_get_fwnode); 247 #endif 248 249 if (rc) 250 pr_err("%pfwP: failed to register IMSIC for MSI functionality (error %d)\n", 251 imsic_acpi_fwnode, rc); 252 253 /* 254 * Even if imsic_platform_acpi_probe() fails, the IPI part of IMSIC can 255 * continue to work. So, no need to return failure. This is similar to 256 * DT where IPI works but MSI probe fails for some reason. 257 */ 258 return 0; 259 } 260 261 IRQCHIP_ACPI_DECLARE(riscv_imsic, ACPI_MADT_TYPE_IMSIC, NULL, 262 1, imsic_early_acpi_init); 263 #endif 264