1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2015 Imagination Technologies Ltd 4 * Author: Qais Yousef <qais.yousef@imgtec.com> 5 * 6 * This file contains driver APIs to the IPI subsystem. 7 */ 8 9 #define pr_fmt(fmt) "genirq/ipi: " fmt 10 11 #include <linux/irqdomain.h> 12 #include <linux/irq.h> 13 14 /** 15 * irq_reserve_ipi() - Setup an IPI to destination cpumask 16 * @domain: IPI domain 17 * @dest: cpumask of CPUs which can receive the IPI 18 * 19 * Allocate a virq that can be used to send IPI to any CPU in dest mask. 20 * 21 * Return: Linux IRQ number on success or error code on failure 22 */ 23 int irq_reserve_ipi(struct irq_domain *domain, 24 const struct cpumask *dest) 25 { 26 unsigned int nr_irqs, offset; 27 struct irq_data *data; 28 int virq, i; 29 30 if (!domain ||!irq_domain_is_ipi(domain)) { 31 pr_warn("Reservation on a non IPI domain\n"); 32 return -EINVAL; 33 } 34 35 if (!cpumask_subset(dest, cpu_possible_mask)) { 36 pr_warn("Reservation is not in possible_cpu_mask\n"); 37 return -EINVAL; 38 } 39 40 nr_irqs = cpumask_weight(dest); 41 if (!nr_irqs) { 42 pr_warn("Reservation for empty destination mask\n"); 43 return -EINVAL; 44 } 45 46 if (irq_domain_is_ipi_single(domain)) { 47 /* 48 * If the underlying implementation uses a single HW irq on 49 * all cpus then we only need a single Linux irq number for 50 * it. We have no restrictions vs. the destination mask. The 51 * underlying implementation can deal with holes nicely. 52 */ 53 nr_irqs = 1; 54 offset = 0; 55 } else { 56 unsigned int next; 57 58 /* 59 * The IPI requires a separate HW irq on each CPU. We require 60 * that the destination mask is consecutive. If an 61 * implementation needs to support holes, it can reserve 62 * several IPI ranges. 63 */ 64 offset = cpumask_first(dest); 65 /* 66 * Find a hole and if found look for another set bit after the 67 * hole. For now we don't support this scenario. 68 */ 69 next = cpumask_next_zero(offset, dest); 70 if (next < nr_cpu_ids) 71 next = cpumask_next(next, dest); 72 if (next < nr_cpu_ids) { 73 pr_warn("Destination mask has holes\n"); 74 return -EINVAL; 75 } 76 } 77 78 virq = irq_domain_alloc_descs(-1, nr_irqs, 0, NUMA_NO_NODE, NULL); 79 if (virq <= 0) { 80 pr_warn("Can't reserve IPI, failed to alloc descs\n"); 81 return -ENOMEM; 82 } 83 84 virq = __irq_domain_alloc_irqs(domain, virq, nr_irqs, NUMA_NO_NODE, 85 (void *) dest, true, NULL); 86 87 if (virq <= 0) { 88 pr_warn("Can't reserve IPI, failed to alloc hw irqs\n"); 89 goto free_descs; 90 } 91 92 for (i = 0; i < nr_irqs; i++) { 93 data = irq_get_irq_data(virq + i); 94 cpumask_copy(data->common->affinity, dest); 95 data->common->ipi_offset = offset; 96 irq_set_status_flags(virq + i, IRQ_NO_BALANCING); 97 } 98 return virq; 99 100 free_descs: 101 irq_free_descs(virq, nr_irqs); 102 return -EBUSY; 103 } 104 105 /** 106 * irq_destroy_ipi() - unreserve an IPI that was previously allocated 107 * @irq: Linux IRQ number to be destroyed 108 * @dest: cpumask of CPUs which should have the IPI removed 109 * 110 * The IPIs allocated with irq_reserve_ipi() are returned to the system 111 * destroying all virqs associated with them. 112 * 113 * Return: %0 on success or error code on failure. 114 */ 115 int irq_destroy_ipi(unsigned int irq, const struct cpumask *dest) 116 { 117 struct irq_data *data = irq_get_irq_data(irq); 118 const struct cpumask *ipimask; 119 struct irq_domain *domain; 120 unsigned int nr_irqs; 121 122 if (!irq || !data) 123 return -EINVAL; 124 125 domain = data->domain; 126 if (WARN_ON(domain == NULL)) 127 return -EINVAL; 128 129 if (!irq_domain_is_ipi(domain)) { 130 pr_warn("Trying to destroy a non IPI domain!\n"); 131 return -EINVAL; 132 } 133 134 ipimask = irq_data_get_affinity_mask(data); 135 if (!ipimask || WARN_ON(!cpumask_subset(dest, ipimask))) 136 /* 137 * Must be destroying a subset of CPUs to which this IPI 138 * was set up to target 139 */ 140 return -EINVAL; 141 142 if (irq_domain_is_ipi_per_cpu(domain)) { 143 irq = irq + cpumask_first(dest) - data->common->ipi_offset; 144 nr_irqs = cpumask_weight(dest); 145 } else { 146 nr_irqs = 1; 147 } 148 149 irq_domain_free_irqs(irq, nr_irqs); 150 return 0; 151 } 152 153 /** 154 * ipi_get_hwirq - Get the hwirq associated with an IPI to a CPU 155 * @irq: Linux IRQ number 156 * @cpu: the target CPU 157 * 158 * When dealing with coprocessors IPI, we need to inform the coprocessor of 159 * the hwirq it needs to use to receive and send IPIs. 160 * 161 * Return: hwirq value on success or INVALID_HWIRQ on failure. 162 */ 163 irq_hw_number_t ipi_get_hwirq(unsigned int irq, unsigned int cpu) 164 { 165 struct irq_data *data = irq_get_irq_data(irq); 166 const struct cpumask *ipimask; 167 168 if (!data || cpu >= nr_cpu_ids) 169 return INVALID_HWIRQ; 170 171 ipimask = irq_data_get_affinity_mask(data); 172 if (!ipimask || !cpumask_test_cpu(cpu, ipimask)) 173 return INVALID_HWIRQ; 174 175 /* 176 * Get the real hardware irq number if the underlying implementation 177 * uses a separate irq per cpu. If the underlying implementation uses 178 * a single hardware irq for all cpus then the IPI send mechanism 179 * needs to take care of the cpu destinations. 180 */ 181 if (irq_domain_is_ipi_per_cpu(data->domain)) 182 data = irq_get_irq_data(irq + cpu - data->common->ipi_offset); 183 184 return data ? irqd_to_hwirq(data) : INVALID_HWIRQ; 185 } 186 EXPORT_SYMBOL_GPL(ipi_get_hwirq); 187 188 static int ipi_send_verify(struct irq_chip *chip, struct irq_data *data, 189 const struct cpumask *dest, unsigned int cpu) 190 { 191 const struct cpumask *ipimask; 192 193 if (!chip || !data) 194 return -EINVAL; 195 196 if (!chip->ipi_send_single && !chip->ipi_send_mask) 197 return -EINVAL; 198 199 if (cpu >= nr_cpu_ids) 200 return -EINVAL; 201 202 ipimask = irq_data_get_affinity_mask(data); 203 if (!ipimask) 204 return -EINVAL; 205 206 if (dest) { 207 if (!cpumask_subset(dest, ipimask)) 208 return -EINVAL; 209 } else { 210 if (!cpumask_test_cpu(cpu, ipimask)) 211 return -EINVAL; 212 } 213 return 0; 214 } 215 216 /** 217 * __ipi_send_single - send an IPI to a target Linux SMP CPU 218 * @desc: pointer to irq_desc of the IRQ 219 * @cpu: destination CPU, must in the destination mask passed to 220 * irq_reserve_ipi() 221 * 222 * This function is for architecture or core code to speed up IPI sending. Not 223 * usable from driver code. 224 * 225 * Return: %0 on success or negative error number on failure. 226 */ 227 int __ipi_send_single(struct irq_desc *desc, unsigned int cpu) 228 { 229 struct irq_data *data = irq_desc_get_irq_data(desc); 230 struct irq_chip *chip = irq_data_get_irq_chip(data); 231 232 #ifdef DEBUG 233 /* 234 * Minimise the overhead by omitting the checks for Linux SMP IPIs. 235 * Since the callers should be arch or core code which is generally 236 * trusted, only check for errors when debugging. 237 */ 238 if (WARN_ON_ONCE(ipi_send_verify(chip, data, NULL, cpu))) 239 return -EINVAL; 240 #endif 241 if (!chip->ipi_send_single) { 242 chip->ipi_send_mask(data, cpumask_of(cpu)); 243 return 0; 244 } 245 246 /* FIXME: Store this information in irqdata flags */ 247 if (irq_domain_is_ipi_per_cpu(data->domain) && 248 cpu != data->common->ipi_offset) { 249 /* use the correct data for that cpu */ 250 unsigned irq = data->irq + cpu - data->common->ipi_offset; 251 252 data = irq_get_irq_data(irq); 253 } 254 chip->ipi_send_single(data, cpu); 255 return 0; 256 } 257 258 /** 259 * __ipi_send_mask - send an IPI to target Linux SMP CPU(s) 260 * @desc: pointer to irq_desc of the IRQ 261 * @dest: dest CPU(s), must be a subset of the mask passed to 262 * irq_reserve_ipi() 263 * 264 * This function is for architecture or core code to speed up IPI sending. Not 265 * usable from driver code. 266 * 267 * Return: %0 on success or negative error number on failure. 268 */ 269 int __ipi_send_mask(struct irq_desc *desc, const struct cpumask *dest) 270 { 271 struct irq_data *data = irq_desc_get_irq_data(desc); 272 struct irq_chip *chip = irq_data_get_irq_chip(data); 273 unsigned int cpu; 274 275 #ifdef DEBUG 276 /* 277 * Minimise the overhead by omitting the checks for Linux SMP IPIs. 278 * Since the callers should be arch or core code which is generally 279 * trusted, only check for errors when debugging. 280 */ 281 if (WARN_ON_ONCE(ipi_send_verify(chip, data, dest, 0))) 282 return -EINVAL; 283 #endif 284 if (chip->ipi_send_mask) { 285 chip->ipi_send_mask(data, dest); 286 return 0; 287 } 288 289 if (irq_domain_is_ipi_per_cpu(data->domain)) { 290 unsigned int base = data->irq; 291 292 for_each_cpu(cpu, dest) { 293 unsigned irq = base + cpu - data->common->ipi_offset; 294 295 data = irq_get_irq_data(irq); 296 chip->ipi_send_single(data, cpu); 297 } 298 } else { 299 for_each_cpu(cpu, dest) 300 chip->ipi_send_single(data, cpu); 301 } 302 return 0; 303 } 304 305 /** 306 * ipi_send_single - Send an IPI to a single CPU 307 * @virq: Linux IRQ number from irq_reserve_ipi() 308 * @cpu: destination CPU, must in the destination mask passed to 309 * irq_reserve_ipi() 310 * 311 * Return: %0 on success or negative error number on failure. 312 */ 313 int ipi_send_single(unsigned int virq, unsigned int cpu) 314 { 315 struct irq_desc *desc = irq_to_desc(virq); 316 struct irq_data *data = desc ? irq_desc_get_irq_data(desc) : NULL; 317 struct irq_chip *chip = data ? irq_data_get_irq_chip(data) : NULL; 318 319 if (WARN_ON_ONCE(ipi_send_verify(chip, data, NULL, cpu))) 320 return -EINVAL; 321 322 return __ipi_send_single(desc, cpu); 323 } 324 EXPORT_SYMBOL_GPL(ipi_send_single); 325 326 /** 327 * ipi_send_mask - Send an IPI to target CPU(s) 328 * @virq: Linux IRQ number from irq_reserve_ipi() 329 * @dest: dest CPU(s), must be a subset of the mask passed to 330 * irq_reserve_ipi() 331 * 332 * Return: %0 on success or negative error number on failure. 333 */ 334 int ipi_send_mask(unsigned int virq, const struct cpumask *dest) 335 { 336 struct irq_desc *desc = irq_to_desc(virq); 337 struct irq_data *data = desc ? irq_desc_get_irq_data(desc) : NULL; 338 struct irq_chip *chip = data ? irq_data_get_irq_chip(data) : NULL; 339 340 if (WARN_ON_ONCE(ipi_send_verify(chip, data, dest, 0))) 341 return -EINVAL; 342 343 return __ipi_send_mask(desc, dest); 344 } 345 EXPORT_SYMBOL_GPL(ipi_send_mask); 346