1 /* 2 * linux/kernel/irq/ipi.c 3 * 4 * Copyright (C) 2015 Imagination Technologies Ltd 5 * Author: Qais Yousef <qais.yousef@imgtec.com> 6 * 7 * This file contains driver APIs to the IPI subsystem. 8 */ 9 10 #define pr_fmt(fmt) "genirq/ipi: " fmt 11 12 #include <linux/irqdomain.h> 13 #include <linux/irq.h> 14 15 /** 16 * irq_reserve_ipi() - Setup an IPI to destination cpumask 17 * @domain: IPI domain 18 * @dest: cpumask of cpus which can receive the IPI 19 * 20 * Allocate a virq that can be used to send IPI to any CPU in dest mask. 21 * 22 * On success it'll return linux irq number and error code on failure 23 */ 24 int irq_reserve_ipi(struct irq_domain *domain, 25 const struct cpumask *dest) 26 { 27 unsigned int nr_irqs, offset; 28 struct irq_data *data; 29 int virq, i; 30 31 if (!domain ||!irq_domain_is_ipi(domain)) { 32 pr_warn("Reservation on a non IPI domain\n"); 33 return -EINVAL; 34 } 35 36 if (!cpumask_subset(dest, cpu_possible_mask)) { 37 pr_warn("Reservation is not in possible_cpu_mask\n"); 38 return -EINVAL; 39 } 40 41 nr_irqs = cpumask_weight(dest); 42 if (!nr_irqs) { 43 pr_warn("Reservation for empty destination mask\n"); 44 return -EINVAL; 45 } 46 47 if (irq_domain_is_ipi_single(domain)) { 48 /* 49 * If the underlying implementation uses a single HW irq on 50 * all cpus then we only need a single Linux irq number for 51 * it. We have no restrictions vs. the destination mask. The 52 * underlying implementation can deal with holes nicely. 53 */ 54 nr_irqs = 1; 55 offset = 0; 56 } else { 57 unsigned int next; 58 59 /* 60 * The IPI requires a seperate HW irq on each CPU. We require 61 * that the destination mask is consecutive. If an 62 * implementation needs to support holes, it can reserve 63 * several IPI ranges. 64 */ 65 offset = cpumask_first(dest); 66 /* 67 * Find a hole and if found look for another set bit after the 68 * hole. For now we don't support this scenario. 69 */ 70 next = cpumask_next_zero(offset, dest); 71 if (next < nr_cpu_ids) 72 next = cpumask_next(next, dest); 73 if (next < nr_cpu_ids) { 74 pr_warn("Destination mask has holes\n"); 75 return -EINVAL; 76 } 77 } 78 79 virq = irq_domain_alloc_descs(-1, nr_irqs, 0, NUMA_NO_NODE, NULL); 80 if (virq <= 0) { 81 pr_warn("Can't reserve IPI, failed to alloc descs\n"); 82 return -ENOMEM; 83 } 84 85 virq = __irq_domain_alloc_irqs(domain, virq, nr_irqs, NUMA_NO_NODE, 86 (void *) dest, true, NULL); 87 88 if (virq <= 0) { 89 pr_warn("Can't reserve IPI, failed to alloc hw irqs\n"); 90 goto free_descs; 91 } 92 93 for (i = 0; i < nr_irqs; i++) { 94 data = irq_get_irq_data(virq + i); 95 cpumask_copy(data->common->affinity, dest); 96 data->common->ipi_offset = offset; 97 irq_set_status_flags(virq + i, IRQ_NO_BALANCING); 98 } 99 return virq; 100 101 free_descs: 102 irq_free_descs(virq, nr_irqs); 103 return -EBUSY; 104 } 105 106 /** 107 * irq_destroy_ipi() - unreserve an IPI that was previously allocated 108 * @irq: linux irq number to be destroyed 109 * @dest: cpumask of cpus which should have the IPI removed 110 * 111 * The IPIs allocated with irq_reserve_ipi() are retuerned to the system 112 * destroying all virqs associated with them. 113 * 114 * Return 0 on success or error code on failure. 115 */ 116 int irq_destroy_ipi(unsigned int irq, const struct cpumask *dest) 117 { 118 struct irq_data *data = irq_get_irq_data(irq); 119 struct cpumask *ipimask = data ? irq_data_get_affinity_mask(data) : NULL; 120 struct irq_domain *domain; 121 unsigned int nr_irqs; 122 123 if (!irq || !data || !ipimask) 124 return -EINVAL; 125 126 domain = data->domain; 127 if (WARN_ON(domain == NULL)) 128 return -EINVAL; 129 130 if (!irq_domain_is_ipi(domain)) { 131 pr_warn("Trying to destroy a non IPI domain!\n"); 132 return -EINVAL; 133 } 134 135 if (WARN_ON(!cpumask_subset(dest, ipimask))) 136 /* 137 * Must be destroying a subset of CPUs to which this IPI 138 * was set up to target 139 */ 140 return -EINVAL; 141 142 if (irq_domain_is_ipi_per_cpu(domain)) { 143 irq = irq + cpumask_first(dest) - data->common->ipi_offset; 144 nr_irqs = cpumask_weight(dest); 145 } else { 146 nr_irqs = 1; 147 } 148 149 irq_domain_free_irqs(irq, nr_irqs); 150 return 0; 151 } 152 153 /** 154 * ipi_get_hwirq - Get the hwirq associated with an IPI to a cpu 155 * @irq: linux irq number 156 * @cpu: the target cpu 157 * 158 * When dealing with coprocessors IPI, we need to inform the coprocessor of 159 * the hwirq it needs to use to receive and send IPIs. 160 * 161 * Returns hwirq value on success and INVALID_HWIRQ on failure. 162 */ 163 irq_hw_number_t ipi_get_hwirq(unsigned int irq, unsigned int cpu) 164 { 165 struct irq_data *data = irq_get_irq_data(irq); 166 struct cpumask *ipimask = data ? irq_data_get_affinity_mask(data) : NULL; 167 168 if (!data || !ipimask || cpu >= nr_cpu_ids) 169 return INVALID_HWIRQ; 170 171 if (!cpumask_test_cpu(cpu, ipimask)) 172 return INVALID_HWIRQ; 173 174 /* 175 * Get the real hardware irq number if the underlying implementation 176 * uses a seperate irq per cpu. If the underlying implementation uses 177 * a single hardware irq for all cpus then the IPI send mechanism 178 * needs to take care of the cpu destinations. 179 */ 180 if (irq_domain_is_ipi_per_cpu(data->domain)) 181 data = irq_get_irq_data(irq + cpu - data->common->ipi_offset); 182 183 return data ? irqd_to_hwirq(data) : INVALID_HWIRQ; 184 } 185 EXPORT_SYMBOL_GPL(ipi_get_hwirq); 186 187 static int ipi_send_verify(struct irq_chip *chip, struct irq_data *data, 188 const struct cpumask *dest, unsigned int cpu) 189 { 190 struct cpumask *ipimask = irq_data_get_affinity_mask(data); 191 192 if (!chip || !ipimask) 193 return -EINVAL; 194 195 if (!chip->ipi_send_single && !chip->ipi_send_mask) 196 return -EINVAL; 197 198 if (cpu >= nr_cpu_ids) 199 return -EINVAL; 200 201 if (dest) { 202 if (!cpumask_subset(dest, ipimask)) 203 return -EINVAL; 204 } else { 205 if (!cpumask_test_cpu(cpu, ipimask)) 206 return -EINVAL; 207 } 208 return 0; 209 } 210 211 /** 212 * __ipi_send_single - send an IPI to a target Linux SMP CPU 213 * @desc: pointer to irq_desc of the IRQ 214 * @cpu: destination CPU, must in the destination mask passed to 215 * irq_reserve_ipi() 216 * 217 * This function is for architecture or core code to speed up IPI sending. Not 218 * usable from driver code. 219 * 220 * Returns zero on success and negative error number on failure. 221 */ 222 int __ipi_send_single(struct irq_desc *desc, unsigned int cpu) 223 { 224 struct irq_data *data = irq_desc_get_irq_data(desc); 225 struct irq_chip *chip = irq_data_get_irq_chip(data); 226 227 #ifdef DEBUG 228 /* 229 * Minimise the overhead by omitting the checks for Linux SMP IPIs. 230 * Since the callers should be arch or core code which is generally 231 * trusted, only check for errors when debugging. 232 */ 233 if (WARN_ON_ONCE(ipi_send_verify(chip, data, NULL, cpu))) 234 return -EINVAL; 235 #endif 236 if (!chip->ipi_send_single) { 237 chip->ipi_send_mask(data, cpumask_of(cpu)); 238 return 0; 239 } 240 241 /* FIXME: Store this information in irqdata flags */ 242 if (irq_domain_is_ipi_per_cpu(data->domain) && 243 cpu != data->common->ipi_offset) { 244 /* use the correct data for that cpu */ 245 unsigned irq = data->irq + cpu - data->common->ipi_offset; 246 247 data = irq_get_irq_data(irq); 248 } 249 chip->ipi_send_single(data, cpu); 250 return 0; 251 } 252 253 /** 254 * ipi_send_mask - send an IPI to target Linux SMP CPU(s) 255 * @desc: pointer to irq_desc of the IRQ 256 * @dest: dest CPU(s), must be a subset of the mask passed to 257 * irq_reserve_ipi() 258 * 259 * This function is for architecture or core code to speed up IPI sending. Not 260 * usable from driver code. 261 * 262 * Returns zero on success and negative error number on failure. 263 */ 264 int __ipi_send_mask(struct irq_desc *desc, const struct cpumask *dest) 265 { 266 struct irq_data *data = irq_desc_get_irq_data(desc); 267 struct irq_chip *chip = irq_data_get_irq_chip(data); 268 unsigned int cpu; 269 270 #ifdef DEBUG 271 /* 272 * Minimise the overhead by omitting the checks for Linux SMP IPIs. 273 * Since the callers should be arch or core code which is generally 274 * trusted, only check for errors when debugging. 275 */ 276 if (WARN_ON_ONCE(ipi_send_verify(chip, data, dest, 0))) 277 return -EINVAL; 278 #endif 279 if (chip->ipi_send_mask) { 280 chip->ipi_send_mask(data, dest); 281 return 0; 282 } 283 284 if (irq_domain_is_ipi_per_cpu(data->domain)) { 285 unsigned int base = data->irq; 286 287 for_each_cpu(cpu, dest) { 288 unsigned irq = base + cpu - data->common->ipi_offset; 289 290 data = irq_get_irq_data(irq); 291 chip->ipi_send_single(data, cpu); 292 } 293 } else { 294 for_each_cpu(cpu, dest) 295 chip->ipi_send_single(data, cpu); 296 } 297 return 0; 298 } 299 300 /** 301 * ipi_send_single - Send an IPI to a single CPU 302 * @virq: linux irq number from irq_reserve_ipi() 303 * @cpu: destination CPU, must in the destination mask passed to 304 * irq_reserve_ipi() 305 * 306 * Returns zero on success and negative error number on failure. 307 */ 308 int ipi_send_single(unsigned int virq, unsigned int cpu) 309 { 310 struct irq_desc *desc = irq_to_desc(virq); 311 struct irq_data *data = desc ? irq_desc_get_irq_data(desc) : NULL; 312 struct irq_chip *chip = data ? irq_data_get_irq_chip(data) : NULL; 313 314 if (WARN_ON_ONCE(ipi_send_verify(chip, data, NULL, cpu))) 315 return -EINVAL; 316 317 return __ipi_send_single(desc, cpu); 318 } 319 EXPORT_SYMBOL_GPL(ipi_send_single); 320 321 /** 322 * ipi_send_mask - Send an IPI to target CPU(s) 323 * @virq: linux irq number from irq_reserve_ipi() 324 * @dest: dest CPU(s), must be a subset of the mask passed to 325 * irq_reserve_ipi() 326 * 327 * Returns zero on success and negative error number on failure. 328 */ 329 int ipi_send_mask(unsigned int virq, const struct cpumask *dest) 330 { 331 struct irq_desc *desc = irq_to_desc(virq); 332 struct irq_data *data = desc ? irq_desc_get_irq_data(desc) : NULL; 333 struct irq_chip *chip = data ? irq_data_get_irq_chip(data) : NULL; 334 335 if (WARN_ON_ONCE(ipi_send_verify(chip, data, dest, 0))) 336 return -EINVAL; 337 338 return __ipi_send_mask(desc, dest); 339 } 340 EXPORT_SYMBOL_GPL(ipi_send_mask); 341