1 /* 2 * linux/kernel/irq/ipi.c 3 * 4 * Copyright (C) 2015 Imagination Technologies Ltd 5 * Author: Qais Yousef <qais.yousef@imgtec.com> 6 * 7 * This file contains driver APIs to the IPI subsystem. 8 */ 9 10 #define pr_fmt(fmt) "genirq/ipi: " fmt 11 12 #include <linux/irqdomain.h> 13 #include <linux/irq.h> 14 15 /** 16 * irq_reserve_ipi() - Setup an IPI to destination cpumask 17 * @domain: IPI domain 18 * @dest: cpumask of cpus which can receive the IPI 19 * 20 * Allocate a virq that can be used to send IPI to any CPU in dest mask. 21 * 22 * On success it'll return linux irq number and 0 on failure 23 */ 24 unsigned int irq_reserve_ipi(struct irq_domain *domain, 25 const struct cpumask *dest) 26 { 27 unsigned int nr_irqs, offset; 28 struct irq_data *data; 29 int virq, i; 30 31 if (!domain ||!irq_domain_is_ipi(domain)) { 32 pr_warn("Reservation on a non IPI domain\n"); 33 return 0; 34 } 35 36 if (!cpumask_subset(dest, cpu_possible_mask)) { 37 pr_warn("Reservation is not in possible_cpu_mask\n"); 38 return 0; 39 } 40 41 nr_irqs = cpumask_weight(dest); 42 if (!nr_irqs) { 43 pr_warn("Reservation for empty destination mask\n"); 44 return 0; 45 } 46 47 if (irq_domain_is_ipi_single(domain)) { 48 /* 49 * If the underlying implementation uses a single HW irq on 50 * all cpus then we only need a single Linux irq number for 51 * it. We have no restrictions vs. the destination mask. The 52 * underlying implementation can deal with holes nicely. 53 */ 54 nr_irqs = 1; 55 offset = 0; 56 } else { 57 unsigned int next; 58 59 /* 60 * The IPI requires a seperate HW irq on each CPU. We require 61 * that the destination mask is consecutive. If an 62 * implementation needs to support holes, it can reserve 63 * several IPI ranges. 64 */ 65 offset = cpumask_first(dest); 66 /* 67 * Find a hole and if found look for another set bit after the 68 * hole. For now we don't support this scenario. 69 */ 70 next = cpumask_next_zero(offset, dest); 71 if (next < nr_cpu_ids) 72 next = cpumask_next(next, dest); 73 if (next < nr_cpu_ids) { 74 pr_warn("Destination mask has holes\n"); 75 return 0; 76 } 77 } 78 79 virq = irq_domain_alloc_descs(-1, nr_irqs, 0, NUMA_NO_NODE); 80 if (virq <= 0) { 81 pr_warn("Can't reserve IPI, failed to alloc descs\n"); 82 return 0; 83 } 84 85 virq = __irq_domain_alloc_irqs(domain, virq, nr_irqs, NUMA_NO_NODE, 86 (void *) dest, true); 87 88 if (virq <= 0) { 89 pr_warn("Can't reserve IPI, failed to alloc hw irqs\n"); 90 goto free_descs; 91 } 92 93 for (i = 0; i < nr_irqs; i++) { 94 data = irq_get_irq_data(virq + i); 95 cpumask_copy(data->common->affinity, dest); 96 data->common->ipi_offset = offset; 97 } 98 return virq; 99 100 free_descs: 101 irq_free_descs(virq, nr_irqs); 102 return 0; 103 } 104 105 /** 106 * irq_destroy_ipi() - unreserve an IPI that was previously allocated 107 * @irq: linux irq number to be destroyed 108 * 109 * Return the IPIs allocated with irq_reserve_ipi() to the system destroying 110 * all virqs associated with them. 111 */ 112 void irq_destroy_ipi(unsigned int irq) 113 { 114 struct irq_data *data = irq_get_irq_data(irq); 115 struct cpumask *ipimask = data ? irq_data_get_affinity_mask(data) : NULL; 116 struct irq_domain *domain; 117 unsigned int nr_irqs; 118 119 if (!irq || !data || !ipimask) 120 return; 121 122 domain = data->domain; 123 if (WARN_ON(domain == NULL)) 124 return; 125 126 if (!irq_domain_is_ipi(domain)) { 127 pr_warn("Trying to destroy a non IPI domain!\n"); 128 return; 129 } 130 131 if (irq_domain_is_ipi_per_cpu(domain)) 132 nr_irqs = cpumask_weight(ipimask); 133 else 134 nr_irqs = 1; 135 136 irq_domain_free_irqs(irq, nr_irqs); 137 } 138 139 /** 140 * ipi_get_hwirq - Get the hwirq associated with an IPI to a cpu 141 * @irq: linux irq number 142 * @cpu: the target cpu 143 * 144 * When dealing with coprocessors IPI, we need to inform the coprocessor of 145 * the hwirq it needs to use to receive and send IPIs. 146 * 147 * Returns hwirq value on success and INVALID_HWIRQ on failure. 148 */ 149 irq_hw_number_t ipi_get_hwirq(unsigned int irq, unsigned int cpu) 150 { 151 struct irq_data *data = irq_get_irq_data(irq); 152 struct cpumask *ipimask = data ? irq_data_get_affinity_mask(data) : NULL; 153 154 if (!data || !ipimask || cpu > nr_cpu_ids) 155 return INVALID_HWIRQ; 156 157 if (!cpumask_test_cpu(cpu, ipimask)) 158 return INVALID_HWIRQ; 159 160 /* 161 * Get the real hardware irq number if the underlying implementation 162 * uses a seperate irq per cpu. If the underlying implementation uses 163 * a single hardware irq for all cpus then the IPI send mechanism 164 * needs to take care of the cpu destinations. 165 */ 166 if (irq_domain_is_ipi_per_cpu(data->domain)) 167 data = irq_get_irq_data(irq + cpu - data->common->ipi_offset); 168 169 return data ? irqd_to_hwirq(data) : INVALID_HWIRQ; 170 } 171 EXPORT_SYMBOL_GPL(ipi_get_hwirq); 172 173 static int ipi_send_verify(struct irq_chip *chip, struct irq_data *data, 174 const struct cpumask *dest, unsigned int cpu) 175 { 176 struct cpumask *ipimask = irq_data_get_affinity_mask(data); 177 178 if (!chip || !ipimask) 179 return -EINVAL; 180 181 if (!chip->ipi_send_single && !chip->ipi_send_mask) 182 return -EINVAL; 183 184 if (cpu > nr_cpu_ids) 185 return -EINVAL; 186 187 if (dest) { 188 if (!cpumask_subset(dest, ipimask)) 189 return -EINVAL; 190 } else { 191 if (!cpumask_test_cpu(cpu, ipimask)) 192 return -EINVAL; 193 } 194 return 0; 195 } 196 197 /** 198 * __ipi_send_single - send an IPI to a target Linux SMP CPU 199 * @desc: pointer to irq_desc of the IRQ 200 * @cpu: destination CPU, must in the destination mask passed to 201 * irq_reserve_ipi() 202 * 203 * This function is for architecture or core code to speed up IPI sending. Not 204 * usable from driver code. 205 * 206 * Returns zero on success and negative error number on failure. 207 */ 208 int __ipi_send_single(struct irq_desc *desc, unsigned int cpu) 209 { 210 struct irq_data *data = irq_desc_get_irq_data(desc); 211 struct irq_chip *chip = irq_data_get_irq_chip(data); 212 213 #ifdef DEBUG 214 /* 215 * Minimise the overhead by omitting the checks for Linux SMP IPIs. 216 * Since the callers should be arch or core code which is generally 217 * trusted, only check for errors when debugging. 218 */ 219 if (WARN_ON_ONCE(ipi_send_verify(chip, data, NULL, cpu))) 220 return -EINVAL; 221 #endif 222 if (!chip->ipi_send_single) { 223 chip->ipi_send_mask(data, cpumask_of(cpu)); 224 return 0; 225 } 226 227 /* FIXME: Store this information in irqdata flags */ 228 if (irq_domain_is_ipi_per_cpu(data->domain) && 229 cpu != data->common->ipi_offset) { 230 /* use the correct data for that cpu */ 231 unsigned irq = data->irq + cpu - data->common->ipi_offset; 232 233 data = irq_get_irq_data(irq); 234 } 235 chip->ipi_send_single(data, cpu); 236 return 0; 237 } 238 239 /** 240 * ipi_send_mask - send an IPI to target Linux SMP CPU(s) 241 * @desc: pointer to irq_desc of the IRQ 242 * @dest: dest CPU(s), must be a subset of the mask passed to 243 * irq_reserve_ipi() 244 * 245 * This function is for architecture or core code to speed up IPI sending. Not 246 * usable from driver code. 247 * 248 * Returns zero on success and negative error number on failure. 249 */ 250 int __ipi_send_mask(struct irq_desc *desc, const struct cpumask *dest) 251 { 252 struct irq_data *data = irq_desc_get_irq_data(desc); 253 struct irq_chip *chip = irq_data_get_irq_chip(data); 254 unsigned int cpu; 255 256 #ifdef DEBUG 257 /* 258 * Minimise the overhead by omitting the checks for Linux SMP IPIs. 259 * Since the callers should be arch or core code which is generally 260 * trusted, only check for errors when debugging. 261 */ 262 if (WARN_ON_ONCE(ipi_send_verify(chip, data, dest, 0))) 263 return -EINVAL; 264 #endif 265 if (chip->ipi_send_mask) { 266 chip->ipi_send_mask(data, dest); 267 return 0; 268 } 269 270 if (irq_domain_is_ipi_per_cpu(data->domain)) { 271 unsigned int base = data->irq; 272 273 for_each_cpu(cpu, dest) { 274 unsigned irq = base + cpu - data->common->ipi_offset; 275 276 data = irq_get_irq_data(irq); 277 chip->ipi_send_single(data, cpu); 278 } 279 } else { 280 for_each_cpu(cpu, dest) 281 chip->ipi_send_single(data, cpu); 282 } 283 return 0; 284 } 285 286 /** 287 * ipi_send_single - Send an IPI to a single CPU 288 * @virq: linux irq number from irq_reserve_ipi() 289 * @cpu: destination CPU, must in the destination mask passed to 290 * irq_reserve_ipi() 291 * 292 * Returns zero on success and negative error number on failure. 293 */ 294 int ipi_send_single(unsigned int virq, unsigned int cpu) 295 { 296 struct irq_desc *desc = irq_to_desc(virq); 297 struct irq_data *data = desc ? irq_desc_get_irq_data(desc) : NULL; 298 struct irq_chip *chip = data ? irq_data_get_irq_chip(data) : NULL; 299 300 if (WARN_ON_ONCE(ipi_send_verify(chip, data, NULL, cpu))) 301 return -EINVAL; 302 303 return __ipi_send_single(desc, cpu); 304 } 305 EXPORT_SYMBOL_GPL(ipi_send_single); 306 307 /** 308 * ipi_send_mask - Send an IPI to target CPU(s) 309 * @virq: linux irq number from irq_reserve_ipi() 310 * @dest: dest CPU(s), must be a subset of the mask passed to 311 * irq_reserve_ipi() 312 * 313 * Returns zero on success and negative error number on failure. 314 */ 315 int ipi_send_mask(unsigned int virq, const struct cpumask *dest) 316 { 317 struct irq_desc *desc = irq_to_desc(virq); 318 struct irq_data *data = desc ? irq_desc_get_irq_data(desc) : NULL; 319 struct irq_chip *chip = data ? irq_data_get_irq_chip(data) : NULL; 320 321 if (WARN_ON_ONCE(ipi_send_verify(chip, data, dest, 0))) 322 return -EINVAL; 323 324 return __ipi_send_mask(desc, dest); 325 } 326 EXPORT_SYMBOL_GPL(ipi_send_mask); 327