xref: /linux/kernel/irq/ipi.c (revision 9a87ffc99ec8eb8d35eed7c4f816d75f5cc9662e)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2015 Imagination Technologies Ltd
4  * Author: Qais Yousef <qais.yousef@imgtec.com>
5  *
6  * This file contains driver APIs to the IPI subsystem.
7  */
8 
9 #define pr_fmt(fmt) "genirq/ipi: " fmt
10 
11 #include <linux/irqdomain.h>
12 #include <linux/irq.h>
13 
14 /**
15  * irq_reserve_ipi() - Setup an IPI to destination cpumask
16  * @domain:	IPI domain
17  * @dest:	cpumask of CPUs which can receive the IPI
18  *
19  * Allocate a virq that can be used to send IPI to any CPU in dest mask.
20  *
21  * Return: Linux IRQ number on success or error code on failure
22  */
irq_reserve_ipi(struct irq_domain * domain,const struct cpumask * dest)23 int irq_reserve_ipi(struct irq_domain *domain,
24 			     const struct cpumask *dest)
25 {
26 	unsigned int nr_irqs, offset;
27 	struct irq_data *data;
28 	int virq, i;
29 
30 	if (!domain ||!irq_domain_is_ipi(domain)) {
31 		pr_warn("Reservation on a non IPI domain\n");
32 		return -EINVAL;
33 	}
34 
35 	if (!cpumask_subset(dest, cpu_possible_mask)) {
36 		pr_warn("Reservation is not in possible_cpu_mask\n");
37 		return -EINVAL;
38 	}
39 
40 	nr_irqs = cpumask_weight(dest);
41 	if (!nr_irqs) {
42 		pr_warn("Reservation for empty destination mask\n");
43 		return -EINVAL;
44 	}
45 
46 	if (irq_domain_is_ipi_single(domain)) {
47 		/*
48 		 * If the underlying implementation uses a single HW irq on
49 		 * all cpus then we only need a single Linux irq number for
50 		 * it. We have no restrictions vs. the destination mask. The
51 		 * underlying implementation can deal with holes nicely.
52 		 */
53 		nr_irqs = 1;
54 		offset = 0;
55 	} else {
56 		unsigned int next;
57 
58 		/*
59 		 * The IPI requires a separate HW irq on each CPU. We require
60 		 * that the destination mask is consecutive. If an
61 		 * implementation needs to support holes, it can reserve
62 		 * several IPI ranges.
63 		 */
64 		offset = cpumask_first(dest);
65 		/*
66 		 * Find a hole and if found look for another set bit after the
67 		 * hole. For now we don't support this scenario.
68 		 */
69 		next = cpumask_next_zero(offset, dest);
70 		if (next < nr_cpu_ids)
71 			next = cpumask_next(next, dest);
72 		if (next < nr_cpu_ids) {
73 			pr_warn("Destination mask has holes\n");
74 			return -EINVAL;
75 		}
76 	}
77 
78 	virq = irq_domain_alloc_descs(-1, nr_irqs, 0, NUMA_NO_NODE, NULL);
79 	if (virq <= 0) {
80 		pr_warn("Can't reserve IPI, failed to alloc descs\n");
81 		return -ENOMEM;
82 	}
83 
84 	virq = __irq_domain_alloc_irqs(domain, virq, nr_irqs, NUMA_NO_NODE,
85 				       (void *) dest, true, NULL);
86 
87 	if (virq <= 0) {
88 		pr_warn("Can't reserve IPI, failed to alloc hw irqs\n");
89 		goto free_descs;
90 	}
91 
92 	for (i = 0; i < nr_irqs; i++) {
93 		data = irq_get_irq_data(virq + i);
94 		cpumask_copy(data->common->affinity, dest);
95 		data->common->ipi_offset = offset;
96 		irq_set_status_flags(virq + i, IRQ_NO_BALANCING);
97 	}
98 	return virq;
99 
100 free_descs:
101 	irq_free_descs(virq, nr_irqs);
102 	return -EBUSY;
103 }
104 
105 /**
106  * irq_destroy_ipi() - unreserve an IPI that was previously allocated
107  * @irq:	Linux IRQ number to be destroyed
108  * @dest:	cpumask of CPUs which should have the IPI removed
109  *
110  * The IPIs allocated with irq_reserve_ipi() are returned to the system
111  * destroying all virqs associated with them.
112  *
113  * Return: %0 on success or error code on failure.
114  */
irq_destroy_ipi(unsigned int irq,const struct cpumask * dest)115 int irq_destroy_ipi(unsigned int irq, const struct cpumask *dest)
116 {
117 	struct irq_data *data = irq_get_irq_data(irq);
118 	const struct cpumask *ipimask;
119 	struct irq_domain *domain;
120 	unsigned int nr_irqs;
121 
122 	if (!irq || !data)
123 		return -EINVAL;
124 
125 	domain = data->domain;
126 	if (WARN_ON(domain == NULL))
127 		return -EINVAL;
128 
129 	if (!irq_domain_is_ipi(domain)) {
130 		pr_warn("Trying to destroy a non IPI domain!\n");
131 		return -EINVAL;
132 	}
133 
134 	ipimask = irq_data_get_affinity_mask(data);
135 	if (!ipimask || WARN_ON(!cpumask_subset(dest, ipimask)))
136 		/*
137 		 * Must be destroying a subset of CPUs to which this IPI
138 		 * was set up to target
139 		 */
140 		return -EINVAL;
141 
142 	if (irq_domain_is_ipi_per_cpu(domain)) {
143 		irq = irq + cpumask_first(dest) - data->common->ipi_offset;
144 		nr_irqs = cpumask_weight(dest);
145 	} else {
146 		nr_irqs = 1;
147 	}
148 
149 	irq_domain_free_irqs(irq, nr_irqs);
150 	return 0;
151 }
152 
153 /**
154  * ipi_get_hwirq - Get the hwirq associated with an IPI to a CPU
155  * @irq:	Linux IRQ number
156  * @cpu:	the target CPU
157  *
158  * When dealing with coprocessors IPI, we need to inform the coprocessor of
159  * the hwirq it needs to use to receive and send IPIs.
160  *
161  * Return: hwirq value on success or INVALID_HWIRQ on failure.
162  */
ipi_get_hwirq(unsigned int irq,unsigned int cpu)163 irq_hw_number_t ipi_get_hwirq(unsigned int irq, unsigned int cpu)
164 {
165 	struct irq_data *data = irq_get_irq_data(irq);
166 	const struct cpumask *ipimask;
167 
168 	if (!data || cpu >= nr_cpu_ids)
169 		return INVALID_HWIRQ;
170 
171 	ipimask = irq_data_get_affinity_mask(data);
172 	if (!ipimask || !cpumask_test_cpu(cpu, ipimask))
173 		return INVALID_HWIRQ;
174 
175 	/*
176 	 * Get the real hardware irq number if the underlying implementation
177 	 * uses a separate irq per cpu. If the underlying implementation uses
178 	 * a single hardware irq for all cpus then the IPI send mechanism
179 	 * needs to take care of the cpu destinations.
180 	 */
181 	if (irq_domain_is_ipi_per_cpu(data->domain))
182 		data = irq_get_irq_data(irq + cpu - data->common->ipi_offset);
183 
184 	return data ? irqd_to_hwirq(data) : INVALID_HWIRQ;
185 }
186 EXPORT_SYMBOL_GPL(ipi_get_hwirq);
187 
ipi_send_verify(struct irq_chip * chip,struct irq_data * data,const struct cpumask * dest,unsigned int cpu)188 static int ipi_send_verify(struct irq_chip *chip, struct irq_data *data,
189 			   const struct cpumask *dest, unsigned int cpu)
190 {
191 	const struct cpumask *ipimask;
192 
193 	if (!chip || !data)
194 		return -EINVAL;
195 
196 	if (!chip->ipi_send_single && !chip->ipi_send_mask)
197 		return -EINVAL;
198 
199 	if (cpu >= nr_cpu_ids)
200 		return -EINVAL;
201 
202 	ipimask = irq_data_get_affinity_mask(data);
203 	if (!ipimask)
204 		return -EINVAL;
205 
206 	if (dest) {
207 		if (!cpumask_subset(dest, ipimask))
208 			return -EINVAL;
209 	} else {
210 		if (!cpumask_test_cpu(cpu, ipimask))
211 			return -EINVAL;
212 	}
213 	return 0;
214 }
215 
216 /**
217  * __ipi_send_single - send an IPI to a target Linux SMP CPU
218  * @desc:	pointer to irq_desc of the IRQ
219  * @cpu:	destination CPU, must in the destination mask passed to
220  *		irq_reserve_ipi()
221  *
222  * This function is for architecture or core code to speed up IPI sending. Not
223  * usable from driver code.
224  *
225  * Return: %0 on success or negative error number on failure.
226  */
__ipi_send_single(struct irq_desc * desc,unsigned int cpu)227 int __ipi_send_single(struct irq_desc *desc, unsigned int cpu)
228 {
229 	struct irq_data *data = irq_desc_get_irq_data(desc);
230 	struct irq_chip *chip = irq_data_get_irq_chip(data);
231 
232 #ifdef DEBUG
233 	/*
234 	 * Minimise the overhead by omitting the checks for Linux SMP IPIs.
235 	 * Since the callers should be arch or core code which is generally
236 	 * trusted, only check for errors when debugging.
237 	 */
238 	if (WARN_ON_ONCE(ipi_send_verify(chip, data, NULL, cpu)))
239 		return -EINVAL;
240 #endif
241 	if (!chip->ipi_send_single) {
242 		chip->ipi_send_mask(data, cpumask_of(cpu));
243 		return 0;
244 	}
245 
246 	/* FIXME: Store this information in irqdata flags */
247 	if (irq_domain_is_ipi_per_cpu(data->domain) &&
248 	    cpu != data->common->ipi_offset) {
249 		/* use the correct data for that cpu */
250 		unsigned irq = data->irq + cpu - data->common->ipi_offset;
251 
252 		data = irq_get_irq_data(irq);
253 	}
254 	chip->ipi_send_single(data, cpu);
255 	return 0;
256 }
257 
258 /**
259  * __ipi_send_mask - send an IPI to target Linux SMP CPU(s)
260  * @desc:	pointer to irq_desc of the IRQ
261  * @dest:	dest CPU(s), must be a subset of the mask passed to
262  *		irq_reserve_ipi()
263  *
264  * This function is for architecture or core code to speed up IPI sending. Not
265  * usable from driver code.
266  *
267  * Return: %0 on success or negative error number on failure.
268  */
__ipi_send_mask(struct irq_desc * desc,const struct cpumask * dest)269 int __ipi_send_mask(struct irq_desc *desc, const struct cpumask *dest)
270 {
271 	struct irq_data *data = irq_desc_get_irq_data(desc);
272 	struct irq_chip *chip = irq_data_get_irq_chip(data);
273 	unsigned int cpu;
274 
275 #ifdef DEBUG
276 	/*
277 	 * Minimise the overhead by omitting the checks for Linux SMP IPIs.
278 	 * Since the callers should be arch or core code which is generally
279 	 * trusted, only check for errors when debugging.
280 	 */
281 	if (WARN_ON_ONCE(ipi_send_verify(chip, data, dest, 0)))
282 		return -EINVAL;
283 #endif
284 	if (chip->ipi_send_mask) {
285 		chip->ipi_send_mask(data, dest);
286 		return 0;
287 	}
288 
289 	if (irq_domain_is_ipi_per_cpu(data->domain)) {
290 		unsigned int base = data->irq;
291 
292 		for_each_cpu(cpu, dest) {
293 			unsigned irq = base + cpu - data->common->ipi_offset;
294 
295 			data = irq_get_irq_data(irq);
296 			chip->ipi_send_single(data, cpu);
297 		}
298 	} else {
299 		for_each_cpu(cpu, dest)
300 			chip->ipi_send_single(data, cpu);
301 	}
302 	return 0;
303 }
304 
305 /**
306  * ipi_send_single - Send an IPI to a single CPU
307  * @virq:	Linux IRQ number from irq_reserve_ipi()
308  * @cpu:	destination CPU, must in the destination mask passed to
309  *		irq_reserve_ipi()
310  *
311  * Return: %0 on success or negative error number on failure.
312  */
ipi_send_single(unsigned int virq,unsigned int cpu)313 int ipi_send_single(unsigned int virq, unsigned int cpu)
314 {
315 	struct irq_desc *desc = irq_to_desc(virq);
316 	struct irq_data *data = desc ? irq_desc_get_irq_data(desc) : NULL;
317 	struct irq_chip *chip = data ? irq_data_get_irq_chip(data) : NULL;
318 
319 	if (WARN_ON_ONCE(ipi_send_verify(chip, data, NULL, cpu)))
320 		return -EINVAL;
321 
322 	return __ipi_send_single(desc, cpu);
323 }
324 EXPORT_SYMBOL_GPL(ipi_send_single);
325 
326 /**
327  * ipi_send_mask - Send an IPI to target CPU(s)
328  * @virq:	Linux IRQ number from irq_reserve_ipi()
329  * @dest:	dest CPU(s), must be a subset of the mask passed to
330  *		irq_reserve_ipi()
331  *
332  * Return: %0 on success or negative error number on failure.
333  */
ipi_send_mask(unsigned int virq,const struct cpumask * dest)334 int ipi_send_mask(unsigned int virq, const struct cpumask *dest)
335 {
336 	struct irq_desc *desc = irq_to_desc(virq);
337 	struct irq_data *data = desc ? irq_desc_get_irq_data(desc) : NULL;
338 	struct irq_chip *chip = data ? irq_data_get_irq_chip(data) : NULL;
339 
340 	if (WARN_ON_ONCE(ipi_send_verify(chip, data, dest, 0)))
341 		return -EINVAL;
342 
343 	return __ipi_send_mask(desc, dest);
344 }
345 EXPORT_SYMBOL_GPL(ipi_send_mask);
346