1 // SPDX-License-Identifier: GPL-2.0-only
3  * cpu_rmap.c: CPU affinity reverse-map support
13  * objects with CPU affinities.  This can be seen as a reverse-map of
14  * CPU affinity.  However, we do not assume that the object affinities
17  * CPU topology.
21  * alloc_cpu_rmap - allocate CPU affinity reverse-map
28 	unsigned int cpu;  in alloc_cpu_rmap()  local
39 	rmap = kzalloc(obj_offset + size * sizeof(rmap->obj[0]), flags);  in alloc_cpu_rmap()
43 	kref_init(&rmap->refcount);  in alloc_cpu_rmap()
44 	rmap->obj = (void **)((char *)rmap + obj_offset);  in alloc_cpu_rmap()
50 	 * any newly-hotplugged CPUs to have some object assigned.  in alloc_cpu_rmap()
52 	for_each_possible_cpu(cpu) {  in alloc_cpu_rmap()
53 		rmap->near[cpu].index = cpu % size;  in alloc_cpu_rmap()
54 		rmap->near[cpu].dist = CPU_RMAP_DIST_INF;  in alloc_cpu_rmap()
57 	rmap->size = size;  in alloc_cpu_rmap()
63  * cpu_rmap_release - internal reclaiming helper called from kref_put
73  * cpu_rmap_get - internal helper to get new ref on a cpu_rmap
74  * @rmap: reverse-map allocated with alloc_cpu_rmap()
78 	kref_get(&rmap->refcount);  in cpu_rmap_get()
82  * cpu_rmap_put - release ref on a cpu_rmap
83  * @rmap: reverse-map allocated with alloc_cpu_rmap()
87 	return kref_put(&rmap->refcount, cpu_rmap_release);  in cpu_rmap_put()
91 /* Reevaluate nearest object for given CPU, comparing with the given
94 static bool cpu_rmap_copy_neigh(struct cpu_rmap *rmap, unsigned int cpu,  in cpu_rmap_copy_neigh()  argument
100 		if (rmap->near[cpu].dist > dist &&  in cpu_rmap_copy_neigh()
101 		    rmap->near[neigh].dist <= dist) {  in cpu_rmap_copy_neigh()
102 			rmap->near[cpu].index = rmap->near[neigh].index;  in cpu_rmap_copy_neigh()
103 			rmap->near[cpu].dist = dist;  in cpu_rmap_copy_neigh()
114 	unsigned int cpu;  in debug_print_rmap()  local
118 	for_each_possible_cpu(cpu) {  in debug_print_rmap()
119 		index = rmap->near[cpu].index;  in debug_print_rmap()
120 		pr_info("cpu %d -> obj %u (distance %u)\n",  in debug_print_rmap()
121 			cpu, index, rmap->near[cpu].dist);  in debug_print_rmap()
135 	for (i = 0; i < rmap->size; i++)  in get_free_index()
136 		if (!rmap->obj[i])  in get_free_index()
139 	return -ENOSPC;  in get_free_index()
143  * cpu_rmap_add - add object to a rmap
144  * @rmap: CPU rmap allocated with alloc_cpu_rmap()
147  * Return index of object or -ENOSPC if no free entry was found
156 	rmap->obj[index] = obj;  in cpu_rmap_add()
162  * cpu_rmap_update - update CPU rmap following a change of object affinity
163  * @rmap: CPU rmap to update
165  * @affinity: New CPU affinity of object
171 	unsigned int cpu;  in cpu_rmap_update()  local
174 		return -ENOMEM;  in cpu_rmap_update()
179 	for_each_online_cpu(cpu) {  in cpu_rmap_update()
180 		if (rmap->near[cpu].index == index) {  in cpu_rmap_update()
181 			rmap->near[cpu].dist = CPU_RMAP_DIST_INF;  in cpu_rmap_update()
182 			cpumask_set_cpu(cpu, update_mask);  in cpu_rmap_update()
191 	for_each_cpu(cpu, affinity) {  in cpu_rmap_update()
192 		rmap->near[cpu].index = index;  in cpu_rmap_update()
193 		rmap->near[cpu].dist = 0;  in cpu_rmap_update()
195 			   cpumask_of_node(cpu_to_node(cpu)));  in cpu_rmap_update()
201 	for_each_cpu(cpu, update_mask) {  in cpu_rmap_update()
202 		if (cpu_rmap_copy_neigh(rmap, cpu,  in cpu_rmap_update()
203 					topology_sibling_cpumask(cpu), 1))  in cpu_rmap_update()
205 		if (cpu_rmap_copy_neigh(rmap, cpu,  in cpu_rmap_update()
206 					topology_core_cpumask(cpu), 2))  in cpu_rmap_update()
208 		if (cpu_rmap_copy_neigh(rmap, cpu,  in cpu_rmap_update()
209 					cpumask_of_node(cpu_to_node(cpu)), 3))  in cpu_rmap_update()
223 /* Glue between IRQ affinity notifiers and CPU rmaps */
232  * free_irq_cpu_rmap - free a CPU affinity reverse-map used for IRQs
233  * @rmap: Reverse-map allocated with alloc_irq_cpu_map(), or %NULL
245 	for (index = 0; index < rmap->size; index++) {  in free_irq_cpu_rmap()
246 		glue = rmap->obj[index];  in free_irq_cpu_rmap()
248 			irq_set_affinity_notifier(glue->notify.irq, NULL);  in free_irq_cpu_rmap()
256  * irq_cpu_rmap_notify - callback for IRQ subsystem when IRQ affinity updated
258  * @mask: cpu mask for new SMP affinity
269 	rc = cpu_rmap_update(glue->rmap, glue->index, mask);  in irq_cpu_rmap_notify()
275  * irq_cpu_rmap_release - reclaiming callback for IRQ subsystem
283 	glue->rmap->obj[glue->index] = NULL;  in irq_cpu_rmap_release()
284 	cpu_rmap_put(glue->rmap);  in irq_cpu_rmap_release()
289  * irq_cpu_rmap_remove - remove an IRQ from a CPU affinity reverse-map
290  * @rmap: The reverse-map
300  * irq_cpu_rmap_add - add an IRQ to a CPU affinity reverse-map
301  * @rmap: The reverse-map
304  * This adds an IRQ affinity notifier that will update the reverse-map
316 		return -ENOMEM;  in irq_cpu_rmap_add()
317 	glue->notify.notify = irq_cpu_rmap_notify;  in irq_cpu_rmap_add()
318 	glue->notify.release = irq_cpu_rmap_release;  in irq_cpu_rmap_add()
319 	glue->rmap = rmap;  in irq_cpu_rmap_add()
325 	glue->index = rc;  in irq_cpu_rmap_add()
326 	rc = irq_set_affinity_notifier(irq, &glue->notify);  in irq_cpu_rmap_add()
333 	rmap->obj[glue->index] = NULL;  in irq_cpu_rmap_add()
335 	cpu_rmap_put(glue->rmap);  in irq_cpu_rmap_add()