1 // SPDX-License-Identifier: GPL-2.0
41  * irq_alloc_matrix - Allocate a irq_matrix structure and initialize it
58 	m->system_map = &m->scratch_map[matrix_size];  in irq_alloc_matrix()
60 	m->matrix_bits = matrix_bits;  in irq_alloc_matrix()
61 	m->alloc_start = alloc_start;  in irq_alloc_matrix()
62 	m->alloc_end = alloc_end;  in irq_alloc_matrix()
63 	m->alloc_size = alloc_end - alloc_start;  in irq_alloc_matrix()
64 	m->maps = __alloc_percpu(struct_size(m->maps, alloc_map, matrix_size * 2),  in irq_alloc_matrix()
65 				 __alignof__(*m->maps));  in irq_alloc_matrix()
66 	if (!m->maps) {  in irq_alloc_matrix()
72 		struct cpumap *cm = per_cpu_ptr(m->maps, cpu);  in irq_alloc_matrix()
74 		cm->managed_map = &cm->alloc_map[matrix_size];  in irq_alloc_matrix()
81  * irq_matrix_online - Bring the local CPU matrix online
86 	struct cpumap *cm = this_cpu_ptr(m->maps);  in irq_matrix_online()
88 	BUG_ON(cm->online);  in irq_matrix_online()
90 	if (!cm->initialized) {  in irq_matrix_online()
91 		cm->available = m->alloc_size;  in irq_matrix_online()
92 		cm->available -= cm->managed + m->systembits_inalloc;  in irq_matrix_online()
93 		cm->initialized = true;  in irq_matrix_online()
95 	m->global_available += cm->available;  in irq_matrix_online()
96 	cm->online = true;  in irq_matrix_online()
97 	m->online_maps++;  in irq_matrix_online()
102  * irq_matrix_offline - Bring the local CPU matrix offline
107 	struct cpumap *cm = this_cpu_ptr(m->maps);  in irq_matrix_offline()
110 	m->global_available -= cm->available;  in irq_matrix_offline()
111 	cm->online = false;  in irq_matrix_offline()
112 	m->online_maps--;  in irq_matrix_offline()
119 	unsigned int area, start = m->alloc_start;  in matrix_alloc_area()
120 	unsigned int end = m->alloc_end;  in matrix_alloc_area()
122 	bitmap_or(m->scratch_map, cm->managed_map, m->system_map, end);  in matrix_alloc_area()
123 	bitmap_or(m->scratch_map, m->scratch_map, cm->alloc_map, end);  in matrix_alloc_area()
124 	area = bitmap_find_next_zero_area(m->scratch_map, end, start, num, 0);  in matrix_alloc_area()
128 		bitmap_set(cm->managed_map, area, num);  in matrix_alloc_area()
130 		bitmap_set(cm->alloc_map, area, num);  in matrix_alloc_area()
144 		cm = per_cpu_ptr(m->maps, cpu);  in matrix_find_best_cpu()
146 		if (!cm->online || cm->available <= maxavl)  in matrix_find_best_cpu()
150 		maxavl = cm->available;  in matrix_find_best_cpu()
155 /* Find the best CPU which has the lowest number of managed IRQs allocated */
165 		cm = per_cpu_ptr(m->maps, cpu);  in matrix_find_best_cpu_managed()
167 		if (!cm->online || cm->managed_allocated > allocated)  in matrix_find_best_cpu_managed()
171 		allocated = cm->managed_allocated;  in matrix_find_best_cpu_managed()
177  * irq_matrix_assign_system - Assign system wide entry in the matrix
190 	struct cpumap *cm = this_cpu_ptr(m->maps);  in irq_matrix_assign_system()
192 	BUG_ON(bit > m->matrix_bits);  in irq_matrix_assign_system()
193 	BUG_ON(m->online_maps > 1 || (m->online_maps && !replace));  in irq_matrix_assign_system()
195 	set_bit(bit, m->system_map);  in irq_matrix_assign_system()
197 		BUG_ON(!test_and_clear_bit(bit, cm->alloc_map));  in irq_matrix_assign_system()
198 		cm->allocated--;  in irq_matrix_assign_system()
199 		m->total_allocated--;  in irq_matrix_assign_system()
201 	if (bit >= m->alloc_start && bit < m->alloc_end)  in irq_matrix_assign_system()
202 		m->systembits_inalloc++;  in irq_matrix_assign_system()
208  * irq_matrix_reserve_managed - Reserve a managed interrupt in a CPU map
210  * @msk:	On which CPUs the bits should be reserved.
221 		struct cpumap *cm = per_cpu_ptr(m->maps, cpu);  in irq_matrix_reserve_managed()
225 		if (bit >= m->alloc_end)  in irq_matrix_reserve_managed()
227 		cm->managed++;  in irq_matrix_reserve_managed()
228 		if (cm->online) {  in irq_matrix_reserve_managed()
229 			cm->available--;  in irq_matrix_reserve_managed()
230 			m->global_available--;  in irq_matrix_reserve_managed()
242 	return -ENOSPC;  in irq_matrix_reserve_managed()
246  * irq_matrix_remove_managed - Remove managed interrupts in a CPU map
262 		struct cpumap *cm = per_cpu_ptr(m->maps, cpu);  in irq_matrix_remove_managed()
263 		unsigned int bit, end = m->alloc_end;  in irq_matrix_remove_managed()
265 		if (WARN_ON_ONCE(!cm->managed))  in irq_matrix_remove_managed()
269 		bitmap_andnot(m->scratch_map, cm->managed_map, cm->alloc_map, end);  in irq_matrix_remove_managed()
271 		bit = find_first_bit(m->scratch_map, end);  in irq_matrix_remove_managed()
275 		clear_bit(bit, cm->managed_map);  in irq_matrix_remove_managed()
277 		cm->managed--;  in irq_matrix_remove_managed()
278 		if (cm->online) {  in irq_matrix_remove_managed()
279 			cm->available++;  in irq_matrix_remove_managed()
280 			m->global_available++;  in irq_matrix_remove_managed()
287  * irq_matrix_alloc_managed - Allocate a managed interrupt in a CPU map
299 		return -EINVAL;  in irq_matrix_alloc_managed()
303 		return -ENOSPC;  in irq_matrix_alloc_managed()
305 	cm = per_cpu_ptr(m->maps, cpu);  in irq_matrix_alloc_managed()
306 	end = m->alloc_end;  in irq_matrix_alloc_managed()
308 	bitmap_andnot(m->scratch_map, cm->managed_map, cm->alloc_map, end);  in irq_matrix_alloc_managed()
309 	bit = find_first_bit(m->scratch_map, end);  in irq_matrix_alloc_managed()
311 		return -ENOSPC;  in irq_matrix_alloc_managed()
312 	set_bit(bit, cm->alloc_map);  in irq_matrix_alloc_managed()
313 	cm->allocated++;  in irq_matrix_alloc_managed()
314 	cm->managed_allocated++;  in irq_matrix_alloc_managed()
315 	m->total_allocated++;  in irq_matrix_alloc_managed()
322  * irq_matrix_assign - Assign a preallocated interrupt in the local CPU map
330 	struct cpumap *cm = this_cpu_ptr(m->maps);  in irq_matrix_assign()
332 	if (WARN_ON_ONCE(bit < m->alloc_start || bit >= m->alloc_end))  in irq_matrix_assign()
334 	if (WARN_ON_ONCE(test_and_set_bit(bit, cm->alloc_map)))  in irq_matrix_assign()
336 	cm->allocated++;  in irq_matrix_assign()
337 	m->total_allocated++;  in irq_matrix_assign()
338 	cm->available--;  in irq_matrix_assign()
339 	m->global_available--;  in irq_matrix_assign()
344  * irq_matrix_reserve - Reserve interrupts
348  * reserved interrupt bits w/o actually allocating them. This allows to
354 	if (m->global_reserved == m->global_available)  in irq_matrix_reserve()
357 	m->global_reserved++;  in irq_matrix_reserve()
362  * irq_matrix_remove_reserved - Remove interrupt reservation
366  * reserved interrupt bits. This is used to undo irq_matrix_reserve() when the
372 	m->global_reserved--;  in irq_matrix_remove_reserved()
377  * irq_matrix_alloc - Allocate a regular interrupt in a CPU map
380  * @reserved:	Allocate previously reserved interrupts
384 		     bool reserved, unsigned int *mapped_cpu)  in irq_matrix_alloc()  argument
394 		return -EINVAL;  in irq_matrix_alloc()
398 		return -ENOSPC;  in irq_matrix_alloc()
400 	cm = per_cpu_ptr(m->maps, cpu);  in irq_matrix_alloc()
402 	if (bit >= m->alloc_end)  in irq_matrix_alloc()
403 		return -ENOSPC;  in irq_matrix_alloc()
404 	cm->allocated++;  in irq_matrix_alloc()
405 	cm->available--;  in irq_matrix_alloc()
406 	m->total_allocated++;  in irq_matrix_alloc()
407 	m->global_available--;  in irq_matrix_alloc()
408 	if (reserved)  in irq_matrix_alloc()
409 		m->global_reserved--;  in irq_matrix_alloc()
417  * irq_matrix_free - Free allocated interrupt in the matrix
427 	struct cpumap *cm = per_cpu_ptr(m->maps, cpu);  in irq_matrix_free()
429 	if (WARN_ON_ONCE(bit < m->alloc_start || bit >= m->alloc_end))  in irq_matrix_free()
432 	if (WARN_ON_ONCE(!test_and_clear_bit(bit, cm->alloc_map)))  in irq_matrix_free()
435 	cm->allocated--;  in irq_matrix_free()
437 		cm->managed_allocated--;  in irq_matrix_free()
439 	if (cm->online)  in irq_matrix_free()
440 		m->total_allocated--;  in irq_matrix_free()
443 		cm->available++;  in irq_matrix_free()
444 		if (cm->online)  in irq_matrix_free()
445 			m->global_available++;  in irq_matrix_free()
451  * irq_matrix_available - Get the number of globally available irqs
454  *		the number of available irqs accordingly
458 	struct cpumap *cm = this_cpu_ptr(m->maps);  in irq_matrix_available()
461 		return m->global_available;  in irq_matrix_available()
462 	return m->global_available - cm->available;  in irq_matrix_available()
466  * irq_matrix_reserved - Get the number of globally reserved irqs
471 	return m->global_reserved;  in irq_matrix_reserved()
475  * irq_matrix_allocated - Get the number of allocated non-managed irqs on the local CPU
478  * This returns number of allocated non-managed interrupts.
482 	struct cpumap *cm = this_cpu_ptr(m->maps);  in irq_matrix_allocated()
484 	return cm->allocated - cm->managed_allocated;  in irq_matrix_allocated()
489  * irq_matrix_debug_show - Show detailed allocation information
498 	unsigned int nsys = bitmap_weight(m->system_map, m->matrix_bits);  in irq_matrix_debug_show()
501 	seq_printf(sf, "Online bitmaps:   %6u\n", m->online_maps);  in irq_matrix_debug_show()
502 	seq_printf(sf, "Global available: %6u\n", m->global_available);  in irq_matrix_debug_show()
503 	seq_printf(sf, "Global reserved:  %6u\n", m->global_reserved);  in irq_matrix_debug_show()
504 	seq_printf(sf, "Total allocated:  %6u\n", m->total_allocated);  in irq_matrix_debug_show()
505 	seq_printf(sf, "System: %u: %*pbl\n", nsys, m->matrix_bits,  in irq_matrix_debug_show()
506 		   m->system_map);  in irq_matrix_debug_show()
510 		struct cpumap *cm = per_cpu_ptr(m->maps, cpu);  in irq_matrix_debug_show()
513 			   cpu, cm->available, cm->managed,  in irq_matrix_debug_show()
514 			   cm->managed_allocated, cm->allocated,  in irq_matrix_debug_show()
515 			   m->matrix_bits, cm->alloc_map);  in irq_matrix_debug_show()