52 	struct irq_matrix *m;  in irq_alloc_matrix()  local
54 	m = kzalloc_flex(*m, scratch_map, matrix_size * 2);  in irq_alloc_matrix()
55 	if (!m)  in irq_alloc_matrix()
58 	m->system_map = &m->scratch_map[matrix_size];  in irq_alloc_matrix()
60 	m->matrix_bits = matrix_bits;  in irq_alloc_matrix()
61 	m->alloc_start = alloc_start;  in irq_alloc_matrix()
62 	m->alloc_end = alloc_end;  in irq_alloc_matrix()
63 	m->alloc_size = alloc_end - alloc_start;  in irq_alloc_matrix()
64 	m->maps = __alloc_percpu(struct_size(m->maps, alloc_map, matrix_size * 2),  in irq_alloc_matrix()
65 				 __alignof__(*m->maps));  in irq_alloc_matrix()
66 	if (!m->maps) {  in irq_alloc_matrix()
67 		kfree(m);  in irq_alloc_matrix()
72 		struct cpumap *cm = per_cpu_ptr(m->maps, cpu);  in irq_alloc_matrix()
77 	return m;  in irq_alloc_matrix()
84 void irq_matrix_online(struct irq_matrix *m)  in irq_matrix_online()  argument
86 	struct cpumap *cm = this_cpu_ptr(m->maps);  in irq_matrix_online()
91 		cm->available = m->alloc_size;  in irq_matrix_online()
92 		cm->available -= cm->managed + m->systembits_inalloc;  in irq_matrix_online()
95 	m->global_available += cm->available;  in irq_matrix_online()
97 	m->online_maps++;  in irq_matrix_online()
98 	trace_irq_matrix_online(m);  in irq_matrix_online()
105 void irq_matrix_offline(struct irq_matrix *m)  in irq_matrix_offline()  argument
107 	struct cpumap *cm = this_cpu_ptr(m->maps);  in irq_matrix_offline()
110 	m->global_available -= cm->available;  in irq_matrix_offline()
112 	m->online_maps--;  in irq_matrix_offline()
113 	trace_irq_matrix_offline(m);  in irq_matrix_offline()
116 static unsigned int matrix_alloc_area(struct irq_matrix *m, struct cpumap *cm,  in matrix_alloc_area()  argument
119 	unsigned int area, start = m->alloc_start;  in matrix_alloc_area()
120 	unsigned int end = m->alloc_end;  in matrix_alloc_area()
122 	bitmap_or(m->scratch_map, cm->managed_map, m->system_map, end);  in matrix_alloc_area()
123 	bitmap_or(m->scratch_map, m->scratch_map, cm->alloc_map, end);  in matrix_alloc_area()
124 	area = bitmap_find_next_zero_area(m->scratch_map, end, start, num, 0);  in matrix_alloc_area()
135 static unsigned int matrix_find_best_cpu(struct irq_matrix *m,  in matrix_find_best_cpu()  argument
144 		cm = per_cpu_ptr(m->maps, cpu);  in matrix_find_best_cpu()
156 static unsigned int matrix_find_best_cpu_managed(struct irq_matrix *m,  in matrix_find_best_cpu_managed()  argument
165 		cm = per_cpu_ptr(m->maps, cpu);  in matrix_find_best_cpu_managed()
187 void irq_matrix_assign_system(struct irq_matrix *m, unsigned int bit,  in irq_matrix_assign_system()  argument
190 	struct cpumap *cm = this_cpu_ptr(m->maps);  in irq_matrix_assign_system()
192 	BUG_ON(bit > m->matrix_bits);  in irq_matrix_assign_system()
193 	BUG_ON(m->online_maps > 1 || (m->online_maps && !replace));  in irq_matrix_assign_system()
195 	set_bit(bit, m->system_map);  in irq_matrix_assign_system()
199 		m->total_allocated--;  in irq_matrix_assign_system()
201 	if (bit >= m->alloc_start && bit < m->alloc_end)  in irq_matrix_assign_system()
202 		m->systembits_inalloc++;  in irq_matrix_assign_system()
204 	trace_irq_matrix_assign_system(bit, m);  in irq_matrix_assign_system()
216 int irq_matrix_reserve_managed(struct irq_matrix *m, const struct cpumask *msk)  in irq_matrix_reserve_managed()  argument
221 		struct cpumap *cm = per_cpu_ptr(m->maps, cpu);  in irq_matrix_reserve_managed()
224 		bit = matrix_alloc_area(m, cm, 1, true);  in irq_matrix_reserve_managed()
225 		if (bit >= m->alloc_end)  in irq_matrix_reserve_managed()
230 			m->global_available--;  in irq_matrix_reserve_managed()
232 		trace_irq_matrix_reserve_managed(bit, cpu, m, cm);  in irq_matrix_reserve_managed()
240 		irq_matrix_remove_managed(m, cpumask_of(cpu));  in irq_matrix_reserve_managed()
257 void irq_matrix_remove_managed(struct irq_matrix *m, const struct cpumask *msk)  in irq_matrix_remove_managed()  argument
262 		struct cpumap *cm = per_cpu_ptr(m->maps, cpu);  in irq_matrix_remove_managed()
263 		unsigned int bit, end = m->alloc_end;  in irq_matrix_remove_managed()
269 		bitmap_andnot(m->scratch_map, cm->managed_map, cm->alloc_map, end);  in irq_matrix_remove_managed()
271 		bit = find_first_bit(m->scratch_map, end);  in irq_matrix_remove_managed()
280 			m->global_available++;  in irq_matrix_remove_managed()
282 		trace_irq_matrix_remove_managed(bit, cpu, m, cm);  in irq_matrix_remove_managed()
292 int irq_matrix_alloc_managed(struct irq_matrix *m, const struct cpumask *msk,  in irq_matrix_alloc_managed()  argument
301 	cpu = matrix_find_best_cpu_managed(m, msk);  in irq_matrix_alloc_managed()
305 	cm = per_cpu_ptr(m->maps, cpu);  in irq_matrix_alloc_managed()
306 	end = m->alloc_end;  in irq_matrix_alloc_managed()
308 	bitmap_andnot(m->scratch_map, cm->managed_map, cm->alloc_map, end);  in irq_matrix_alloc_managed()
309 	bit = find_first_bit(m->scratch_map, end);  in irq_matrix_alloc_managed()
315 	m->total_allocated++;  in irq_matrix_alloc_managed()
317 	trace_irq_matrix_alloc_managed(bit, cpu, m, cm);  in irq_matrix_alloc_managed()
328 void irq_matrix_assign(struct irq_matrix *m, unsigned int bit)  in irq_matrix_assign()  argument
330 	struct cpumap *cm = this_cpu_ptr(m->maps);  in irq_matrix_assign()
332 	if (WARN_ON_ONCE(bit < m->alloc_start || bit >= m->alloc_end))  in irq_matrix_assign()
337 	m->total_allocated++;  in irq_matrix_assign()
339 	m->global_available--;  in irq_matrix_assign()
340 	trace_irq_matrix_assign(bit, smp_processor_id(), m, cm);  in irq_matrix_assign()
352 void irq_matrix_reserve(struct irq_matrix *m)  in irq_matrix_reserve()  argument
354 	if (m->global_reserved == m->global_available)  in irq_matrix_reserve()
357 	m->global_reserved++;  in irq_matrix_reserve()
358 	trace_irq_matrix_reserve(m);  in irq_matrix_reserve()
370 void irq_matrix_remove_reserved(struct irq_matrix *m)  in irq_matrix_remove_reserved()  argument
372 	m->global_reserved--;  in irq_matrix_remove_reserved()
373 	trace_irq_matrix_remove_reserved(m);  in irq_matrix_remove_reserved()
383 int irq_matrix_alloc(struct irq_matrix *m, const struct cpumask *msk,  in irq_matrix_alloc()  argument
396 	cpu = matrix_find_best_cpu(m, msk);  in irq_matrix_alloc()
400 	cm = per_cpu_ptr(m->maps, cpu);  in irq_matrix_alloc()
401 	bit = matrix_alloc_area(m, cm, 1, false);  in irq_matrix_alloc()
402 	if (bit >= m->alloc_end)  in irq_matrix_alloc()
406 	m->total_allocated++;  in irq_matrix_alloc()
407 	m->global_available--;  in irq_matrix_alloc()
409 		m->global_reserved--;  in irq_matrix_alloc()
411 	trace_irq_matrix_alloc(bit, cpu, m, cm);  in irq_matrix_alloc()
424 void irq_matrix_free(struct irq_matrix *m, unsigned int cpu,  in irq_matrix_free()  argument
427 	struct cpumap *cm = per_cpu_ptr(m->maps, cpu);  in irq_matrix_free()
429 	if (WARN_ON_ONCE(bit < m->alloc_start || bit >= m->alloc_end))  in irq_matrix_free()
440 		m->total_allocated--;  in irq_matrix_free()
445 			m->global_available++;  in irq_matrix_free()
447 	trace_irq_matrix_free(bit, cpu, m, cm);  in irq_matrix_free()
456 unsigned int irq_matrix_available(struct irq_matrix *m, bool cpudown)  in irq_matrix_available()  argument
458 	struct cpumap *cm = this_cpu_ptr(m->maps);  in irq_matrix_available()
461 		return m->global_available;  in irq_matrix_available()
462 	return m->global_available - cm->available;  in irq_matrix_available()
469 unsigned int irq_matrix_reserved(struct irq_matrix *m)  in irq_matrix_reserved()  argument
471 	return m->global_reserved;  in irq_matrix_reserved()
480 unsigned int irq_matrix_allocated(struct irq_matrix *m)  in irq_matrix_allocated()  argument
482 	struct cpumap *cm = this_cpu_ptr(m->maps);  in irq_matrix_allocated()
496 void irq_matrix_debug_show(struct seq_file *sf, struct irq_matrix *m, int ind)  in irq_matrix_debug_show()  argument
498 	unsigned int nsys = bitmap_weight(m->system_map, m->matrix_bits);  in irq_matrix_debug_show()
501 	seq_printf(sf, "Online bitmaps:   %6u\n", m->online_maps);  in irq_matrix_debug_show()
502 	seq_printf(sf, "Global available: %6u\n", m->global_available);  in irq_matrix_debug_show()
503 	seq_printf(sf, "Global reserved:  %6u\n", m->global_reserved);  in irq_matrix_debug_show()
504 	seq_printf(sf, "Total allocated:  %6u\n", m->total_allocated);  in irq_matrix_debug_show()
505 	seq_printf(sf, "System: %u: %*pbl\n", nsys, m->matrix_bits,  in irq_matrix_debug_show()
506 		   m->system_map);  in irq_matrix_debug_show()
510 		struct cpumap *cm = per_cpu_ptr(m->maps, cpu);  in irq_matrix_debug_show()
515 			   m->matrix_bits, cm->alloc_map);  in irq_matrix_debug_show()