1 // SPDX-License-Identifier: GPL-2.0
12  * objpool: ring-array based lockless MPMC/FIFO queues
24 	void *obj = (void *)&slot->entries[pool->capacity];  in objpool_init_percpu_slot()
28 	slot->mask = pool->capacity - 1;  in objpool_init_percpu_slot()
36 		slot->entries[slot->tail & slot->mask] = obj;  in objpool_init_percpu_slot()
37 		obj = obj + pool->obj_size;  in objpool_init_percpu_slot()
38 		slot->tail++;  in objpool_init_percpu_slot()
39 		slot->last = slot->tail;  in objpool_init_percpu_slot()
40 		pool->nr_objs++;  in objpool_init_percpu_slot()
58 		/* skip the cpu node which could never be present */  in objpool_init_percpu_slots()
63 		nodes = nr_objs / pool->nr_possible_cpus;  in objpool_init_percpu_slots()
64 		if (cpu_count < (nr_objs % pool->nr_possible_cpus))  in objpool_init_percpu_slots()
68 		size = struct_size(slot, entries, pool->capacity) +  in objpool_init_percpu_slots()
69 			pool->obj_size * nodes;  in objpool_init_percpu_slots()
72 		 * here we allocate percpu-slot & objs together in a single  in objpool_init_percpu_slots()
83 		if ((pool->gfp & (GFP_ATOMIC | GFP_KERNEL)) != GFP_ATOMIC)  in objpool_init_percpu_slots()
84 			slot = __vmalloc_node(size, sizeof(void *), pool->gfp,  in objpool_init_percpu_slots()
88 			slot = kmalloc_node(size, pool->gfp, cpu_to_node(i));  in objpool_init_percpu_slots()
90 				return -ENOMEM;  in objpool_init_percpu_slots()
93 		pool->cpu_slots[i] = slot;  in objpool_init_percpu_slots()
95 		/* initialize the objpool_slot of cpu node i */  in objpool_init_percpu_slots()
109 	if (!pool->cpu_slots)  in objpool_fini_percpu_slots()
113 		kvfree(pool->cpu_slots[i]);  in objpool_fini_percpu_slots()
114 	kfree(pool->cpu_slots);  in objpool_fini_percpu_slots()
117 /* initialize object pool and pre-allocate objects */
122 	int rc, capacity, slot_size;  in objpool_init()  local
127 		return -EINVAL;  in objpool_init()
132 	/* calculate capacity of percpu objpool_slot */  in objpool_init()
133 	capacity = roundup_pow_of_two(nr_objs);  in objpool_init()
134 	if (!capacity)  in objpool_init()
135 		return -EINVAL;  in objpool_init()
139 	pool->nr_possible_cpus = num_possible_cpus();  in objpool_init()
140 	pool->obj_size = object_size;  in objpool_init()
141 	pool->capacity = capacity;  in objpool_init()
142 	pool->gfp = gfp & ~__GFP_ZERO;  in objpool_init()
143 	pool->context = context;  in objpool_init()
144 	pool->release = release;  in objpool_init()
146 	pool->cpu_slots = kzalloc(slot_size, pool->gfp);  in objpool_init()
147 	if (!pool->cpu_slots)  in objpool_init()
148 		return -ENOMEM;  in objpool_init()
150 	/* initialize per-cpu slots */  in objpool_init()
155 		refcount_set(&pool->ref, pool->nr_objs + 1);  in objpool_init()
164 	if (!pool->cpu_slots)  in objpool_free()
171 	if (pool->release)  in objpool_free()
172 		pool->release(pool, pool->context);  in objpool_free()
180 		return -EINVAL;  in objpool_drop()
182 	if (refcount_dec_and_test(&pool->ref)) {  in objpool_drop()
187 	return -EAGAIN;  in objpool_drop()
200 	if (refcount_sub_and_test(count, &pool->ref))  in objpool_fini()