Lines Matching +full:pre +full:- +full:stored
1 /* SPDX-License-Identifier: GPL-2.0 */
14 * objpool: ring-array based lockless MPMC queue
21 * With leveraging percpu ring-array to mitigate hot spots of memory
22 * contention, it delivers near-linear scalability for high parallel
29 * 2) All pre-allocated objects are managed in percpu ring array,
34 * struct objpool_slot - percpu ring array of objpool
41 * Represents a cpu-local array-based ring buffer, its size is specialized
44 * continuous memory: CPU assigned number of objects are stored just after
64 * caller-specified callback for object initial setup, it's only called
69 /* caller-specified cleanup callback for objpool destruction */
73 * struct objpool_head - object pooling metadata
75 * @nr_objs: total objs (to be pre-allocated with objpool)
83 * @context: caller-provided context
102 * objpool_init() - initialize objpool and pre-allocated objects
104 * @nr_objs: total objects to be pre-allocated by this object pool
113 * All pre-allocated objects are to be zeroed after memory allocation.
128 struct objpool_slot *slot = pool->cpu_slots[cpu]; in __objpool_try_get_slot()
130 uint32_t head = smp_load_acquire(&slot->head); in __objpool_try_get_slot()
132 while (head != READ_ONCE(slot->last)) { in __objpool_try_get_slot()
143 * by condition 'last != head && last - head <= nr_objs' in __objpool_try_get_slot()
144 * that is equivalent to 'last - head - 1 < nr_objs' as in __objpool_try_get_slot()
147 if (READ_ONCE(slot->last) - head - 1 >= pool->nr_objs) { in __objpool_try_get_slot()
148 head = READ_ONCE(slot->head); in __objpool_try_get_slot()
153 obj = READ_ONCE(slot->entries[head & slot->mask]); in __objpool_try_get_slot()
156 if (try_cmpxchg_release(&slot->head, &head, head + 1)) in __objpool_try_get_slot()
164 * objpool_pop() - allocate an object from objpool
179 for (i = 0; i < pool->nr_possible_cpus; i++) { in objpool_pop()
183 cpu = cpumask_next_wrap(cpu, cpu_possible_mask, -1, 1); in objpool_pop()
194 struct objpool_slot *slot = pool->cpu_slots[cpu]; in __objpool_try_add_slot()
198 tail = READ_ONCE(slot->tail); in __objpool_try_add_slot()
201 head = READ_ONCE(slot->head); in __objpool_try_add_slot()
203 WARN_ON_ONCE(tail - head > pool->nr_objs); in __objpool_try_add_slot()
204 } while (!try_cmpxchg_acquire(&slot->tail, &tail, tail + 1)); in __objpool_try_add_slot()
207 WRITE_ONCE(slot->entries[tail & slot->mask], obj); in __objpool_try_add_slot()
209 smp_store_release(&slot->last, tail + 1); in __objpool_try_add_slot()
215 * objpool_push() - reclaim the object and return back to objpool
237 * objpool_drop() - discard the object and deref objpool
241 * return: 0 if objpool was released; -EAGAIN if there are still
255 * objpool_free() - release objpool forcely (all objects to be freed)
261 * objpool_fini() - deref object pool (also releasing unused objects)