57 ck_fifo_spsc_enqueue_trylock(struct ck_fifo_spsc *fifo)  in ck_fifo_spsc_enqueue_trylock()  argument
60 	return ck_spinlock_trylock(&fifo->m_tail);  in ck_fifo_spsc_enqueue_trylock()
64 ck_fifo_spsc_enqueue_lock(struct ck_fifo_spsc *fifo)  in ck_fifo_spsc_enqueue_lock()  argument
67 	ck_spinlock_lock(&fifo->m_tail);  in ck_fifo_spsc_enqueue_lock()
72 ck_fifo_spsc_enqueue_unlock(struct ck_fifo_spsc *fifo)  in ck_fifo_spsc_enqueue_unlock()  argument
75 	ck_spinlock_unlock(&fifo->m_tail);  in ck_fifo_spsc_enqueue_unlock()
80 ck_fifo_spsc_dequeue_trylock(struct ck_fifo_spsc *fifo)  in ck_fifo_spsc_dequeue_trylock()  argument
83 	return ck_spinlock_trylock(&fifo->m_head);  in ck_fifo_spsc_dequeue_trylock()
87 ck_fifo_spsc_dequeue_lock(struct ck_fifo_spsc *fifo)  in ck_fifo_spsc_dequeue_lock()  argument
90 	ck_spinlock_lock(&fifo->m_head);  in ck_fifo_spsc_dequeue_lock()
95 ck_fifo_spsc_dequeue_unlock(struct ck_fifo_spsc *fifo)  in ck_fifo_spsc_dequeue_unlock()  argument
98 	ck_spinlock_unlock(&fifo->m_head);  in ck_fifo_spsc_dequeue_unlock()
103 ck_fifo_spsc_init(struct ck_fifo_spsc *fifo, struct ck_fifo_spsc_entry *stub)  in ck_fifo_spsc_init()  argument
106 	ck_spinlock_init(&fifo->m_head);  in ck_fifo_spsc_init()
107 	ck_spinlock_init(&fifo->m_tail);  in ck_fifo_spsc_init()
110 	fifo->head = fifo->tail = fifo->head_snapshot = fifo->garbage = stub;  in ck_fifo_spsc_init()
115 ck_fifo_spsc_deinit(struct ck_fifo_spsc *fifo, struct ck_fifo_spsc_entry **garbage)  in ck_fifo_spsc_deinit()  argument
118 	*garbage = fifo->garbage;  in ck_fifo_spsc_deinit()
119 	fifo->head = fifo->tail = NULL;  in ck_fifo_spsc_deinit()
124 ck_fifo_spsc_enqueue(struct ck_fifo_spsc *fifo,  in ck_fifo_spsc_enqueue()  argument
134 	ck_pr_store_ptr(&fifo->tail->next, entry);  in ck_fifo_spsc_enqueue()
135 	fifo->tail = entry;  in ck_fifo_spsc_enqueue()
140 ck_fifo_spsc_dequeue(struct ck_fifo_spsc *fifo, void *value)  in ck_fifo_spsc_dequeue()  argument
149 	entry = ck_pr_load_ptr(&fifo->head->next);  in ck_fifo_spsc_dequeue()
156 	ck_pr_store_ptr(&fifo->head, entry);  in ck_fifo_spsc_dequeue()
165 ck_fifo_spsc_recycle(struct ck_fifo_spsc *fifo)  in ck_fifo_spsc_recycle()  argument
169 	if (fifo->head_snapshot == fifo->garbage) {  in ck_fifo_spsc_recycle()
170 		fifo->head_snapshot = ck_pr_load_ptr(&fifo->head);  in ck_fifo_spsc_recycle()
171 		if (fifo->head_snapshot == fifo->garbage)  in ck_fifo_spsc_recycle()
175 	garbage = fifo->garbage;  in ck_fifo_spsc_recycle()
176 	fifo->garbage = garbage->next;  in ck_fifo_spsc_recycle()
181 ck_fifo_spsc_isempty(struct ck_fifo_spsc *fifo)  in ck_fifo_spsc_isempty()  argument
183 	struct ck_fifo_spsc_entry *head = ck_pr_load_ptr(&fifo->head);  in ck_fifo_spsc_isempty()
191 #define CK_FIFO_SPSC_FOREACH(fifo, entry)			\  argument
192 	for ((entry) = CK_FIFO_SPSC_FIRST(fifo);		\
195 #define CK_FIFO_SPSC_FOREACH_SAFE(fifo, entry, T)		\  argument
196 	for ((entry) = CK_FIFO_SPSC_FIRST(fifo);		\
225 ck_fifo_mpmc_init(struct ck_fifo_mpmc *fifo, struct ck_fifo_mpmc_entry *stub)  in ck_fifo_mpmc_init()  argument
230 	fifo->head.pointer = fifo->tail.pointer = stub;  in ck_fifo_mpmc_init()
231 	fifo->head.generation = fifo->tail.generation = NULL;  in ck_fifo_mpmc_init()
236 ck_fifo_mpmc_deinit(struct ck_fifo_mpmc *fifo, struct ck_fifo_mpmc_entry **garbage)  in ck_fifo_mpmc_deinit()  argument
239 	*garbage = fifo->head.pointer;  in ck_fifo_mpmc_deinit()
240 	fifo->head.pointer = fifo->tail.pointer = NULL;  in ck_fifo_mpmc_deinit()
245 ck_fifo_mpmc_enqueue(struct ck_fifo_mpmc *fifo,  in ck_fifo_mpmc_enqueue()  argument
261 		tail.generation = ck_pr_load_ptr(&fifo->tail.generation);  in ck_fifo_mpmc_enqueue()
263 		tail.pointer = ck_pr_load_ptr(&fifo->tail.pointer);  in ck_fifo_mpmc_enqueue()
268 		if (ck_pr_load_ptr(&fifo->tail.generation) != tail.generation)  in ck_fifo_mpmc_enqueue()
280 			ck_pr_cas_ptr_2(&fifo->tail, &tail, &update);  in ck_fifo_mpmc_enqueue()
297 	ck_pr_cas_ptr_2(&fifo->tail, &tail, &update);  in ck_fifo_mpmc_enqueue()
302 ck_fifo_mpmc_tryenqueue(struct ck_fifo_mpmc *fifo,  in ck_fifo_mpmc_tryenqueue()  argument
314 	tail.generation = ck_pr_load_ptr(&fifo->tail.generation);  in ck_fifo_mpmc_tryenqueue()
316 	tail.pointer = ck_pr_load_ptr(&fifo->tail.pointer);  in ck_fifo_mpmc_tryenqueue()
321 	if (ck_pr_load_ptr(&fifo->tail.generation) != tail.generation)  in ck_fifo_mpmc_tryenqueue()
333 		ck_pr_cas_ptr_2(&fifo->tail, &tail, &update);  in ck_fifo_mpmc_tryenqueue()
350 	ck_pr_cas_ptr_2(&fifo->tail, &tail, &update);  in ck_fifo_mpmc_tryenqueue()
355 ck_fifo_mpmc_dequeue(struct ck_fifo_mpmc *fifo,  in ck_fifo_mpmc_dequeue()  argument
362 		head.generation = ck_pr_load_ptr(&fifo->head.generation);  in ck_fifo_mpmc_dequeue()
364 		head.pointer = ck_pr_load_ptr(&fifo->head.pointer);  in ck_fifo_mpmc_dequeue()
365 		tail.generation = ck_pr_load_ptr(&fifo->tail.generation);  in ck_fifo_mpmc_dequeue()
367 		tail.pointer = ck_pr_load_ptr(&fifo->tail.pointer);  in ck_fifo_mpmc_dequeue()
385 			ck_pr_cas_ptr_2(&fifo->tail, &tail, &update);  in ck_fifo_mpmc_dequeue()
400 			if (ck_pr_cas_ptr_2(&fifo->head, &head, &update) == true)  in ck_fifo_mpmc_dequeue()
410 ck_fifo_mpmc_trydequeue(struct ck_fifo_mpmc *fifo,  in ck_fifo_mpmc_trydequeue()  argument
416 	head.generation = ck_pr_load_ptr(&fifo->head.generation);  in ck_fifo_mpmc_trydequeue()
418 	head.pointer = ck_pr_load_ptr(&fifo->head.pointer);  in ck_fifo_mpmc_trydequeue()
420 	tail.generation = ck_pr_load_ptr(&fifo->tail.generation);  in ck_fifo_mpmc_trydequeue()
422 	tail.pointer = ck_pr_load_ptr(&fifo->tail.pointer);  in ck_fifo_mpmc_trydequeue()
440 		ck_pr_cas_ptr_2(&fifo->tail, &tail, &update);  in ck_fifo_mpmc_trydequeue()
455 		if (ck_pr_cas_ptr_2(&fifo->head, &head, &update) == false)  in ck_fifo_mpmc_trydequeue()
466 #define CK_FIFO_MPMC_FOREACH(fifo, entry)				\  argument
467 	for ((entry) = CK_FIFO_MPMC_FIRST(fifo);			\
470 #define CK_FIFO_MPMC_FOREACH_SAFE(fifo, entry, T)			\  argument
471 	for ((entry) = CK_FIFO_MPMC_FIRST(fifo);			\