50 	struct ck_fifo_spsc_entry *tail;  member
110 	fifo->head = fifo->tail = fifo->head_snapshot = fifo->garbage = stub;  in ck_fifo_spsc_init()
119 	fifo->head = fifo->tail = NULL;  in ck_fifo_spsc_deinit()
134 	ck_pr_store_ptr(&fifo->tail->next, entry);  in ck_fifo_spsc_enqueue()
135 	fifo->tail = entry;  in ck_fifo_spsc_enqueue()
220 	struct ck_fifo_mpmc_pointer tail;  member
230 	fifo->head.pointer = fifo->tail.pointer = stub;  in ck_fifo_mpmc_init()
231 	fifo->head.generation = fifo->tail.generation = NULL;  in ck_fifo_mpmc_init()
240 	fifo->head.pointer = fifo->tail.pointer = NULL;  in ck_fifo_mpmc_deinit()
249 	struct ck_fifo_mpmc_pointer tail, next, update;  in ck_fifo_mpmc_enqueue()  local
261 		tail.generation = ck_pr_load_ptr(&fifo->tail.generation);  in ck_fifo_mpmc_enqueue()
263 		tail.pointer = ck_pr_load_ptr(&fifo->tail.pointer);  in ck_fifo_mpmc_enqueue()
264 		next.generation = ck_pr_load_ptr(&tail.pointer->next.generation);  in ck_fifo_mpmc_enqueue()
266 		next.pointer = ck_pr_load_ptr(&tail.pointer->next.pointer);  in ck_fifo_mpmc_enqueue()
268 		if (ck_pr_load_ptr(&fifo->tail.generation) != tail.generation)  in ck_fifo_mpmc_enqueue()
273 			 * If the tail pointer has an entry following it then  in ck_fifo_mpmc_enqueue()
279 			update.generation = tail.generation + 1;  in ck_fifo_mpmc_enqueue()
280 			ck_pr_cas_ptr_2(&fifo->tail, &tail, &update);  in ck_fifo_mpmc_enqueue()
284 			 * current tail.  in ck_fifo_mpmc_enqueue()
288 			if (ck_pr_cas_ptr_2(&tail.pointer->next, &next, &update) == true)  in ck_fifo_mpmc_enqueue()
295 	/* After a successful insert, forward the tail to the new entry. */  in ck_fifo_mpmc_enqueue()
296 	update.generation = tail.generation + 1;  in ck_fifo_mpmc_enqueue()
297 	ck_pr_cas_ptr_2(&fifo->tail, &tail, &update);  in ck_fifo_mpmc_enqueue()
306 	struct ck_fifo_mpmc_pointer tail, next, update;  in ck_fifo_mpmc_tryenqueue()  local
314 	tail.generation = ck_pr_load_ptr(&fifo->tail.generation);  in ck_fifo_mpmc_tryenqueue()
316 	tail.pointer = ck_pr_load_ptr(&fifo->tail.pointer);  in ck_fifo_mpmc_tryenqueue()
317 	next.generation = ck_pr_load_ptr(&tail.pointer->next.generation);  in ck_fifo_mpmc_tryenqueue()
319 	next.pointer = ck_pr_load_ptr(&tail.pointer->next.pointer);  in ck_fifo_mpmc_tryenqueue()
321 	if (ck_pr_load_ptr(&fifo->tail.generation) != tail.generation)  in ck_fifo_mpmc_tryenqueue()
326 		 * If the tail pointer has an entry following it then  in ck_fifo_mpmc_tryenqueue()
332 		update.generation = tail.generation + 1;  in ck_fifo_mpmc_tryenqueue()
333 		ck_pr_cas_ptr_2(&fifo->tail, &tail, &update);  in ck_fifo_mpmc_tryenqueue()
338 		 * current tail.  in ck_fifo_mpmc_tryenqueue()
342 		if (ck_pr_cas_ptr_2(&tail.pointer->next, &next, &update) == false)  in ck_fifo_mpmc_tryenqueue()
348 	/* After a successful insert, forward the tail to the new entry. */  in ck_fifo_mpmc_tryenqueue()
349 	update.generation = tail.generation + 1;  in ck_fifo_mpmc_tryenqueue()
350 	ck_pr_cas_ptr_2(&fifo->tail, &tail, &update);  in ck_fifo_mpmc_tryenqueue()
359 	struct ck_fifo_mpmc_pointer head, tail, next, update;  in ck_fifo_mpmc_dequeue()  local
365 		tail.generation = ck_pr_load_ptr(&fifo->tail.generation);  in ck_fifo_mpmc_dequeue()
367 		tail.pointer = ck_pr_load_ptr(&fifo->tail.pointer);  in ck_fifo_mpmc_dequeue()
374 		if (head.pointer == tail.pointer) {  in ck_fifo_mpmc_dequeue()
383 			/* Forward the tail pointer if necessary. */  in ck_fifo_mpmc_dequeue()
384 			update.generation = tail.generation + 1;  in ck_fifo_mpmc_dequeue()
385 			ck_pr_cas_ptr_2(&fifo->tail, &tail, &update);  in ck_fifo_mpmc_dequeue()
414 	struct ck_fifo_mpmc_pointer head, tail, next, update;  in ck_fifo_mpmc_trydequeue()  local
420 	tail.generation = ck_pr_load_ptr(&fifo->tail.generation);  in ck_fifo_mpmc_trydequeue()
422 	tail.pointer = ck_pr_load_ptr(&fifo->tail.pointer);  in ck_fifo_mpmc_trydequeue()
429 	if (head.pointer == tail.pointer) {  in ck_fifo_mpmc_trydequeue()
438 		/* Forward the tail pointer if necessary. */  in ck_fifo_mpmc_trydequeue()
439 		update.generation = tail.generation + 1;  in ck_fifo_mpmc_trydequeue()
440 		ck_pr_cas_ptr_2(&fifo->tail, &tail, &update);  in ck_fifo_mpmc_trydequeue()