Lines Matching +full:t +full:- +full:head
2 * Copyright 2010-2015 Samy Al Bahra.
47 struct ck_fifo_spsc_entry *head; member
48 char pad[CK_MD_CACHELINE - sizeof(struct ck_fifo_spsc_entry *) - sizeof(ck_spinlock_t)];
60 return ck_spinlock_trylock(&fifo->m_tail); in ck_fifo_spsc_enqueue_trylock()
67 ck_spinlock_lock(&fifo->m_tail); in ck_fifo_spsc_enqueue_lock()
75 ck_spinlock_unlock(&fifo->m_tail); in ck_fifo_spsc_enqueue_unlock()
83 return ck_spinlock_trylock(&fifo->m_head); in ck_fifo_spsc_dequeue_trylock()
90 ck_spinlock_lock(&fifo->m_head); in ck_fifo_spsc_dequeue_lock()
98 ck_spinlock_unlock(&fifo->m_head); in ck_fifo_spsc_dequeue_unlock()
106 ck_spinlock_init(&fifo->m_head); in ck_fifo_spsc_init()
107 ck_spinlock_init(&fifo->m_tail); in ck_fifo_spsc_init()
109 stub->next = NULL; in ck_fifo_spsc_init()
110 fifo->head = fifo->tail = fifo->head_snapshot = fifo->garbage = stub; in ck_fifo_spsc_init()
118 *garbage = fifo->garbage; in ck_fifo_spsc_deinit()
119 fifo->head = fifo->tail = NULL; in ck_fifo_spsc_deinit()
129 entry->value = value; in ck_fifo_spsc_enqueue()
130 entry->next = NULL; in ck_fifo_spsc_enqueue()
132 /* If stub->next is visible, guarantee that entry is consistent. */ in ck_fifo_spsc_enqueue()
134 ck_pr_store_ptr(&fifo->tail->next, entry); in ck_fifo_spsc_enqueue()
135 fifo->tail = entry; in ck_fifo_spsc_enqueue()
145 * The head pointer is guaranteed to always point to a stub entry. in ck_fifo_spsc_dequeue()
149 entry = ck_pr_load_ptr(&fifo->head->next); in ck_fifo_spsc_dequeue()
154 ck_pr_store_ptr_unsafe(value, entry->value); in ck_fifo_spsc_dequeue()
156 ck_pr_store_ptr(&fifo->head, entry); in ck_fifo_spsc_dequeue()
169 if (fifo->head_snapshot == fifo->garbage) { in ck_fifo_spsc_recycle()
170 fifo->head_snapshot = ck_pr_load_ptr(&fifo->head); in ck_fifo_spsc_recycle()
171 if (fifo->head_snapshot == fifo->garbage) in ck_fifo_spsc_recycle()
175 garbage = fifo->garbage; in ck_fifo_spsc_recycle()
176 fifo->garbage = garbage->next; in ck_fifo_spsc_recycle()
183 struct ck_fifo_spsc_entry *head = ck_pr_load_ptr(&fifo->head); in ck_fifo_spsc_isempty() local
184 return ck_pr_load_ptr(&head->next) == NULL; in ck_fifo_spsc_isempty()
187 #define CK_FIFO_SPSC_ISEMPTY(f) ((f)->head->next == NULL)
188 #define CK_FIFO_SPSC_FIRST(f) ((f)->head->next)
189 #define CK_FIFO_SPSC_NEXT(m) ((m)->next)
190 #define CK_FIFO_SPSC_SPARE(f) ((f)->head)
195 #define CK_FIFO_SPSC_FOREACH_SAFE(fifo, entry, T) \ argument
197 (entry) != NULL && ((T) = (entry)->next, 1); \
198 (entry) = (T))
218 struct ck_fifo_mpmc_pointer head; member
219 char pad[CK_MD_CACHELINE - sizeof(struct ck_fifo_mpmc_pointer)];
228 stub->next.pointer = NULL; in ck_fifo_mpmc_init()
229 stub->next.generation = NULL; in ck_fifo_mpmc_init()
230 fifo->head.pointer = fifo->tail.pointer = stub; in ck_fifo_mpmc_init()
231 fifo->head.generation = fifo->tail.generation = NULL; in ck_fifo_mpmc_init()
239 *garbage = fifo->head.pointer; in ck_fifo_mpmc_deinit()
240 fifo->head.pointer = fifo->tail.pointer = NULL; in ck_fifo_mpmc_deinit()
255 entry->value = value; in ck_fifo_mpmc_enqueue()
256 entry->next.pointer = NULL; in ck_fifo_mpmc_enqueue()
257 entry->next.generation = 0; in ck_fifo_mpmc_enqueue()
261 tail.generation = ck_pr_load_ptr(&fifo->tail.generation); in ck_fifo_mpmc_enqueue()
263 tail.pointer = ck_pr_load_ptr(&fifo->tail.pointer); in ck_fifo_mpmc_enqueue()
264 next.generation = ck_pr_load_ptr(&tail.pointer->next.generation); in ck_fifo_mpmc_enqueue()
266 next.pointer = ck_pr_load_ptr(&tail.pointer->next.pointer); in ck_fifo_mpmc_enqueue()
268 if (ck_pr_load_ptr(&fifo->tail.generation) != tail.generation) in ck_fifo_mpmc_enqueue()
280 ck_pr_cas_ptr_2(&fifo->tail, &tail, &update); in ck_fifo_mpmc_enqueue()
288 if (ck_pr_cas_ptr_2(&tail.pointer->next, &next, &update) == true) in ck_fifo_mpmc_enqueue()
297 ck_pr_cas_ptr_2(&fifo->tail, &tail, &update); in ck_fifo_mpmc_enqueue()
308 entry->value = value; in ck_fifo_mpmc_tryenqueue()
309 entry->next.pointer = NULL; in ck_fifo_mpmc_tryenqueue()
310 entry->next.generation = 0; in ck_fifo_mpmc_tryenqueue()
314 tail.generation = ck_pr_load_ptr(&fifo->tail.generation); in ck_fifo_mpmc_tryenqueue()
316 tail.pointer = ck_pr_load_ptr(&fifo->tail.pointer); in ck_fifo_mpmc_tryenqueue()
317 next.generation = ck_pr_load_ptr(&tail.pointer->next.generation); in ck_fifo_mpmc_tryenqueue()
319 next.pointer = ck_pr_load_ptr(&tail.pointer->next.pointer); in ck_fifo_mpmc_tryenqueue()
321 if (ck_pr_load_ptr(&fifo->tail.generation) != tail.generation) in ck_fifo_mpmc_tryenqueue()
333 ck_pr_cas_ptr_2(&fifo->tail, &tail, &update); in ck_fifo_mpmc_tryenqueue()
342 if (ck_pr_cas_ptr_2(&tail.pointer->next, &next, &update) == false) in ck_fifo_mpmc_tryenqueue()
350 ck_pr_cas_ptr_2(&fifo->tail, &tail, &update); in ck_fifo_mpmc_tryenqueue()
359 struct ck_fifo_mpmc_pointer head, tail, next, update; in ck_fifo_mpmc_dequeue() local
362 head.generation = ck_pr_load_ptr(&fifo->head.generation); in ck_fifo_mpmc_dequeue()
364 head.pointer = ck_pr_load_ptr(&fifo->head.pointer); in ck_fifo_mpmc_dequeue()
365 tail.generation = ck_pr_load_ptr(&fifo->tail.generation); in ck_fifo_mpmc_dequeue()
367 tail.pointer = ck_pr_load_ptr(&fifo->tail.pointer); in ck_fifo_mpmc_dequeue()
369 next.generation = ck_pr_load_ptr(&head.pointer->next.generation); in ck_fifo_mpmc_dequeue()
371 next.pointer = ck_pr_load_ptr(&head.pointer->next.pointer); in ck_fifo_mpmc_dequeue()
374 if (head.pointer == tail.pointer) { in ck_fifo_mpmc_dequeue()
376 * The head is guaranteed to always point at a stub in ck_fifo_mpmc_dequeue()
385 ck_pr_cas_ptr_2(&fifo->tail, &tail, &update); in ck_fifo_mpmc_dequeue()
388 * It is possible for head snapshot to have been in ck_fifo_mpmc_dequeue()
389 * re-used. Avoid deferencing during enqueue in ck_fifo_mpmc_dequeue()
390 * re-use. in ck_fifo_mpmc_dequeue()
396 *(void **)value = ck_pr_load_ptr(&next.pointer->value); in ck_fifo_mpmc_dequeue()
398 /* Forward the head pointer to the next entry. */ in ck_fifo_mpmc_dequeue()
399 update.generation = head.generation + 1; in ck_fifo_mpmc_dequeue()
400 if (ck_pr_cas_ptr_2(&fifo->head, &head, &update) == true) in ck_fifo_mpmc_dequeue()
405 *garbage = head.pointer; in ck_fifo_mpmc_dequeue()
414 struct ck_fifo_mpmc_pointer head, tail, next, update; in ck_fifo_mpmc_trydequeue() local
416 head.generation = ck_pr_load_ptr(&fifo->head.generation); in ck_fifo_mpmc_trydequeue()
418 head.pointer = ck_pr_load_ptr(&fifo->head.pointer); in ck_fifo_mpmc_trydequeue()
420 tail.generation = ck_pr_load_ptr(&fifo->tail.generation); in ck_fifo_mpmc_trydequeue()
422 tail.pointer = ck_pr_load_ptr(&fifo->tail.pointer); in ck_fifo_mpmc_trydequeue()
424 next.generation = ck_pr_load_ptr(&head.pointer->next.generation); in ck_fifo_mpmc_trydequeue()
426 next.pointer = ck_pr_load_ptr(&head.pointer->next.pointer); in ck_fifo_mpmc_trydequeue()
429 if (head.pointer == tail.pointer) { in ck_fifo_mpmc_trydequeue()
431 * The head is guaranteed to always point at a stub in ck_fifo_mpmc_trydequeue()
440 ck_pr_cas_ptr_2(&fifo->tail, &tail, &update); in ck_fifo_mpmc_trydequeue()
444 * It is possible for head snapshot to have been in ck_fifo_mpmc_trydequeue()
445 * re-used. Avoid deferencing during enqueue. in ck_fifo_mpmc_trydequeue()
451 *(void **)value = ck_pr_load_ptr(&next.pointer->value); in ck_fifo_mpmc_trydequeue()
453 /* Forward the head pointer to the next entry. */ in ck_fifo_mpmc_trydequeue()
454 update.generation = head.generation + 1; in ck_fifo_mpmc_trydequeue()
455 if (ck_pr_cas_ptr_2(&fifo->head, &head, &update) == false) in ck_fifo_mpmc_trydequeue()
459 *garbage = head.pointer; in ck_fifo_mpmc_trydequeue()
463 #define CK_FIFO_MPMC_ISEMPTY(f) ((f)->head.pointer->next.pointer == NULL)
464 #define CK_FIFO_MPMC_FIRST(f) ((f)->head.pointer->next.pointer)
465 #define CK_FIFO_MPMC_NEXT(m) ((m)->next.pointer)
470 #define CK_FIFO_MPMC_FOREACH_SAFE(fifo, entry, T) \ argument
472 (entry) != NULL && ((T) = (entry)->next.pointer, 1); \
473 (entry) = (T))