108 	/* The consumer counter must always be smaller than the producer. */  in ck_ring_valid()
112 	/* The producer may only be up to size slots ahead of consumer. */  in ck_ring_valid()
137  * for a single producer.
146 	unsigned int consumer, producer, delta;  in _ck_ring_enqueue_reserve_sp()  local
149 	producer = ring->p_tail;  in _ck_ring_enqueue_reserve_sp()
150 	delta = producer + 1;  in _ck_ring_enqueue_reserve_sp()
152 		*size = (producer - consumer) & mask;  in _ck_ring_enqueue_reserve_sp()
157 	return (char *)buffer + ts * (producer & mask);  in _ck_ring_enqueue_reserve_sp()
181 	unsigned int consumer, producer, delta;  in _ck_ring_enqueue_sp()  local
184 	producer = ring->p_tail;  in _ck_ring_enqueue_sp()
185 	delta = producer + 1;  in _ck_ring_enqueue_sp()
187 		*size = (producer - consumer) & mask;  in _ck_ring_enqueue_sp()
192 	buffer = (char *)buffer + ts * (producer & mask);  in _ck_ring_enqueue_sp()
226 	unsigned int consumer, producer;  in _ck_ring_dequeue_sc()  local
229 	producer = ck_pr_load_uint(&ring->p_tail);  in _ck_ring_dequeue_sc()
231 	if (CK_CC_UNLIKELY(consumer == producer))  in _ck_ring_dequeue_sc()
236 	 * of the producer counter.  in _ck_ring_dequeue_sc()
260 	unsigned int producer, consumer, delta;  in _ck_ring_enqueue_reserve_mp()  local
262 	producer = ck_pr_load_uint(&ring->p_head);  in _ck_ring_enqueue_reserve_mp()
268 		delta = producer + 1;  in _ck_ring_enqueue_reserve_mp()
270 		if (CK_CC_LIKELY((producer - consumer) < mask)) {  in _ck_ring_enqueue_reserve_mp()
272 			    producer, delta, &producer) == true) {  in _ck_ring_enqueue_reserve_mp()
281 			if (producer == new_producer) {  in _ck_ring_enqueue_reserve_mp()
283 					*size = (producer - consumer) & mask;  in _ck_ring_enqueue_reserve_mp()
288 			producer = new_producer;  in _ck_ring_enqueue_reserve_mp()
292 	*ticket = producer;  in _ck_ring_enqueue_reserve_mp()
294 		*size = (producer - consumer) & mask;  in _ck_ring_enqueue_reserve_mp()
296 	return (char *)buffer + ts * (producer & mask);  in _ck_ring_enqueue_reserve_mp()
300 _ck_ring_enqueue_commit_mp(struct ck_ring *ring, unsigned int producer)  in _ck_ring_enqueue_commit_mp()  argument
303 	while (ck_pr_load_uint(&ring->p_tail) != producer)  in _ck_ring_enqueue_commit_mp()
307 	ck_pr_store_uint(&ring->p_tail, producer + 1);  in _ck_ring_enqueue_commit_mp()
319 	unsigned int producer, consumer, delta;  in _ck_ring_enqueue_mp()  local
322 	producer = ck_pr_load_uint(&ring->p_head);  in _ck_ring_enqueue_mp()
326 		 * The snapshot of producer must be up to date with respect to  in _ck_ring_enqueue_mp()
332 		delta = producer + 1;  in _ck_ring_enqueue_mp()
335 		 * Only try to CAS if the producer is not clearly stale (not  in _ck_ring_enqueue_mp()
338 		if (CK_CC_LIKELY((producer - consumer) < mask)) {  in _ck_ring_enqueue_mp()
340 			    producer, delta, &producer) == true) {  in _ck_ring_enqueue_mp()
361 			if (producer == new_producer) {  in _ck_ring_enqueue_mp()
369 			producer = new_producer;  in _ck_ring_enqueue_mp()
373 	buffer = (char *)buffer + ts * (producer & mask);  in _ck_ring_enqueue_mp()
380 	while (ck_pr_load_uint(&ring->p_tail) != producer)  in _ck_ring_enqueue_mp()
384 	 * Ensure that copy is completed before updating shared producer  in _ck_ring_enqueue_mp()
392 		*size = (producer - consumer) & mask;  in _ck_ring_enqueue_mp()
419 	unsigned int consumer, producer;  in _ck_ring_trydequeue_mc()  local
423 	producer = ck_pr_load_uint(&ring->p_tail);  in _ck_ring_trydequeue_mc()
425 	if (CK_CC_UNLIKELY(consumer == producer))  in _ck_ring_trydequeue_mc()
444 	unsigned int consumer, producer;  in _ck_ring_dequeue_mc()  local
452 		 * Producer counter must represent state relative to  in _ck_ring_dequeue_mc()
456 		producer = ck_pr_load_uint(&ring->p_tail);  in _ck_ring_dequeue_mc()
458 		if (CK_CC_UNLIKELY(consumer == producer))  in _ck_ring_dequeue_mc()
479  * to one concurrent consumer and up to one concurrent producer.
616  * consumers with up to one concurrent producer.
977  * A single producer with one concurrent consumer.
991  * A single producer with any number of concurrent consumers.