93 drain_ring_locked(struct ifmp_ring *r, union ring_state os, uint16_t prev, int budget)  in drain_ring_locked()  argument
97 	uint16_t cidx = os.cidx;  in drain_ring_locked()
98 	uint16_t pidx = os.pidx_tail;  in drain_ring_locked()
100 	MPASS(os.flags == BUSY);  in drain_ring_locked()
112 			os.state = ns.state = r->state;  in drain_ring_locked()
136 		os.state = ns.state = r->state;  in drain_ring_locked()
167 drain_ring_lockless(struct ifmp_ring *r, union ring_state os, uint16_t prev, int budget)  in drain_ring_lockless()  argument
171 	uint16_t cidx = os.cidx;  in drain_ring_lockless()
172 	uint16_t pidx = os.pidx_tail;  in drain_ring_lockless()
174 	MPASS(os.flags == BUSY);  in drain_ring_lockless()
187 			os.state = r->state;  in drain_ring_lockless()
189 				ns.state = os.state;  in drain_ring_lockless()
192 			} while (atomic_fcmpset_64(&r->state, &os.state,  in drain_ring_lockless()
215 		os.state = r->state;  in drain_ring_lockless()
217 			ns.state = os.state;  in drain_ring_lockless()
220 		} while (atomic_fcmpset_acq_64(&r->state, &os.state,  in drain_ring_lockless()
320 	union ring_state os, ns;  in ifmp_ring_enqueue()  local
332 	os.state = r->state;  in ifmp_ring_enqueue()
333 	if (n >= space_available(r, os)) {  in ifmp_ring_enqueue()
335 		MPASS(os.flags != IDLE);  in ifmp_ring_enqueue()
337 		if (os.flags == STALLED)  in ifmp_ring_enqueue()
341 	ns.state = os.state;  in ifmp_ring_enqueue()
342 	ns.pidx_head = increment_idx(r, os.pidx_head, n);  in ifmp_ring_enqueue()
344 	pidx_start = os.pidx_head;  in ifmp_ring_enqueue()
369 	os.state = ns.state = r->state;  in ifmp_ring_enqueue()
372 		if (os.flags == IDLE)  in ifmp_ring_enqueue()
384 		if (os.flags != BUSY)  in ifmp_ring_enqueue()
385 			drain_ring_locked(r, ns, os.flags, budget);  in ifmp_ring_enqueue()
395 	union ring_state os, ns;  in ifmp_ring_enqueue()  local
406 	os.state = r->state;  in ifmp_ring_enqueue()
408 		if (n >= space_available(r, os)) {  in ifmp_ring_enqueue()
410 			MPASS(os.flags != IDLE);  in ifmp_ring_enqueue()
411 			if (os.flags == STALLED)  in ifmp_ring_enqueue()
415 		ns.state = os.state;  in ifmp_ring_enqueue()
416 		ns.pidx_head = increment_idx(r, os.pidx_head, n);  in ifmp_ring_enqueue()
418 		if (atomic_fcmpset_64(&r->state, &os.state, ns.state))  in ifmp_ring_enqueue()
423 	pidx_start = os.pidx_head;  in ifmp_ring_enqueue()
448 	os.state = r->state;  in ifmp_ring_enqueue()
450 		ns.state = os.state;  in ifmp_ring_enqueue()
453 			if (os.flags == IDLE)  in ifmp_ring_enqueue()
457 	} while (atomic_fcmpset_rel_64(&r->state, &os.state, ns.state) == 0);  in ifmp_ring_enqueue()
466 		if (os.flags != BUSY)  in ifmp_ring_enqueue()
467 			drain_ring_lockless(r, ns, os.flags, budget);  in ifmp_ring_enqueue()
477 	union ring_state os, ns;  in ifmp_ring_check_drainage()  local
479 	os.state = r->state;  in ifmp_ring_check_drainage()
480 	if ((os.flags != STALLED && os.flags != ABDICATED) ||	// Only continue in STALLED and ABDICATED  in ifmp_ring_check_drainage()
481 	    os.pidx_head != os.pidx_tail ||			// Require work to be available  in ifmp_ring_check_drainage()
482 	    (os.flags != ABDICATED && r->can_drain(r) == 0))	// Can either drain, or everyone left  in ifmp_ring_check_drainage()
485 	MPASS(os.cidx != os.pidx_tail);	/* implied by STALLED */  in ifmp_ring_check_drainage()
486 	ns.state = os.state;  in ifmp_ring_check_drainage()
491 	if (r->state != os.state) {  in ifmp_ring_check_drainage()
496 	drain_ring_locked(r, ns, os.flags, budget);  in ifmp_ring_check_drainage()
503 	if (!atomic_cmpset_acq_64(&r->state, os.state, ns.state))  in ifmp_ring_check_drainage()
506 	drain_ring_lockless(r, ns, os.flags, budget);  in ifmp_ring_check_drainage()