95 	union ring_state ns;  in drain_ring_locked()  local
112 			os.state = ns.state = r->state;  in drain_ring_locked()
113 			ns.cidx = cidx;  in drain_ring_locked()
114 			ns.flags = STALLED;  in drain_ring_locked()
115 			r->state = ns.state;  in drain_ring_locked()
136 		os.state = ns.state = r->state;  in drain_ring_locked()
137 		ns.cidx = cidx;  in drain_ring_locked()
138 		ns.flags = state_to_flags(ns, total >= budget);  in drain_ring_locked()
139 		r->state = ns.state;  in drain_ring_locked()
141 		if (ns.flags == ABDICATED)  in drain_ring_locked()
143 		if (ns.flags != BUSY) {  in drain_ring_locked()
145 			MPASS(ns.flags != STALLED);  in drain_ring_locked()
157 		pidx = ns.pidx_tail;  in drain_ring_locked()
169 	union ring_state ns;  in drain_ring_lockless()  local
189 				ns.state = os.state;  in drain_ring_lockless()
190 				ns.cidx = cidx;  in drain_ring_lockless()
191 				ns.flags = STALLED;  in drain_ring_lockless()
193 			    ns.state) == 0);  in drain_ring_lockless()
217 			ns.state = os.state;  in drain_ring_lockless()
218 			ns.cidx = cidx;  in drain_ring_lockless()
219 			ns.flags = state_to_flags(ns, total >= budget);  in drain_ring_lockless()
221 		    ns.state) == 0);  in drain_ring_lockless()
224 		if (ns.flags == ABDICATED)  in drain_ring_lockless()
226 		if (ns.flags != BUSY) {  in drain_ring_lockless()
228 			MPASS(ns.flags != STALLED);  in drain_ring_lockless()
240 		pidx = ns.pidx_tail;  in drain_ring_lockless()
320 	union ring_state os, ns;  in ifmp_ring_enqueue()  local
341 	ns.state = os.state;  in ifmp_ring_enqueue()
342 	ns.pidx_head = increment_idx(r, os.pidx_head, n);  in ifmp_ring_enqueue()
343 	r->state = ns.state;  in ifmp_ring_enqueue()
345 	pidx_stop = ns.pidx_head;  in ifmp_ring_enqueue()
352 	while (ns.pidx_tail != pidx_start) {  in ifmp_ring_enqueue()
354 		ns.state = r->state;  in ifmp_ring_enqueue()
369 	os.state = ns.state = r->state;  in ifmp_ring_enqueue()
370 	ns.pidx_tail = pidx_stop;  in ifmp_ring_enqueue()
373 			ns.flags = ABDICATED;  in ifmp_ring_enqueue()
375 		ns.flags = BUSY;  in ifmp_ring_enqueue()
376 	r->state = ns.state;  in ifmp_ring_enqueue()
385 			drain_ring_locked(r, ns, os.flags, budget);  in ifmp_ring_enqueue()
395 	union ring_state os, ns;  in ifmp_ring_enqueue()  local
415 		ns.state = os.state;  in ifmp_ring_enqueue()
416 		ns.pidx_head = increment_idx(r, os.pidx_head, n);  in ifmp_ring_enqueue()
418 		if (atomic_fcmpset_64(&r->state, &os.state, ns.state))  in ifmp_ring_enqueue()
424 	pidx_stop = ns.pidx_head;  in ifmp_ring_enqueue()
431 	while (ns.pidx_tail != pidx_start) {  in ifmp_ring_enqueue()
433 		ns.state = r->state;  in ifmp_ring_enqueue()
450 		ns.state = os.state;  in ifmp_ring_enqueue()
451 		ns.pidx_tail = pidx_stop;  in ifmp_ring_enqueue()
454 				ns.flags = ABDICATED;  in ifmp_ring_enqueue()
456 			ns.flags = BUSY;  in ifmp_ring_enqueue()
457 	} while (atomic_fcmpset_rel_64(&r->state, &os.state, ns.state) == 0);  in ifmp_ring_enqueue()
467 			drain_ring_lockless(r, ns, os.flags, budget);  in ifmp_ring_enqueue()
477 	union ring_state os, ns;  in ifmp_ring_check_drainage()  local
486 	ns.state = os.state;  in ifmp_ring_check_drainage()
487 	ns.flags = BUSY;  in ifmp_ring_check_drainage()
495 	r->state = ns.state;  in ifmp_ring_check_drainage()
496 	drain_ring_locked(r, ns, os.flags, budget);  in ifmp_ring_check_drainage()
503 	if (!atomic_cmpset_acq_64(&r->state, os.state, ns.state))  in ifmp_ring_check_drainage()
506 	drain_ring_lockless(r, ns, os.flags, budget);  in ifmp_ring_check_drainage()