Lines Matching refs:r

57 space_available(struct ifmp_ring *r, union ring_state s)  in space_available()  argument
59 uint16_t x = r->size - 1; in space_available()
70 increment_idx(struct ifmp_ring *r, uint16_t idx, uint16_t n) in increment_idx() argument
72 int x = r->size - idx; in increment_idx()
93 drain_ring_locked(struct ifmp_ring *r, union ring_state os, uint16_t prev, int budget) in drain_ring_locked() argument
104 counter_u64_add(r->starts, 1); in drain_ring_locked()
110 n = r->drain(r, cidx, pidx); in drain_ring_locked()
112 os.state = ns.state = r->state; in drain_ring_locked()
115 r->state = ns.state; in drain_ring_locked()
117 counter_u64_add(r->stalls, 1); in drain_ring_locked()
119 counter_u64_add(r->restarts, 1); in drain_ring_locked()
120 counter_u64_add(r->stalls, 1); in drain_ring_locked()
124 cidx = increment_idx(r, cidx, n); in drain_ring_locked()
136 os.state = ns.state = r->state; in drain_ring_locked()
139 r->state = ns.state; in drain_ring_locked()
142 counter_u64_add(r->abdications, 1); in drain_ring_locked()
148 counter_u64_add(r->restarts, 1); in drain_ring_locked()
167 drain_ring_lockless(struct ifmp_ring *r, union ring_state os, uint16_t prev, int budget) in drain_ring_lockless() argument
178 counter_u64_add(r->starts, 1); in drain_ring_lockless()
184 n = r->drain(r, cidx, pidx); in drain_ring_lockless()
187 os.state = r->state; in drain_ring_lockless()
192 } while (atomic_fcmpset_64(&r->state, &os.state, in drain_ring_lockless()
196 counter_u64_add(r->stalls, 1); in drain_ring_lockless()
198 counter_u64_add(r->restarts, 1); in drain_ring_lockless()
199 counter_u64_add(r->stalls, 1); in drain_ring_lockless()
203 cidx = increment_idx(r, cidx, n); in drain_ring_lockless()
215 os.state = r->state; in drain_ring_lockless()
220 } while (atomic_fcmpset_acq_64(&r->state, &os.state, in drain_ring_lockless()
225 counter_u64_add(r->abdications, 1); in drain_ring_lockless()
231 counter_u64_add(r->restarts, 1); in drain_ring_lockless()
250 struct ifmp_ring *r; in ifmp_ring_alloc() local
260 r = malloc(__offsetof(struct ifmp_ring, items[size]), mt, flags | M_ZERO); in ifmp_ring_alloc()
261 if (r == NULL) in ifmp_ring_alloc()
263 r->size = size; in ifmp_ring_alloc()
264 r->cookie = cookie; in ifmp_ring_alloc()
265 r->mt = mt; in ifmp_ring_alloc()
266 r->drain = drain; in ifmp_ring_alloc()
267 r->can_drain = can_drain; in ifmp_ring_alloc()
268 r->enqueues = counter_u64_alloc(flags); in ifmp_ring_alloc()
269 r->drops = counter_u64_alloc(flags); in ifmp_ring_alloc()
270 r->starts = counter_u64_alloc(flags); in ifmp_ring_alloc()
271 r->stalls = counter_u64_alloc(flags); in ifmp_ring_alloc()
272 r->restarts = counter_u64_alloc(flags); in ifmp_ring_alloc()
273 r->abdications = counter_u64_alloc(flags); in ifmp_ring_alloc()
274 if (r->enqueues == NULL || r->drops == NULL || r->starts == NULL || in ifmp_ring_alloc()
275 r->stalls == NULL || r->restarts == NULL || in ifmp_ring_alloc()
276 r->abdications == NULL) { in ifmp_ring_alloc()
277 ifmp_ring_free(r); in ifmp_ring_alloc()
281 *pr = r; in ifmp_ring_alloc()
283 mtx_init(&r->lock, "mp_ring lock", NULL, MTX_DEF); in ifmp_ring_alloc()
289 ifmp_ring_free(struct ifmp_ring *r) in ifmp_ring_free() argument
292 if (r == NULL) in ifmp_ring_free()
295 if (r->enqueues != NULL) in ifmp_ring_free()
296 counter_u64_free(r->enqueues); in ifmp_ring_free()
297 if (r->drops != NULL) in ifmp_ring_free()
298 counter_u64_free(r->drops); in ifmp_ring_free()
299 if (r->starts != NULL) in ifmp_ring_free()
300 counter_u64_free(r->starts); in ifmp_ring_free()
301 if (r->stalls != NULL) in ifmp_ring_free()
302 counter_u64_free(r->stalls); in ifmp_ring_free()
303 if (r->restarts != NULL) in ifmp_ring_free()
304 counter_u64_free(r->restarts); in ifmp_ring_free()
305 if (r->abdications != NULL) in ifmp_ring_free()
306 counter_u64_free(r->abdications); in ifmp_ring_free()
308 free(r, r->mt); in ifmp_ring_free()
318 ifmp_ring_enqueue(struct ifmp_ring *r, void **items, int n, int budget, int abdicate) in ifmp_ring_enqueue() argument
327 mtx_lock(&r->lock); in ifmp_ring_enqueue()
332 os.state = r->state; in ifmp_ring_enqueue()
333 if (n >= space_available(r, os)) { in ifmp_ring_enqueue()
334 counter_u64_add(r->drops, n); in ifmp_ring_enqueue()
336 mtx_unlock(&r->lock); in ifmp_ring_enqueue()
338 ifmp_ring_check_drainage(r, 0); in ifmp_ring_enqueue()
342 ns.pidx_head = increment_idx(r, os.pidx_head, n); in ifmp_ring_enqueue()
343 r->state = ns.state; in ifmp_ring_enqueue()
354 ns.state = r->state; in ifmp_ring_enqueue()
360 r->items[i] = *items++; in ifmp_ring_enqueue()
361 if (__predict_false(++i == r->size)) in ifmp_ring_enqueue()
369 os.state = ns.state = r->state; in ifmp_ring_enqueue()
376 r->state = ns.state; in ifmp_ring_enqueue()
377 counter_u64_add(r->enqueues, n); in ifmp_ring_enqueue()
385 drain_ring_locked(r, ns, os.flags, budget); in ifmp_ring_enqueue()
388 mtx_unlock(&r->lock); in ifmp_ring_enqueue()
393 ifmp_ring_enqueue(struct ifmp_ring *r, void **items, int n, int budget, int abdicate) in ifmp_ring_enqueue() argument
406 os.state = r->state; in ifmp_ring_enqueue()
408 if (n >= space_available(r, os)) { in ifmp_ring_enqueue()
409 counter_u64_add(r->drops, n); in ifmp_ring_enqueue()
412 ifmp_ring_check_drainage(r, 0); in ifmp_ring_enqueue()
416 ns.pidx_head = increment_idx(r, os.pidx_head, n); in ifmp_ring_enqueue()
418 if (atomic_fcmpset_64(&r->state, &os.state, ns.state)) in ifmp_ring_enqueue()
433 ns.state = r->state; in ifmp_ring_enqueue()
439 r->items[i] = *items++; in ifmp_ring_enqueue()
440 if (__predict_false(++i == r->size)) in ifmp_ring_enqueue()
448 os.state = r->state; in ifmp_ring_enqueue()
457 } while (atomic_fcmpset_rel_64(&r->state, &os.state, ns.state) == 0); in ifmp_ring_enqueue()
459 counter_u64_add(r->enqueues, n); in ifmp_ring_enqueue()
467 drain_ring_lockless(r, ns, os.flags, budget); in ifmp_ring_enqueue()
475 ifmp_ring_check_drainage(struct ifmp_ring *r, int budget) in ifmp_ring_check_drainage() argument
479 os.state = r->state; in ifmp_ring_check_drainage()
482 (os.flags != ABDICATED && r->can_drain(r) == 0)) // Can either drain, or everyone left in ifmp_ring_check_drainage()
490 mtx_lock(&r->lock); in ifmp_ring_check_drainage()
491 if (r->state != os.state) { in ifmp_ring_check_drainage()
492 mtx_unlock(&r->lock); in ifmp_ring_check_drainage()
495 r->state = ns.state; in ifmp_ring_check_drainage()
496 drain_ring_locked(r, ns, os.flags, budget); in ifmp_ring_check_drainage()
497 mtx_unlock(&r->lock); in ifmp_ring_check_drainage()
503 if (!atomic_cmpset_acq_64(&r->state, os.state, ns.state)) in ifmp_ring_check_drainage()
506 drain_ring_lockless(r, ns, os.flags, budget); in ifmp_ring_check_drainage()
511 ifmp_ring_reset_stats(struct ifmp_ring *r) in ifmp_ring_reset_stats() argument
514 counter_u64_zero(r->enqueues); in ifmp_ring_reset_stats()
515 counter_u64_zero(r->drops); in ifmp_ring_reset_stats()
516 counter_u64_zero(r->starts); in ifmp_ring_reset_stats()
517 counter_u64_zero(r->stalls); in ifmp_ring_reset_stats()
518 counter_u64_zero(r->restarts); in ifmp_ring_reset_stats()
519 counter_u64_zero(r->abdications); in ifmp_ring_reset_stats()
523 ifmp_ring_is_idle(struct ifmp_ring *r) in ifmp_ring_is_idle() argument
527 s.state = r->state; in ifmp_ring_is_idle()
536 ifmp_ring_is_stalled(struct ifmp_ring *r) in ifmp_ring_is_stalled() argument
540 s.state = r->state; in ifmp_ring_is_stalled()