mp_ring.c (fe51d4cdfee05113967b19dc5828e58d125cddd5) mp_ring.c (5881181d8c80609034204c128975fbbac77baaed)
1/*-
2 * Copyright (c) 2014 Chelsio Communications, Inc.
3 * All rights reserved.
4 * Written by: Navdeep Parhar <np@FreeBSD.org>
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:

--- 23 unchanged lines hidden (view full) ---

32#include <sys/param.h>
33#include <sys/systm.h>
34#include <sys/counter.h>
35#include <sys/lock.h>
36#include <sys/mutex.h>
37#include <sys/malloc.h>
38#include <machine/cpu.h>
39
1/*-
2 * Copyright (c) 2014 Chelsio Communications, Inc.
3 * All rights reserved.
4 * Written by: Navdeep Parhar <np@FreeBSD.org>
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:

--- 23 unchanged lines hidden (view full) ---

32#include <sys/param.h>
33#include <sys/systm.h>
34#include <sys/counter.h>
35#include <sys/lock.h>
36#include <sys/mutex.h>
37#include <sys/malloc.h>
38#include <machine/cpu.h>
39
40#if defined(__powerpc__) || defined(__mips__) || defined(__i386__)
41#define NO_64BIT_ATOMICS
42#endif
43
44#if defined(__i386__)
45#define atomic_cmpset_acq_64 atomic_cmpset_64
46#define atomic_cmpset_rel_64 atomic_cmpset_64
47#endif
48
49#include <net/mp_ring.h>
50
51union ring_state {

--- 44 unchanged lines hidden (view full) ---

96 if (s.cidx == s.pidx_tail)
97 return (IDLE);
98 else if (abdicate && s.pidx_tail != s.pidx_head)
99 return (ABDICATED);
100
101 return (BUSY);
102}
103
40#if defined(__i386__)
41#define atomic_cmpset_acq_64 atomic_cmpset_64
42#define atomic_cmpset_rel_64 atomic_cmpset_64
43#endif
44
45#include <net/mp_ring.h>
46
47union ring_state {

--- 44 unchanged lines hidden (view full) ---

92 if (s.cidx == s.pidx_tail)
93 return (IDLE);
94 else if (abdicate && s.pidx_tail != s.pidx_head)
95 return (ABDICATED);
96
97 return (BUSY);
98}
99
104#ifdef NO_64BIT_ATOMICS
100#ifdef MP_RING_NO_64BIT_ATOMICS
105static void
106drain_ring_locked(struct ifmp_ring *r, union ring_state os, uint16_t prev, int budget)
107{
108 union ring_state ns;
109 int n, pending, total;
110 uint16_t cidx = os.cidx;
111 uint16_t pidx = os.pidx_tail;
112

--- 173 unchanged lines hidden (view full) ---

286 if (r->enqueues == NULL || r->drops == NULL || r->starts == NULL ||
287 r->stalls == NULL || r->restarts == NULL ||
288 r->abdications == NULL) {
289 ifmp_ring_free(r);
290 return (ENOMEM);
291 }
292
293 *pr = r;
101static void
102drain_ring_locked(struct ifmp_ring *r, union ring_state os, uint16_t prev, int budget)
103{
104 union ring_state ns;
105 int n, pending, total;
106 uint16_t cidx = os.cidx;
107 uint16_t pidx = os.pidx_tail;
108

--- 173 unchanged lines hidden (view full) ---

282 if (r->enqueues == NULL || r->drops == NULL || r->starts == NULL ||
283 r->stalls == NULL || r->restarts == NULL ||
284 r->abdications == NULL) {
285 ifmp_ring_free(r);
286 return (ENOMEM);
287 }
288
289 *pr = r;
294#ifdef NO_64BIT_ATOMICS
290#ifdef MP_RING_NO_64BIT_ATOMICS
295 mtx_init(&r->lock, "mp_ring lock", NULL, MTX_DEF);
296#endif
297 return (0);
298}
299
300void
301ifmp_ring_free(struct ifmp_ring *r)
302{

--- 17 unchanged lines hidden (view full) ---

320 free(r, r->mt);
321}
322
323/*
324 * Enqueue n items and maybe drain the ring for some time.
325 *
326 * Returns an errno.
327 */
291 mtx_init(&r->lock, "mp_ring lock", NULL, MTX_DEF);
292#endif
293 return (0);
294}
295
296void
297ifmp_ring_free(struct ifmp_ring *r)
298{

--- 17 unchanged lines hidden (view full) ---

316 free(r, r->mt);
317}
318
319/*
320 * Enqueue n items and maybe drain the ring for some time.
321 *
322 * Returns an errno.
323 */
328#ifdef NO_64BIT_ATOMICS
324#ifdef MP_RING_NO_64BIT_ATOMICS
329int
330ifmp_ring_enqueue(struct ifmp_ring *r, void **items, int n, int budget, int abdicate)
331{
332 union ring_state os, ns;
333 uint16_t pidx_start, pidx_stop;
334 int i;
335
336 MPASS(items != NULL);

--- 161 unchanged lines hidden (view full) ---

498 (os.flags != ABDICATED && r->can_drain(r) == 0)) // Can either drain, or everyone left
499 return;
500
501 MPASS(os.cidx != os.pidx_tail); /* implied by STALLED */
502 ns.state = os.state;
503 ns.flags = BUSY;
504
505
325int
326ifmp_ring_enqueue(struct ifmp_ring *r, void **items, int n, int budget, int abdicate)
327{
328 union ring_state os, ns;
329 uint16_t pidx_start, pidx_stop;
330 int i;
331
332 MPASS(items != NULL);

--- 161 unchanged lines hidden (view full) ---

494 (os.flags != ABDICATED && r->can_drain(r) == 0)) // Can either drain, or everyone left
495 return;
496
497 MPASS(os.cidx != os.pidx_tail); /* implied by STALLED */
498 ns.state = os.state;
499 ns.flags = BUSY;
500
501
506#ifdef NO_64BIT_ATOMICS
502#ifdef MP_RING_NO_64BIT_ATOMICS
507 mtx_lock(&r->lock);
508 if (r->state != os.state) {
509 mtx_unlock(&r->lock);
510 return;
511 }
512 r->state = ns.state;
513 drain_ring_locked(r, ns, os.flags, budget);
514 mtx_unlock(&r->lock);

--- 49 unchanged lines hidden --- 		503 mtx_lock(&r->lock);
504 if (r->state != os.state) {
505 mtx_unlock(&r->lock);
506 return;
507 }
508 r->state = ns.state;
509 drain_ring_locked(r, ns, os.flags, budget);
510 mtx_unlock(&r->lock);

--- 49 unchanged lines hidden ---