xref: /titanic_51/usr/src/uts/common/sys/mac_soft_ring.h (revision 791a814c934fcd4deb13b26c1f116ff283272a0d)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #ifndef	_SYS_MAC_SOFT_RING_H
28 #define	_SYS_MAC_SOFT_RING_H
29 
30 #ifdef	__cplusplus
31 extern "C" {
32 #endif
33 
34 #include <sys/types.h>
35 #include <sys/cpuvar.h>
36 #include <sys/processor.h>
37 #include <sys/stream.h>
38 #include <sys/squeue.h>
39 #include <sys/dlpi.h>
40 #include <sys/mac_impl.h>
41 
42 #define	S_RING_NAMELEN 64
43 
44 #define	MAX_SR_FANOUT	32
45 
46 extern boolean_t mac_soft_ring_enable;
47 extern boolean_t mac_latency_optimize;
48 
49 typedef struct mac_soft_ring_s mac_soft_ring_t;
50 typedef struct mac_soft_ring_set_s mac_soft_ring_set_t;
51 
52 typedef void (*mac_soft_ring_drain_func_t)(mac_soft_ring_t *);
53 typedef mac_tx_cookie_t (*mac_tx_func_t)(mac_soft_ring_set_t *, mblk_t *,
54     uintptr_t, uint16_t, mblk_t **);
55 
56 
57 /* Tx notify callback */
58 typedef struct mac_tx_notify_cb_s {
59 	mac_cb_t		mtnf_link;	/* Linked list of callbacks */
60 	mac_tx_notify_t		mtnf_fn;	/* The callback function */
61 	void			*mtnf_arg;	/* Callback function argument */
62 } mac_tx_notify_cb_t;
63 
64 struct mac_soft_ring_s {
65 	/* Keep the most used members 64bytes cache aligned */
66 	kmutex_t	s_ring_lock;	/* lock before using any member */
67 	uint16_t	s_ring_type;	/* processing model of the sq */
68 	uint16_t	s_ring_state;	/* state flags and message count */
69 	int		s_ring_count;	/* # of mblocks in mac_soft_ring */
70 	size_t		s_ring_size;	/* Size of data queued */
71 	mblk_t		*s_ring_first;	/* first mblk chain or NULL */
72 	mblk_t		*s_ring_last;	/* last mblk chain or NULL */
73 
74 	mac_direct_rx_t	s_ring_rx_func;
75 	void		*s_ring_rx_arg1;
76 	mac_resource_handle_t  s_ring_rx_arg2;
77 
78 	/*
79 	 * Threshold after which packets get dropped.
80 	 * Is always greater than s_ring_tx_hiwat
81 	 */
82 	int		s_ring_tx_max_q_cnt;
83 	/* # of mblocks after which to apply flow control */
84 	int		s_ring_tx_hiwat;
85 	/* # of mblocks after which to relieve flow control */
86 	int		s_ring_tx_lowat;
87 	boolean_t	s_ring_tx_woken_up;
88 	uint32_t	s_ring_blocked_cnt;	/* times blocked for Tx descs */
89 	uint32_t	s_ring_unblocked_cnt;	/* unblock calls from driver */
90 	uint32_t	s_ring_hiwat_cnt;	/* times blocked for Tx descs */
91 
92 	void		*s_ring_tx_arg1;
93 	void		*s_ring_tx_arg2;
94 
95 	/* Tx notify callback */
96 	mac_cb_info_t	s_ring_notify_cb_info;		/* cb list info */
97 	mac_cb_t	*s_ring_notify_cb_list;		/* The cb list */
98 
99 	clock_t		s_ring_awaken;	/* time async thread was awakened */
100 
101 	kthread_t	*s_ring_run;	/* Current thread processing sq */
102 	processorid_t	s_ring_cpuid;	/* processor to bind to */
103 	processorid_t	s_ring_cpuid_save;	/* saved cpuid during offline */
104 	kcondvar_t	s_ring_async;	/* async thread blocks on */
105 	clock_t		s_ring_wait;	/* lbolts to wait after a fill() */
106 	timeout_id_t	s_ring_tid;	/* timer id of pending timeout() */
107 	kthread_t	*s_ring_worker;	/* kernel thread id */
108 	char		s_ring_name[S_RING_NAMELEN + 1];
109 	uint32_t	s_ring_total_inpkt;
110 	uint32_t	s_ring_drops;
111 	struct mac_client_impl_s *s_ring_mcip;
112 	void		*s_ring_flent;
113 	kstat_t		*s_ring_ksp;
114 
115 	/* Teardown, poll disable control ops */
116 	kcondvar_t	s_ring_client_cv; /* Client wait for control op */
117 
118 	mac_soft_ring_set_t *s_ring_set;   /* The SRS this ring belongs to */
119 	mac_soft_ring_t	*s_ring_next;
120 	mac_soft_ring_t	*s_ring_prev;
121 	mac_soft_ring_drain_func_t s_ring_drain_func;
122 };
123 
124 typedef void (*mac_srs_drain_proc_t)(mac_soft_ring_set_t *, uint_t);
125 
126 /* Transmit side Soft Ring Set */
127 typedef struct mac_srs_tx_s {
128 	/* Members for Tx size processing */
129 	uint32_t	st_mode;
130 	mac_tx_func_t	st_func;
131 	void		*st_arg1;
132 	void		*st_arg2;
133 	mac_group_t	*st_group;	/* TX group for share */
134 	uint32_t	st_ring_count;	/* no. of tx rings */
135 	mac_ring_handle_t	*st_rings;
136 
137 	boolean_t	st_woken_up;
138 
139 	/*
140 	 * st_max_q_cnt is the queue depth threshold to limit
141 	 * outstanding packets on the Tx SRS. Once the limit
142 	 * is reached, Tx SRS will drop packets until the
143 	 * limit goes below the threshold.
144 	 */
145 	uint32_t	st_max_q_cnt;	/* max. outstanding packets */
146 	/*
147 	 * st_hiwat is used Tx serializer and bandwidth mode.
148 	 * This is the queue depth threshold upto which
149 	 * packets will get buffered with no flow-control
150 	 * back pressure applied to the caller. Once this
151 	 * threshold is reached, back pressure will be
152 	 * applied to the caller of mac_tx() (mac_tx() starts
153 	 * returning a cookie to indicate a blocked SRS).
154 	 * st_hiwat should always be lesser than or equal to
155 	 * st_max_q_cnt.
156 	 */
157 	uint32_t	st_hiwat;	/* mblk cnt to apply flow control */
158 	uint32_t	st_lowat;	/* mblk cnt to relieve flow control */
159 	uint32_t	st_drop_count;
160 	/*
161 	 * Number of times the srs gets blocked due to lack of Tx
162 	 * desc is noted down. Corresponding wakeup from driver
163 	 * to unblock is also noted down. They should match in a
164 	 * correctly working setup. If there is less unblocks
165 	 * than blocks, then Tx side waits forever for a wakeup
166 	 * from below. The following protected by srs_lock.
167 	 */
168 	uint32_t	st_blocked_cnt; /* times blocked for Tx descs */
169 	uint32_t	st_unblocked_cnt; /* unblock calls from driver */
170 	uint32_t	st_hiwat_cnt; /* times blocked for Tx descs */
171 } mac_srs_tx_t;
172 
173 /* Receive side Soft Ring Set */
174 typedef struct mac_srs_rx_s {
175 	/*
176 	 * Upcall Function for fanout, Rx processing etc. Perhaps
177 	 * the same 3 members below can be used for Tx
178 	 * processing, but looking around, mac_rx_func_t has
179 	 * proliferated too much into various files at different
180 	 * places. I am leaving the consolidation battle for
181 	 * another day.
182 	 */
183 	mac_direct_rx_t		sr_func;	/* srs_lock */
184 	void			*sr_arg1;	/* srs_lock */
185 	mac_resource_handle_t 	sr_arg2;	/* srs_lock */
186 	mac_rx_func_t		sr_lower_proc;	/* Atomically changed */
187 	uint32_t		sr_poll_pkt_cnt;
188 	uint32_t		sr_poll_thres;
189 
190 	/* mblk cnt to apply flow control */
191 	uint32_t		sr_hiwat;
192 	/* mblk cnt to relieve flow control */
193 	uint32_t		sr_lowat;
194 	uint32_t		sr_poll_count;
195 	uint32_t		sr_intr_count;
196 	uint32_t		sr_drop_count;
197 
198 	/* Times polling was enabled */
199 	uint32_t		sr_poll_on;
200 	/* Times polling was enabled by worker thread */
201 	uint32_t		sr_worker_poll_on;
202 	/* Times polling was disabled */
203 	uint32_t		sr_poll_off;
204 	/* Poll thread signalled count */
205 	uint32_t		sr_poll_thr_sig;
206 	/* Poll thread busy */
207 	uint32_t		sr_poll_thr_busy;
208 	/* SRS drains, stays in poll mode but doesn't poll */
209 	uint32_t		sr_poll_drain_no_poll;
210 	/*
211 	 * SRS has nothing to do and no packets in H/W but
212 	 * there is a backlog in softrings. SRS stays in
213 	 * poll mode but doesn't do polling.
214 	 */
215 	uint32_t		sr_poll_no_poll;
216 	/* Active polling restarted */
217 	uint32_t		sr_below_hiwat;
218 	/* Found packets in last poll so try and poll again */
219 	uint32_t		sr_poll_again;
220 	/*
221 	 * Packets in queue but poll thread not allowed to process so
222 	 * signal the worker thread.
223 	 */
224 	uint32_t		sr_poll_sig_worker;
225 	/*
226 	 * Poll thread has nothing to do and H/W has nothing so
227 	 * reenable the interrupts.
228 	 */
229 	uint32_t		sr_poll_intr_enable;
230 	/*
231 	 * Poll thread has nothing to do and worker thread was already
232 	 * running so it can decide to reenable interrupt or poll again.
233 	 */
234 	uint32_t		sr_poll_goto_sleep;
235 	/* Worker thread goes back to draining the queue */
236 	uint32_t		sr_drain_again;
237 	/* More Packets in queue so signal the poll thread to drain */
238 	uint32_t		sr_drain_poll_sig;
239 	/* More Packets in queue so signal the worker thread to drain */
240 	uint32_t		sr_drain_worker_sig;
241 	/* Poll thread is already running so worker has nothing to do */
242 	uint32_t		sr_drain_poll_running;
243 	/* We have packets already queued so keep polling */
244 	uint32_t		sr_drain_keep_polling;
245 	/* Drain is done and interrupts are reenabled */
246 	uint32_t		sr_drain_finish_intr;
247 	/* Polling thread needs to schedule worker wakeup */
248 	uint32_t		sr_poll_worker_wakeup;
249 
250 	/* Chains less than 10 pkts */
251 	uint32_t		sr_chain_cnt_undr10;
252 	/* Chains between 10 & 50 pkts */
253 	uint32_t		sr_chain_cnt_10to50;
254 	/* Chains over 50 pkts */
255 	uint32_t		sr_chain_cnt_over50;
256 } mac_srs_rx_t;
257 
258 /*
259  * mac_soft_ring_set_s:
260  * This is used both for Tx and Rx side. The srs_type identifies Rx or
261  * Tx type.
262  *
263  * Note that the structure is carefully crafted, with Rx elements coming
264  * first followed by Tx specific members. Future additions to this
265  * structure should follow the same guidelines.
266  *
267  * Rx-side notes:
268  * mac_rx_classify_flow_add() always creates a mac_soft_ring_set_t and fn_flow
269  * points to info from it (func = srs_lower_proc, arg = soft_ring_set). On
270  * interrupt path, srs_lower_proc does B/W adjustment and switch to polling mode
271  * (if poll capable) and feeds the packets to soft_ring_list via choosen
272  * fanout type (specified by srs_type). In poll mode, the poll thread which is
273  * also a pointer can pick up the packets and feed them to various
274  * soft_ring_list.
275  *
276  * The srs_type can either be protocol based or fanout based where fanout itelf
277  * can be various types
278  *
279  * The polling works by turning off interrupts as soon as a packets
280  * are queued on the soft ring set. Once the backlog is clear and poll
281  * thread return empty handed i.e. Rx ring doesn't have anything, the
282  * interrupt is turned back on. For this purpose we keep a separate
283  * srs_poll_pkt_cnt counter which tracks the packets queued between SRS
284  * and the soft rings as well. The counter is incremented when packets
285  * are queued and decremented when SRS processes them (in case it has
286  * no soft rings) or the soft ring process them. Its important that
287  * in case SRS has softrings, the decrement doesn't happen till the
288  * packet is processed by the soft rings since it takes very little time
289  * for SRS to queue packet from SRS to soft rings and it will keep
290  * bringing more packets in the system faster than soft rings can
291  * process them.
292  *
293  * Tx side notes:
294  * The srs structure acts as a serializer with a worker thread. The
295  * default behavior of srs though is to act as a pass-thru. The queues
296  * (srs_first, srs_last, srs_count) get used when Tx ring runs out of Tx
297  * descriptors or to enforce bandwidth limits.
298  *
299  * When multiple Tx rings are present, the SRS state will be set to
300  * SRS_FANOUT_OTH. Outgoing packets coming into mac_tx_srs_process()
301  * function will be fanned out to one of the Tx side soft rings based on
302  * a hint passed in mac_tx_srs_process(). Each soft ring, in turn, will
303  * be associated with a distinct h/w Tx ring.
304  */
305 
306 struct mac_soft_ring_set_s {
307 	/*
308 	 * Common elements, common to both Rx and Tx SRS type.
309 	 * The following block of fields are protected by srs_lock
310 	 */
311 	kmutex_t	srs_lock;
312 	uint32_t	srs_type;
313 	uint32_t	srs_state;	/* state flags */
314 	uint32_t	srs_count;
315 	mblk_t		*srs_first;	/* first mblk chain or NULL */
316 	mblk_t		*srs_last;	/* last mblk chain or NULL */
317 	kcondvar_t	srs_async;	/* cv for worker thread */
318 	kcondvar_t	srs_cv;		/* cv for poll thread */
319 	kcondvar_t	srs_quiesce_done_cv;	/* cv for removal */
320 	timeout_id_t	srs_tid;	/* timeout id for pending timeout */
321 
322 	/*
323 	 * List of soft rings & processing function.
324 	 * The following block is protected by Rx quiescence.
325 	 * i.e. they can be changed only after quiescing the SRS
326 	 * Protected by srs_lock.
327 	 */
328 	mac_soft_ring_t	*srs_soft_ring_head;
329 	mac_soft_ring_t	*srs_soft_ring_tail;
330 	int		srs_soft_ring_count;
331 	int		srs_soft_ring_quiesced_count;
332 	int		srs_soft_ring_condemned_count;
333 	mac_soft_ring_t	**srs_tcp_soft_rings;
334 	int		srs_tcp_ring_count;
335 	mac_soft_ring_t	**srs_udp_soft_rings;
336 	int		srs_udp_ring_count;
337 	/*
338 	 * srs_oth_soft_rings is also used by tx_srs in
339 	 * when operating in multi tx ring mode.
340 	 */
341 	mac_soft_ring_t	**srs_oth_soft_rings;
342 	int		srs_oth_ring_count;
343 
344 	/*
345 	 * Bandwidth control related members.
346 	 * They are common to both Rx- and Tx-side.
347 	 * Following protected by srs_lock
348 	 */
349 	mac_bw_ctl_t	*srs_bw;
350 	size_t		srs_size;	/* Size of packets queued in bytes */
351 	pri_t		srs_pri;
352 
353 	mac_soft_ring_set_t	*srs_next;	/* mac_srs_g_lock */
354 	mac_soft_ring_set_t	*srs_prev;	/* mac_srs_g_lock */
355 
356 	/* Attribute specific drain func (BW ctl vs non-BW ctl)	*/
357 	mac_srs_drain_proc_t	srs_drain_func;	/* Write once (WO) */
358 
359 	/*
360 	 * If the associated ring is exclusively used by a mac client, e.g.,
361 	 * an aggregation, this fields is used to keep a reference to the
362 	 * MAC client's pseudo ring.
363 	 */
364 	mac_resource_handle_t	srs_mrh;
365 	/*
366 	 * The following blocks are write once (WO) and valid for the life
367 	 * of the SRS
368 	 */
369 	struct mac_client_impl_s *srs_mcip;	/* back ptr to mac client */
370 	void			*srs_flent;	/* back ptr to flent */
371 	mac_ring_t		*srs_ring;	/*  Ring Descriptor */
372 
373 	/* Teardown, disable control ops */
374 	kcondvar_t	srs_client_cv;	/* Client wait for the control op */
375 
376 	kthread_t	*srs_worker;	/* WO, worker thread */
377 	kthread_t	*srs_poll_thr;	/* WO, poll thread */
378 
379 	uint_t		srs_ind;	/* Round Robin indx for picking up SR */
380 	processorid_t	srs_worker_cpuid;	/* processor to bind to */
381 	processorid_t	srs_worker_cpuid_save;	/* saved cpuid during offline */
382 	processorid_t	srs_poll_cpuid;		/* processor to bind to */
383 	processorid_t	srs_poll_cpuid_save;	/* saved cpuid during offline */
384 	uint_t		srs_fanout_state;
385 	mac_cpus_t	srs_cpu;
386 
387 	mac_srs_rx_t	srs_rx;
388 	mac_srs_tx_t	srs_tx;
389 };
390 
391 /*
392  * type flags - combination allowed to process and drain the queue
393  */
394 #define	ST_RING_WORKER_ONLY  	0x0001	/* Worker thread only */
395 #define	ST_RING_ANY		0x0002	/* Any thread can process the queue */
396 #define	ST_RING_TCP		0x0004
397 #define	ST_RING_UDP		0x0008
398 #define	ST_RING_OTH		0x0010
399 
400 #define	ST_RING_BW_CTL		0x0020
401 #define	ST_RING_TX		0x0040
402 
403 /*
404  * State flags.
405  */
406 #define	S_RING_PROC		0x0001	/* being processed */
407 #define	S_RING_BOUND		0x0002	/* Worker thread is bound to a cpu */
408 #define	S_RING_BLOCK		0x0004	/* No Tx descs */
409 #define	S_RING_TX_HIWAT		0x0008	/* Tx high watermark reached */
410 
411 #define	S_RING_WAKEUP_CLIENT	0x0010	/* flow ctrl, client wakeup needed */
412 #define	S_RING_BLANK		0x0020	/* Has been put into polling mode */
413 #define	S_RING_CLIENT_WAIT	0x0040	/* Client waiting for control op */
414 
415 #define	S_RING_CONDEMNED	0x0100	/* Being torn down */
416 #define	S_RING_CONDEMNED_DONE	0x0200	/* Being torn down */
417 #define	S_RING_QUIESCE		0x0400	/* No traffic flow, transient flag */
418 #define	S_RING_QUIESCE_DONE	0x0800	/* No traffic flow, transient flag */
419 
420 #define	S_RING_RESTART		0x1000	/* Go back to normal traffic flow */
421 #define	S_RING_ENQUEUED		0x2000	/* Pkts enqueued in Tx soft ring */
422 
423 /*
424  * arguments for processors to bind to
425  */
426 #define	S_RING_BIND_NONE	-1
427 
428 /*
429  * defines for srs_type - identifies a link or a sub-flow
430  * and other static characteristics of a SRS like a tx
431  * srs, tcp only srs, etc.
432  */
433 #define	SRST_LINK		0x00000001
434 #define	SRST_FLOW		0x00000002
435 #define	SRST_NO_SOFT_RINGS	0x00000004
436 #define	SRST_TCP_ONLY		0x00000008
437 
438 #define	SRST_FANOUT_PROTO	0x00000010
439 #define	SRST_FANOUT_SRC_IP	0x00000020
440 #define	SRST_FANOUT_OTH		0x00000040
441 #define	SRST_DEFAULT_GRP	0x00000080
442 
443 #define	SRST_TX			0x00000100
444 #define	SRST_BW_CONTROL		0x00000200
445 #define	SRST_DIRECT_POLL	0x00000400
446 
447 #define	SRST_DLS_BYPASS		0x00001000
448 #define	SRST_CLIENT_POLL_ENABLED 0x00002000
449 
450 /*
451  * soft ring set flags. These bits are dynamic in nature and get
452  * applied to srs_state. They reflect the state of SRS at any
453  * point of time
454  */
455 #define	SRS_BLANK		0x00000001
456 #define	SRS_WORKER_BOUND	0x00000002
457 #define	SRS_POLL_BOUND		0x00000004
458 #define	SRS_POLLING_CAPAB	0x00000008
459 
460 #define	SRS_PROC		0x00000010
461 #define	SRS_GET_PKTS		0x00000020
462 #define	SRS_POLLING		0x00000040
463 #define	SRS_BW_ENFORCED		0x00000080
464 
465 #define	SRS_WORKER		0x00000100
466 #define	SRS_ENQUEUED		0x00000200
467 #define	SRS_ANY_PROCESS		0x00000400
468 #define	SRS_PROC_FAST		0x00000800
469 
470 #define	SRS_POLL_PROC		0x00001000
471 #define	SRS_TX_BLOCKED		0x00002000	/* out of Tx descs */
472 #define	SRS_TX_HIWAT		0x00004000	/* Tx count exceeds hiwat */
473 #define	SRS_TX_WAKEUP_CLIENT	0x00008000	/* Flow-ctl: wakeup client */
474 
475 #define	SRS_CLIENT_PROC		0x00010000
476 #define	SRS_CLIENT_WAIT		0x00020000
477 #define	SRS_QUIESCE		0x00040000
478 #define	SRS_QUIESCE_DONE	0x00080000
479 
480 #define	SRS_CONDEMNED		0x00100000
481 #define	SRS_CONDEMNED_DONE	0x00200000
482 #define	SRS_POLL_THR_QUIESCED	0x00400000
483 #define	SRS_RESTART		0x00800000
484 
485 #define	SRS_RESTART_DONE	0x01000000
486 #define	SRS_POLL_THR_RESTART	0x02000000
487 #define	SRS_IN_GLIST		0x04000000
488 #define	SRS_POLL_THR_EXITED	0x08000000
489 
490 #define	SRS_QUIESCE_PERM	0x10000000
491 #define	SRS_LATENCY_OPT		0x20000000
492 #define	SRS_SOFTRING_QUEUE	0x40000000
493 
494 #define	SRS_QUIESCED(srs)	(srs->srs_state & SRS_QUIESCE_DONE)
495 
496 /*
497  * If the SRS_QUIESCE_PERM flag is set, the SRS worker thread will not be
498  * able to be restarted.
499  */
500 #define	SRS_QUIESCED_PERMANENT(srs)	(srs->srs_state & SRS_QUIESCE_PERM)
501 
502 /*
503  * soft ring set (SRS) Tx modes
504  */
505 typedef enum {
506 	SRS_TX_DEFAULT = 0,
507 	SRS_TX_SERIALIZE,
508 	SRS_TX_FANOUT,
509 	SRS_TX_BW,
510 	SRS_TX_BW_FANOUT
511 } mac_tx_srs_mode_t;
512 
513 /*
514  * SRS fanout states
515  */
516 typedef enum {
517 	SRS_FANOUT_UNINIT = 0,
518 	SRS_FANOUT_INIT,
519 	SRS_FANOUT_REINIT
520 } mac_srs_fanout_state_t;
521 
522 /*
523  * Structure for dls statistics
524  */
525 struct dls_kstats {
526 	kstat_named_t	dlss_soft_ring_pkt_drop;
527 };
528 
529 extern struct dls_kstats dls_kstat;
530 
531 #define	DLS_BUMP_STAT(x, y)	(dls_kstat.x.value.ui32 += y)
532 
533 /* Turn dynamic polling off */
534 #define	MAC_SRS_POLLING_OFF(mac_srs) {					\
535 	ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock));			\
536 	if (((mac_srs)->srs_state & (SRS_POLLING_CAPAB|SRS_POLLING)) == \
537 	    (SRS_POLLING_CAPAB|SRS_POLLING)) {				\
538 		(mac_srs)->srs_state &= ~SRS_POLLING;			\
539 		(void) mac_hwring_enable_intr((mac_ring_handle_t)	\
540 		    (mac_srs)->srs_ring);				\
541 		(mac_srs)->srs_rx.sr_poll_off++;			\
542 	}								\
543 }
544 
545 #define	MAC_COUNT_CHAIN(mac_srs, head, tail, cnt, sz)	{	\
546 	mblk_t 		*tmp;		       			\
547 	boolean_t	bw_ctl = B_FALSE;			\
548 								\
549 	ASSERT((head) != NULL);					\
550 	cnt = 0;						\
551 	sz = 0;							\
552 	if ((mac_srs)->srs_type & SRST_BW_CONTROL)		\
553 		bw_ctl = B_TRUE;				\
554 	tmp = tail = (head);					\
555 	if ((head)->b_next == NULL) {				\
556 		cnt = 1;					\
557 		if (bw_ctl)					\
558 			sz += msgdsize(head);			\
559 	} else {						\
560 		while (tmp != NULL) {				\
561 			tail = tmp;				\
562 			cnt++;					\
563 			if (bw_ctl)				\
564 				sz += msgdsize(tmp);		\
565 			tmp = tmp->b_next;			\
566 		}						\
567 	}							\
568 }
569 
570 /*
571  * Decrement the cumulative packet count in SRS and its
572  * soft rings. If the srs_poll_pkt_cnt goes below lowat, then check
573  * if if the interface was left in a polling mode and no one
574  * is really processing the queue (to get the interface out
575  * of poll mode). If no one is processing the queue, then
576  * acquire the PROC and signal the poll thread to check the
577  * interface for packets and get the interface back to interrupt
578  * mode if nothing is found.
579  */
580 #define	MAC_UPDATE_SRS_COUNT_LOCKED(mac_srs, cnt) {		        \
581 	mac_srs_rx_t	*srs_rx = &(mac_srs)->srs_rx;			\
582 	ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock));			\
583 									\
584 	srs_rx->sr_poll_pkt_cnt -= cnt;					\
585 	if ((srs_rx->sr_poll_pkt_cnt <= srs_rx->sr_poll_thres) && 	\
586 		(((mac_srs)->srs_state &				\
587 		(SRS_POLLING|SRS_PROC|SRS_GET_PKTS)) == SRS_POLLING))	\
588 	{								\
589 		(mac_srs)->srs_state |= (SRS_PROC|SRS_GET_PKTS);	\
590 		cv_signal(&(mac_srs)->srs_cv); 				\
591 		srs_rx->sr_below_hiwat++;				\
592 	}								\
593 }
594 
595 /*
596  * The following two macros are used to update the inbound packet and byte.
597  * count. The packet and byte count reflect the packets and bytes that are
598  * taken out of the SRS's queue, i.e. indicating they are being delivered.
599  * The srs_count and srs_size are updated in different locations as the
600  * srs_size is also used to take into account any bandwidth limits. The
601  * srs_size is updated only when a soft ring, if any, sends a packet up,
602  * as opposed to updating it when the SRS sends a packet to the SR, i.e.
603  * the srs_size reflects the packets in the SRS and SRs. These
604  * macros decrement the srs_size and srs_count and also increment the
605  * ipackets and ibytes stats resp.
606  *
607  * xxx-venu These are done under srs_lock, for now we still update
608  * mci_stat_ibytes/mci_stat_ipackets atomically, need to check if
609  * just updating them would be accurate enough.
610  *
611  * If we are updating these for a sub-flow SRS, then we need to also
612  * updated it's MAC client bandwidth info, if the MAC client is also
613  * bandwidth regulated.
614  */
615 #define	MAC_UPDATE_SRS_SIZE_LOCKED(srs, sz) {				\
616 	if ((srs)->srs_type & SRST_BW_CONTROL) {			\
617 		mutex_enter(&(srs)->srs_bw->mac_bw_lock);		\
618 		(srs)->srs_bw->mac_bw_sz -= (sz);			\
619 		(srs)->srs_bw->mac_bw_used += (sz);			\
620 		mutex_exit(&(srs)->srs_bw->mac_bw_lock);		\
621 	}								\
622 }
623 
624 #define	MAC_TX_UPDATE_BW_INFO(srs, sz) {				\
625 	(srs)->srs_bw->mac_bw_sz -= (sz);				\
626 	(srs)->srs_bw->mac_bw_used += (sz);				\
627 }
628 
629 #define	TX_MULTI_RING_MODE(mac_srs)				\
630 	((mac_srs)->srs_tx.st_mode == SRS_TX_FANOUT || 		\
631 	    (mac_srs)->srs_tx.st_mode == SRS_TX_BW_FANOUT)
632 
633 /* Soft ring flags for teardown */
634 #define	SRS_POLL_THR_OWNER	(SRS_PROC | SRS_POLLING | SRS_GET_PKTS)
635 #define	SRS_PAUSE		(SRS_CONDEMNED | SRS_QUIESCE)
636 #define	S_RING_PAUSE		(S_RING_CONDEMNED | S_RING_QUIESCE)
637 
638 /* Soft rings */
639 extern void mac_soft_ring_init(void);
640 extern void mac_soft_ring_finish(void);
641 extern void mac_fanout_setup(mac_client_impl_t *, flow_entry_t *,
642     mac_resource_props_t *, mac_direct_rx_t, void *, mac_resource_handle_t);
643 
644 extern void mac_soft_ring_worker_wakeup(mac_soft_ring_t *);
645 extern void mac_soft_ring_blank(void *, time_t, uint_t, int);
646 extern mblk_t *mac_soft_ring_poll(mac_soft_ring_t *, int);
647 extern void mac_soft_ring_destroy(mac_soft_ring_t *);
648 extern void mac_soft_ring_dls_bypass(void *, mac_direct_rx_t, void *);
649 
650 /* Rx SRS */
651 extern mac_soft_ring_set_t *mac_srs_create(struct mac_client_impl_s *,
652     flow_entry_t *, uint32_t, mac_direct_rx_t, void *, mac_resource_handle_t,
653     mac_ring_t *);
654 extern void mac_srs_free(mac_soft_ring_set_t *);
655 extern void mac_srs_signal(mac_soft_ring_set_t *, uint_t);
656 extern cpu_t *mac_srs_bind(mac_soft_ring_set_t *, processorid_t);
657 
658 extern void mac_srs_change_upcall(void *, mac_direct_rx_t, void *);
659 extern void mac_srs_quiesce_initiate(mac_soft_ring_set_t *);
660 extern void mac_srs_client_poll_enable(struct mac_client_impl_s *,
661     mac_soft_ring_set_t *);
662 extern void mac_srs_client_poll_disable(struct mac_client_impl_s *,
663     mac_soft_ring_set_t *);
664 extern void mac_srs_client_poll_quiesce(struct mac_client_impl_s *,
665     mac_soft_ring_set_t *);
666 extern void mac_srs_client_poll_restart(struct mac_client_impl_s *,
667     mac_soft_ring_set_t *);
668 extern void mac_rx_srs_quiesce(mac_soft_ring_set_t *, uint_t);
669 extern void mac_rx_srs_restart(mac_soft_ring_set_t *);
670 extern void mac_rx_srs_subflow_process(void *, mac_resource_handle_t, mblk_t *,
671     boolean_t);
672 extern void mac_tx_srs_quiesce(mac_soft_ring_set_t *, uint_t);
673 
674 /* Tx SRS, Tx softring */
675 extern void mac_tx_srs_wakeup(mac_soft_ring_set_t *, mac_ring_handle_t);
676 extern void mac_tx_srs_setup(struct mac_client_impl_s *,
677     flow_entry_t *, uint32_t);
678 extern mac_tx_func_t mac_tx_get_func(uint32_t);
679 extern mblk_t *mac_tx_send(mac_client_handle_t, mac_ring_handle_t, mblk_t *,
680     mac_tx_stats_t *);
681 extern boolean_t mac_tx_srs_ring_present(mac_soft_ring_set_t *, mac_ring_t *);
682 extern void mac_tx_srs_add_ring(mac_soft_ring_set_t *, mac_ring_t *);
683 extern void mac_tx_srs_del_ring(mac_soft_ring_set_t *, mac_ring_t *);
684 extern mac_tx_cookie_t mac_tx_srs_no_desc(mac_soft_ring_set_t *, mblk_t *,
685     uint16_t, mblk_t **);
686 
687 /* Subflow specific stuff */
688 extern int mac_srs_flow_create(struct mac_client_impl_s *, flow_entry_t *,
689     mac_resource_props_t *, int, int, mac_direct_rx_t);
690 extern void mac_srs_update_bwlimit(flow_entry_t *, mac_resource_props_t *);
691 extern void mac_srs_adjust_subflow_bwlimit(struct mac_client_impl_s *);
692 extern void mac_srs_update_drv(struct mac_client_impl_s *);
693 extern void mac_update_srs_priority(mac_soft_ring_set_t *, pri_t);
694 extern void mac_client_update_classifier(mac_client_impl_t *, boolean_t);
695 
696 extern void mac_soft_ring_intr_enable(void *);
697 extern boolean_t mac_soft_ring_intr_disable(void *);
698 extern mac_soft_ring_t *mac_soft_ring_create(int, clock_t, void *, uint16_t,
699     pri_t, mac_client_impl_t *, mac_soft_ring_set_t *,
700     processorid_t, mac_direct_rx_t, void *, mac_resource_handle_t);
701 extern cpu_t *mac_soft_ring_bind(mac_soft_ring_t *, processorid_t);
702 	extern void mac_soft_ring_unbind(mac_soft_ring_t *);
703 extern void mac_soft_ring_free(mac_soft_ring_t *, boolean_t);
704 extern void mac_soft_ring_signal(mac_soft_ring_t *, uint_t);
705 extern void mac_rx_soft_ring_process(mac_client_impl_t *, mac_soft_ring_t *,
706     mblk_t *, mblk_t *, int, size_t);
707 extern mac_tx_cookie_t mac_tx_soft_ring_process(mac_soft_ring_t *,
708     mblk_t *, uint16_t, mblk_t **);
709 extern void mac_srs_worker_quiesce(mac_soft_ring_set_t *);
710 extern void mac_srs_worker_restart(mac_soft_ring_set_t *);
711 extern void mac_rx_attach_flow_srs(mac_impl_t *, flow_entry_t *,
712     mac_soft_ring_set_t *, mac_ring_t *, mac_classify_type_t);
713 
714 extern void mac_rx_srs_drain_bw(mac_soft_ring_set_t *, uint_t);
715 extern void mac_rx_srs_drain(mac_soft_ring_set_t *, uint_t);
716 extern void mac_rx_srs_process(void *, mac_resource_handle_t, mblk_t *,
717     boolean_t);
718 extern void mac_srs_worker(mac_soft_ring_set_t *);
719 extern void mac_rx_srs_poll_ring(mac_soft_ring_set_t *);
720 extern void mac_tx_srs_drain(mac_soft_ring_set_t *, uint_t);
721 
722 extern void mac_tx_srs_restart(mac_soft_ring_set_t *);
723 extern void mac_rx_srs_remove(mac_soft_ring_set_t *);
724 
725 #ifdef	__cplusplus
726 }
727 #endif
728 
729 #endif	/* _SYS_MAC_SOFT_RING_H */
730