xref: /titanic_41/usr/src/uts/common/io/mac/mac_sched.c (revision fe1c642d06e14b412cd83ae2179303186ab08972)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #include <sys/types.h>
27 #include <sys/callb.h>
28 #include <sys/sdt.h>
29 #include <sys/strsubr.h>
30 #include <sys/strsun.h>
31 #include <sys/vlan.h>
32 #include <inet/ipsec_impl.h>
33 #include <inet/ip_impl.h>
34 #include <inet/sadb.h>
35 #include <inet/ipsecesp.h>
36 #include <inet/ipsecah.h>
37 #include <inet/ip6.h>
38 
39 #include <sys/mac_impl.h>
40 #include <sys/mac_client_impl.h>
41 #include <sys/mac_client_priv.h>
42 #include <sys/mac_soft_ring.h>
43 #include <sys/mac_flow_impl.h>
44 
45 static mac_tx_cookie_t mac_tx_single_ring_mode(mac_soft_ring_set_t *, mblk_t *,
46     uintptr_t, uint16_t, mblk_t **);
47 static mac_tx_cookie_t mac_tx_serializer_mode(mac_soft_ring_set_t *, mblk_t *,
48     uintptr_t, uint16_t, mblk_t **);
49 static mac_tx_cookie_t mac_tx_fanout_mode(mac_soft_ring_set_t *, mblk_t *,
50     uintptr_t, uint16_t, mblk_t **);
51 static mac_tx_cookie_t mac_tx_bw_mode(mac_soft_ring_set_t *, mblk_t *,
52     uintptr_t, uint16_t, mblk_t **);
53 
54 typedef struct mac_tx_mode_s {
55 	mac_tx_srs_mode_t	mac_tx_mode;
56 	mac_tx_func_t		mac_tx_func;
57 } mac_tx_mode_t;
58 
59 /*
60  * There are five modes of operation on the Tx side. These modes get set
61  * in mac_tx_srs_setup(). Except for the experimental TX_SERIALIZE mode,
62  * none of the other modes are user configurable. They get selected by
63  * the system depending upon whether the link (or flow) has multiple Tx
64  * rings or a bandwidth configured, etc.
65  */
66 mac_tx_mode_t mac_tx_mode_list[] = {
67 	{SRS_TX_DEFAULT,	mac_tx_single_ring_mode},
68 	{SRS_TX_SERIALIZE,	mac_tx_serializer_mode},
69 	{SRS_TX_FANOUT,		mac_tx_fanout_mode},
70 	{SRS_TX_BW,		mac_tx_bw_mode},
71 	{SRS_TX_BW_FANOUT,	mac_tx_bw_mode}
72 };
73 
74 /*
75  * Soft Ring Set (SRS) - The Run time code that deals with
76  * dynamic polling from the hardware, bandwidth enforcement,
77  * fanout etc.
78  *
79  * We try to use H/W classification on NIC and assign traffic for
80  * a MAC address to a particular Rx ring or ring group. There is a
81  * 1-1 mapping between a SRS and a Rx ring. The SRS dynamically
82  * switches the underlying Rx ring between interrupt and
83  * polling mode and enforces any specified B/W control.
84  *
85  * There is always a SRS created and tied to each H/W and S/W rule.
86  * Whenever we create a H/W rule, we always add the the same rule to
87  * S/W classifier and tie a SRS to it.
88  *
89  * In case a B/W control is specified, it is broken into bytes
90  * per ticks and as soon as the quota for a tick is exhausted,
91  * the underlying Rx ring is forced into poll mode for remainder of
92  * the tick. The SRS poll thread only polls for bytes that are
93  * allowed to come in the SRS. We typically let 4x the configured
94  * B/W worth of packets to come in the SRS (to prevent unnecessary
95  * drops due to bursts) but only process the specified amount.
96  *
97  * A MAC client (e.g. a VNIC or aggr) can have 1 or more
98  * Rx rings (and corresponding SRSs) assigned to it. The SRS
99  * in turn can have softrings to do protocol level fanout or
100  * softrings to do S/W based fanout or both. In case the NIC
101  * has no Rx rings, we do S/W classification to respective SRS.
102  * The S/W classification rule is always setup and ready. This
103  * allows the MAC layer to reassign Rx rings whenever needed
104  * but packets still continue to flow via the default path and
105  * getting S/W classified to correct SRS.
106  *
107  * The SRS's are used on both Tx and Rx side. They use the same
108  * data structure but the processing routines have slightly different
109  * semantics due to the fact that Rx side needs to do dynamic
110  * polling etc.
111  *
112  * Dynamic Polling Notes
113  * =====================
114  *
115  * Each Soft ring set is capable of switching its Rx ring between
116  * interrupt and poll mode and actively 'polls' for packets in
117  * poll mode. If the SRS is implementing a B/W limit, it makes
118  * sure that only Max allowed packets are pulled in poll mode
119  * and goes to poll mode as soon as B/W limit is exceeded. As
120  * such, there are no overheads to implement B/W limits.
121  *
122  * In poll mode, its better to keep the pipeline going where the
123  * SRS worker thread keeps processing packets and poll thread
124  * keeps bringing more packets (specially if they get to run
125  * on different CPUs). This also prevents the overheads associated
126  * by excessive signalling (on NUMA machines, this can be
127  * pretty devastating). The exception is latency optimized case
128  * where worker thread does no work and interrupt and poll thread
129  * are allowed to do their own drain.
130  *
131  * We use the following policy to control Dynamic Polling:
132  * 1) We switch to poll mode anytime the processing
133  *    thread causes a backlog to build up in SRS and
134  *    its associated Soft Rings (sr_poll_pkt_cnt > 0).
135  * 2) As long as the backlog stays under the low water
136  *    mark (sr_lowat), we poll the H/W for more packets.
137  * 3) If the backlog (sr_poll_pkt_cnt) exceeds low
138  *    water mark, we stay in poll mode but don't poll
139  *    the H/W for more packets.
140  * 4) Anytime in polling mode, if we poll the H/W for
141  *    packets and find nothing plus we have an existing
142  *    backlog (sr_poll_pkt_cnt > 0), we stay in polling
143  *    mode but don't poll the H/W for packets anymore
144  *    (let the polling thread go to sleep).
145  * 5) Once the backlog is relived (packets are processed)
146  *    we reenable polling (by signalling the poll thread)
147  *    only when the backlog dips below sr_poll_thres.
148  * 6) sr_hiwat is used exclusively when we are not
149  *    polling capable and is used to decide when to
150  *    drop packets so the SRS queue length doesn't grow
151  *    infinitely.
152  *
153  * NOTE: Also see the block level comment on top of mac_soft_ring.c
154  */
155 
156 /*
157  * mac_latency_optimize
158  *
159  * Controls whether the poll thread can process the packets inline
160  * or let the SRS worker thread do the processing. This applies if
161  * the SRS was not being processed. For latency sensitive traffic,
162  * this needs to be true to allow inline processing. For throughput
163  * under load, this should be false.
164  *
165  * This (and other similar) tunable should be rolled into a link
166  * or flow specific workload hint that can be set using dladm
167  * linkprop (instead of multiple such tunables).
168  */
169 boolean_t mac_latency_optimize = B_TRUE;
170 
171 /*
172  * MAC_RX_SRS_ENQUEUE_CHAIN and MAC_TX_SRS_ENQUEUE_CHAIN
173  *
174  * queue a mp or chain in soft ring set and increment the
175  * local count (srs_count) for the SRS and the shared counter
176  * (srs_poll_pkt_cnt - shared between SRS and its soft rings
177  * to track the total unprocessed packets for polling to work
178  * correctly).
179  *
180  * The size (total bytes queued) counters are incremented only
181  * if we are doing B/W control.
182  */
183 #define	MAC_SRS_ENQUEUE_CHAIN(mac_srs, head, tail, count, sz) {		\
184 	ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock));			\
185 	if ((mac_srs)->srs_last != NULL)				\
186 		(mac_srs)->srs_last->b_next = (head);			\
187 	else								\
188 		(mac_srs)->srs_first = (head);				\
189 	(mac_srs)->srs_last = (tail);					\
190 	(mac_srs)->srs_count += count;					\
191 }
192 
193 #define	MAC_RX_SRS_ENQUEUE_CHAIN(mac_srs, head, tail, count, sz) {	\
194 	mac_srs_rx_t	*srs_rx = &(mac_srs)->srs_rx;			\
195 									\
196 	MAC_SRS_ENQUEUE_CHAIN(mac_srs, head, tail, count, sz);		\
197 	srs_rx->sr_poll_pkt_cnt += count;				\
198 	ASSERT(srs_rx->sr_poll_pkt_cnt > 0);				\
199 	if ((mac_srs)->srs_type & SRST_BW_CONTROL) {			\
200 		(mac_srs)->srs_size += (sz);				\
201 		mutex_enter(&(mac_srs)->srs_bw->mac_bw_lock);		\
202 		(mac_srs)->srs_bw->mac_bw_sz += (sz);			\
203 		mutex_exit(&(mac_srs)->srs_bw->mac_bw_lock);		\
204 	}								\
205 }
206 
207 #define	MAC_TX_SRS_ENQUEUE_CHAIN(mac_srs, head, tail, count, sz) {	\
208 	mac_srs->srs_state |= SRS_ENQUEUED;				\
209 	MAC_SRS_ENQUEUE_CHAIN(mac_srs, head, tail, count, sz);		\
210 	if ((mac_srs)->srs_type & SRST_BW_CONTROL) {			\
211 		(mac_srs)->srs_size += (sz);				\
212 		(mac_srs)->srs_bw->mac_bw_sz += (sz);			\
213 	}								\
214 }
215 
216 /*
217  * Turn polling on routines
218  */
219 #define	MAC_SRS_POLLING_ON(mac_srs) {					\
220 	ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock));			\
221 	if (((mac_srs)->srs_state &					\
222 	    (SRS_POLLING_CAPAB|SRS_POLLING)) == SRS_POLLING_CAPAB) {	\
223 		(mac_srs)->srs_state |= SRS_POLLING;			\
224 		(void) mac_hwring_disable_intr((mac_ring_handle_t)	\
225 		    (mac_srs)->srs_ring);				\
226 		(mac_srs)->srs_rx.sr_poll_on++;				\
227 	}								\
228 }
229 
230 #define	MAC_SRS_WORKER_POLLING_ON(mac_srs) {				\
231 	ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock));			\
232 	if (((mac_srs)->srs_state &					\
233 	    (SRS_POLLING_CAPAB|SRS_WORKER|SRS_POLLING)) == 		\
234 	    (SRS_POLLING_CAPAB|SRS_WORKER)) {				\
235 		(mac_srs)->srs_state |= SRS_POLLING;			\
236 		(void) mac_hwring_disable_intr((mac_ring_handle_t)	\
237 		    (mac_srs)->srs_ring);				\
238 		(mac_srs)->srs_rx.sr_worker_poll_on++;			\
239 	}								\
240 }
241 
242 /*
243  * MAC_SRS_POLL_RING
244  *
245  * Signal the SRS poll thread to poll the underlying H/W ring
246  * provided it wasn't already polling (SRS_GET_PKTS was set).
247  *
248  * Poll thread gets to run only from mac_rx_srs_drain() and only
249  * if the drain was being done by the worker thread.
250  */
251 #define	MAC_SRS_POLL_RING(mac_srs) {					\
252 	mac_srs_rx_t	*srs_rx = &(mac_srs)->srs_rx;			\
253 									\
254 	ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock));			\
255 	srs_rx->sr_poll_thr_sig++;					\
256 	if (((mac_srs)->srs_state & 					\
257 	    (SRS_POLLING_CAPAB|SRS_WORKER|SRS_GET_PKTS)) ==		\
258 		(SRS_WORKER|SRS_POLLING_CAPAB)) {			\
259 		(mac_srs)->srs_state |= SRS_GET_PKTS;			\
260 		cv_signal(&(mac_srs)->srs_cv);   			\
261 	} else {							\
262 		srs_rx->sr_poll_thr_busy++;				\
263 	}								\
264 }
265 
266 /*
267  * MAC_SRS_CHECK_BW_CONTROL
268  *
269  * Check to see if next tick has started so we can reset the
270  * SRS_BW_ENFORCED flag and allow more packets to come in the
271  * system.
272  */
273 #define	MAC_SRS_CHECK_BW_CONTROL(mac_srs) {				\
274 	ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock));			\
275 	ASSERT(((mac_srs)->srs_type & SRST_TX) ||			\
276 	    MUTEX_HELD(&(mac_srs)->srs_bw->mac_bw_lock));		\
277 	clock_t now = ddi_get_lbolt();					\
278 	if ((mac_srs)->srs_bw->mac_bw_curr_time != now) {		\
279 		(mac_srs)->srs_bw->mac_bw_curr_time = now;		\
280 		(mac_srs)->srs_bw->mac_bw_used = 0;	       		\
281 		if ((mac_srs)->srs_bw->mac_bw_state & SRS_BW_ENFORCED)	\
282 			(mac_srs)->srs_bw->mac_bw_state &= ~SRS_BW_ENFORCED; \
283 	}								\
284 }
285 
286 /*
287  * MAC_SRS_WORKER_WAKEUP
288  *
289  * Wake up the SRS worker thread to process the queue as long as
290  * no one else is processing the queue. If we are optimizing for
291  * latency, we wake up the worker thread immediately or else we
292  * wait mac_srs_worker_wakeup_ticks before worker thread gets
293  * woken up.
294  */
295 int mac_srs_worker_wakeup_ticks = 0;
296 #define	MAC_SRS_WORKER_WAKEUP(mac_srs) {				\
297 	ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock));			\
298 	if (!((mac_srs)->srs_state & SRS_PROC) &&			\
299 		(mac_srs)->srs_tid == NULL) {				\
300 		if (((mac_srs)->srs_state & SRS_LATENCY_OPT) ||		\
301 			(mac_srs_worker_wakeup_ticks == 0))		\
302 			cv_signal(&(mac_srs)->srs_async);		\
303 		else							\
304 			(mac_srs)->srs_tid =				\
305 				timeout(mac_srs_fire, (mac_srs),	\
306 					mac_srs_worker_wakeup_ticks);	\
307 	}								\
308 }
309 
310 #define	TX_SINGLE_RING_MODE(mac_srs)				\
311 	((mac_srs)->srs_tx.st_mode == SRS_TX_DEFAULT || 	\
312 	    (mac_srs)->srs_tx.st_mode == SRS_TX_SERIALIZE ||	\
313 	    (mac_srs)->srs_tx.st_mode == SRS_TX_BW)
314 
315 #define	TX_BANDWIDTH_MODE(mac_srs)				\
316 	((mac_srs)->srs_tx.st_mode == SRS_TX_BW ||		\
317 	    (mac_srs)->srs_tx.st_mode == SRS_TX_BW_FANOUT)
318 
319 #define	TX_SRS_TO_SOFT_RING(mac_srs, head, hint) {			\
320 	uint_t hash, indx;						\
321 	hash = HASH_HINT(hint);					\
322 	indx = COMPUTE_INDEX(hash, mac_srs->srs_oth_ring_count);	\
323 	softring = mac_srs->srs_oth_soft_rings[indx];			\
324 	(void) (mac_tx_soft_ring_process(softring, head, 0, NULL));	\
325 }
326 
327 /*
328  * MAC_TX_SRS_BLOCK
329  *
330  * Always called from mac_tx_srs_drain() function. SRS_TX_BLOCKED
331  * will be set only if srs_tx_woken_up is FALSE. If
332  * srs_tx_woken_up is TRUE, it indicates that the wakeup arrived
333  * before we grabbed srs_lock to set SRS_TX_BLOCKED. We need to
334  * attempt to transmit again and not setting SRS_TX_BLOCKED does
335  * that.
336  */
337 #define	MAC_TX_SRS_BLOCK(srs, mp)	{			\
338 	ASSERT(MUTEX_HELD(&(srs)->srs_lock));			\
339 	if ((srs)->srs_tx.st_woken_up) {			\
340 		(srs)->srs_tx.st_woken_up = B_FALSE;		\
341 	} else {						\
342 		ASSERT(!((srs)->srs_state & SRS_TX_BLOCKED));	\
343 		(srs)->srs_state |= SRS_TX_BLOCKED;		\
344 		(srs)->srs_tx.st_blocked_cnt++;			\
345 	}							\
346 }
347 
348 /*
349  * MAC_TX_SRS_TEST_HIWAT
350  *
351  * Called before queueing a packet onto Tx SRS to test and set
352  * SRS_TX_HIWAT if srs_count exceeds srs_tx_hiwat.
353  */
354 #define	MAC_TX_SRS_TEST_HIWAT(srs, mp, tail, cnt, sz, cookie) {		\
355 	boolean_t enqueue = 1;						\
356 									\
357 	if ((srs)->srs_count > (srs)->srs_tx.st_hiwat) {		\
358 		/*							\
359 		 * flow-controlled. Store srs in cookie so that it	\
360 		 * can be returned as mac_tx_cookie_t to client		\
361 		 */							\
362 		(srs)->srs_state |= SRS_TX_HIWAT;			\
363 		cookie = (mac_tx_cookie_t)srs;				\
364 		(srs)->srs_tx.st_hiwat_cnt++;				\
365 		if ((srs)->srs_count > (srs)->srs_tx.st_max_q_cnt) {	\
366 			/* increment freed stats */			\
367 			(srs)->srs_tx.st_drop_count += cnt;		\
368 			/*						\
369 			 * b_prev may be set to the fanout hint		\
370 			 * hence can't use freemsg directly		\
371 			 */						\
372 			mac_pkt_drop(NULL, NULL, mp_chain, B_FALSE);	\
373 			DTRACE_PROBE1(tx_queued_hiwat,			\
374 			    mac_soft_ring_set_t *, srs);		\
375 			enqueue = 0;					\
376 		}							\
377 	}								\
378 	if (enqueue)							\
379 		MAC_TX_SRS_ENQUEUE_CHAIN(srs, mp, tail, cnt, sz);	\
380 }
381 
382 /* Some utility macros */
383 #define	MAC_SRS_BW_LOCK(srs)						\
384 	if (!(srs->srs_type & SRST_TX))					\
385 		mutex_enter(&srs->srs_bw->mac_bw_lock);
386 
387 #define	MAC_SRS_BW_UNLOCK(srs)						\
388 	if (!(srs->srs_type & SRST_TX))					\
389 		mutex_exit(&srs->srs_bw->mac_bw_lock);
390 
391 #define	MAC_TX_SRS_DROP_MESSAGE(srs, mp, cookie) {		\
392 	mac_pkt_drop(NULL, NULL, mp, B_FALSE);			\
393 	/* increment freed stats */				\
394 	mac_srs->srs_tx.st_drop_count++;			\
395 	cookie = (mac_tx_cookie_t)srs;				\
396 }
397 
398 #define	MAC_TX_SET_NO_ENQUEUE(srs, mp_chain, ret_mp, cookie) {		\
399 	mac_srs->srs_state |= SRS_TX_WAKEUP_CLIENT;			\
400 	cookie = (mac_tx_cookie_t)srs;					\
401 	*ret_mp = mp_chain;						\
402 }
403 
404 /*
405  * Drop the rx packet and advance to the next one in the chain.
406  */
407 static void
408 mac_rx_drop_pkt(mac_soft_ring_set_t *srs, mblk_t *mp)
409 {
410 	mac_srs_rx_t	*srs_rx = &srs->srs_rx;
411 
412 	ASSERT(mp->b_next == NULL);
413 	mutex_enter(&srs->srs_lock);
414 	MAC_UPDATE_SRS_COUNT_LOCKED(srs, 1);
415 	MAC_UPDATE_SRS_SIZE_LOCKED(srs, msgdsize(mp));
416 	mutex_exit(&srs->srs_lock);
417 
418 	srs_rx->sr_drop_count++;
419 	freemsg(mp);
420 }
421 
422 /* DATAPATH RUNTIME ROUTINES */
423 
424 /*
425  * mac_srs_fire
426  *
427  * Timer callback routine for waking up the SRS worker thread.
428  */
429 static void
430 mac_srs_fire(void *arg)
431 {
432 	mac_soft_ring_set_t *mac_srs = (mac_soft_ring_set_t *)arg;
433 
434 	mutex_enter(&mac_srs->srs_lock);
435 	if (mac_srs->srs_tid == 0) {
436 		mutex_exit(&mac_srs->srs_lock);
437 		return;
438 	}
439 
440 	mac_srs->srs_tid = 0;
441 	if (!(mac_srs->srs_state & SRS_PROC))
442 		cv_signal(&mac_srs->srs_async);
443 
444 	mutex_exit(&mac_srs->srs_lock);
445 }
446 
447 /*
448  * 'hint' is fanout_hint (type of uint64_t) which is given by the TCP/IP stack,
449  * and it is used on the TX path.
450  */
451 #define	HASH_HINT(hint)	(((hint) << 17) | ((hint) >> 16))
452 
453 /*
454  * hash based on the src address and the port information.
455  */
456 #define	HASH_ADDR(src, ports)					\
457 	(ntohl((src)) ^ ((ports) >> 24) ^ ((ports) >> 16) ^	\
458 	((ports) >> 8) ^ (ports))
459 
460 #define	COMPUTE_INDEX(key, sz)	(key % sz)
461 
462 #define	FANOUT_ENQUEUE_MP(head, tail, cnt, bw_ctl, sz, sz0, mp) {	\
463 	if ((tail) != NULL) {						\
464 		ASSERT((tail)->b_next == NULL);				\
465 		(tail)->b_next = (mp);					\
466 	} else {							\
467 		ASSERT((head) == NULL);					\
468 		(head) = (mp);						\
469 	}								\
470 	(tail) = (mp);							\
471 	(cnt)++;							\
472 	if ((bw_ctl))							\
473 		(sz) += (sz0);						\
474 }
475 
476 #define	MAC_FANOUT_DEFAULT	0
477 #define	MAC_FANOUT_RND_ROBIN	1
478 int mac_fanout_type = MAC_FANOUT_DEFAULT;
479 
480 #define	MAX_SR_TYPES	3
481 /* fanout types for port based hashing */
482 enum pkt_type {
483 	V4_TCP = 0,
484 	V4_UDP,
485 	OTH,
486 	UNDEF
487 };
488 
489 /*
490  * In general we do port based hashing to spread traffic over different
491  * softrings. The below tunable allows to override that behavior. Setting it
492  * to B_TRUE allows to do a fanout based on src ipv6 address. This behavior
493  * is also the applicable to ipv6 packets carrying multiple optional headers
494  * and other uncommon packet types.
495  */
496 boolean_t mac_src_ipv6_fanout = B_FALSE;
497 
498 /*
499  * Pair of local and remote ports in the transport header
500  */
501 #define	PORTS_SIZE 4
502 
503 /*
504  * mac_rx_srs_proto_fanout
505  *
506  * This routine delivers packets destined to an SRS into one of the
507  * protocol soft rings.
508  *
509  * Given a chain of packets we need to split it up into multiple sub chains
510  * destined into TCP, UDP or OTH soft ring. Instead of entering
511  * the soft ring one packet at a time, we want to enter it in the form of a
512  * chain otherwise we get this start/stop behaviour where the worker thread
513  * goes to sleep and then next packets comes in forcing it to wake up etc.
514  */
515 static void
516 mac_rx_srs_proto_fanout(mac_soft_ring_set_t *mac_srs, mblk_t *head)
517 {
518 	struct ether_header		*ehp;
519 	struct ether_vlan_header	*evhp;
520 	uint32_t			sap;
521 	ipha_t				*ipha;
522 	uint8_t				*dstaddr;
523 	size_t				hdrsize;
524 	mblk_t				*mp;
525 	mblk_t				*headmp[MAX_SR_TYPES];
526 	mblk_t				*tailmp[MAX_SR_TYPES];
527 	int				cnt[MAX_SR_TYPES];
528 	size_t				sz[MAX_SR_TYPES];
529 	size_t				sz1;
530 	boolean_t			bw_ctl;
531 	boolean_t			hw_classified;
532 	boolean_t			dls_bypass;
533 	boolean_t			is_ether;
534 	boolean_t			is_unicast;
535 	enum pkt_type			type;
536 	mac_client_impl_t		*mcip = mac_srs->srs_mcip;
537 
538 	is_ether = (mcip->mci_mip->mi_info.mi_nativemedia == DL_ETHER);
539 	bw_ctl = ((mac_srs->srs_type & SRST_BW_CONTROL) != 0);
540 
541 	/*
542 	 * If we don't have a Rx ring, S/W classification would have done
543 	 * its job and its a packet meant for us. If we were polling on
544 	 * the default ring (i.e. there was a ring assigned to this SRS),
545 	 * then we need to make sure that the mac address really belongs
546 	 * to us.
547 	 */
548 	hw_classified = mac_srs->srs_ring != NULL &&
549 	    mac_srs->srs_ring->mr_classify_type == MAC_HW_CLASSIFIER;
550 
551 	/*
552 	 * Special clients (eg. VLAN, non ether, etc) need DLS
553 	 * processing in the Rx path. SRST_DLS_BYPASS will be clear for
554 	 * such SRSs. Another way of disabling bypass is to set the
555 	 * MCIS_RX_BYPASS_DISABLE flag.
556 	 */
557 	dls_bypass = ((mac_srs->srs_type & SRST_DLS_BYPASS) != 0) &&
558 	    ((mcip->mci_state_flags & MCIS_RX_BYPASS_DISABLE) == 0);
559 
560 	bzero(headmp, MAX_SR_TYPES * sizeof (mblk_t *));
561 	bzero(tailmp, MAX_SR_TYPES * sizeof (mblk_t *));
562 	bzero(cnt, MAX_SR_TYPES * sizeof (int));
563 	bzero(sz, MAX_SR_TYPES * sizeof (size_t));
564 
565 	/*
566 	 * We got a chain from SRS that we need to send to the soft rings.
567 	 * Since squeues for TCP & IPv4 sap poll their soft rings (for
568 	 * performance reasons), we need to separate out v4_tcp, v4_udp
569 	 * and the rest goes in other.
570 	 */
571 	while (head != NULL) {
572 		mp = head;
573 		head = head->b_next;
574 		mp->b_next = NULL;
575 
576 		type = OTH;
577 		sz1 = (mp->b_cont == NULL) ? MBLKL(mp) : msgdsize(mp);
578 
579 		if (is_ether) {
580 			/*
581 			 * At this point we can be sure the packet at least
582 			 * has an ether header.
583 			 */
584 			if (sz1 < sizeof (struct ether_header)) {
585 				mac_rx_drop_pkt(mac_srs, mp);
586 				continue;
587 			}
588 			ehp = (struct ether_header *)mp->b_rptr;
589 
590 			/*
591 			 * Determine if this is a VLAN or non-VLAN packet.
592 			 */
593 			if ((sap = ntohs(ehp->ether_type)) == VLAN_TPID) {
594 				evhp = (struct ether_vlan_header *)mp->b_rptr;
595 				sap = ntohs(evhp->ether_type);
596 				hdrsize = sizeof (struct ether_vlan_header);
597 				/*
598 				 * Check if the VID of the packet, if any,
599 				 * belongs to this client.
600 				 */
601 				if (!mac_client_check_flow_vid(mcip,
602 				    VLAN_ID(ntohs(evhp->ether_tci)))) {
603 					mac_rx_drop_pkt(mac_srs, mp);
604 					continue;
605 				}
606 			} else {
607 				hdrsize = sizeof (struct ether_header);
608 			}
609 			is_unicast =
610 			    ((((uint8_t *)&ehp->ether_dhost)[0] & 0x01) == 0);
611 			dstaddr = (uint8_t *)&ehp->ether_dhost;
612 		} else {
613 			mac_header_info_t		mhi;
614 
615 			if (mac_header_info((mac_handle_t)mcip->mci_mip,
616 			    mp, &mhi) != 0) {
617 				mac_rx_drop_pkt(mac_srs, mp);
618 				continue;
619 			}
620 			hdrsize = mhi.mhi_hdrsize;
621 			sap = mhi.mhi_bindsap;
622 			is_unicast = (mhi.mhi_dsttype == MAC_ADDRTYPE_UNICAST);
623 			dstaddr = (uint8_t *)mhi.mhi_daddr;
624 		}
625 
626 		if (!dls_bypass) {
627 			FANOUT_ENQUEUE_MP(headmp[type], tailmp[type],
628 			    cnt[type], bw_ctl, sz[type], sz1, mp);
629 			continue;
630 		}
631 
632 		if (sap == ETHERTYPE_IP) {
633 			/*
634 			 * If we are H/W classified, but we have promisc
635 			 * on, then we need to check for the unicast address.
636 			 */
637 			if (hw_classified && mcip->mci_promisc_list != NULL) {
638 				mac_address_t		*map;
639 
640 				rw_enter(&mcip->mci_rw_lock, RW_READER);
641 				map = mcip->mci_unicast;
642 				if (bcmp(dstaddr, map->ma_addr,
643 				    map->ma_len) == 0)
644 					type = UNDEF;
645 				rw_exit(&mcip->mci_rw_lock);
646 			} else if (is_unicast) {
647 				type = UNDEF;
648 			}
649 		}
650 
651 		/*
652 		 * This needs to become a contract with the driver for
653 		 * the fast path.
654 		 *
655 		 * In the normal case the packet will have at least the L2
656 		 * header and the IP + Transport header in the same mblk.
657 		 * This is usually the case when the NIC driver sends up
658 		 * the packet. This is also true when the stack generates
659 		 * a packet that is looped back and when the stack uses the
660 		 * fastpath mechanism. The normal case is optimized for
661 		 * performance and may bypass DLS. All other cases go through
662 		 * the 'OTH' type path without DLS bypass.
663 		 */
664 
665 		ipha = (ipha_t *)(mp->b_rptr + hdrsize);
666 		if ((type != OTH) && MBLK_RX_FANOUT_SLOWPATH(mp, ipha))
667 			type = OTH;
668 
669 		if (type == OTH) {
670 			FANOUT_ENQUEUE_MP(headmp[type], tailmp[type],
671 			    cnt[type], bw_ctl, sz[type], sz1, mp);
672 			continue;
673 		}
674 
675 		ASSERT(type == UNDEF);
676 		/*
677 		 * We look for at least 4 bytes past the IP header to get
678 		 * the port information. If we get an IP fragment, we don't
679 		 * have the port information, and we use just the protocol
680 		 * information.
681 		 */
682 		switch (ipha->ipha_protocol) {
683 		case IPPROTO_TCP:
684 			type = V4_TCP;
685 			mp->b_rptr += hdrsize;
686 			break;
687 		case IPPROTO_UDP:
688 			type = V4_UDP;
689 			mp->b_rptr += hdrsize;
690 			break;
691 		default:
692 			type = OTH;
693 			break;
694 		}
695 
696 		FANOUT_ENQUEUE_MP(headmp[type], tailmp[type], cnt[type],
697 		    bw_ctl, sz[type], sz1, mp);
698 	}
699 
700 	for (type = V4_TCP; type < UNDEF; type++) {
701 		if (headmp[type] != NULL) {
702 			mac_soft_ring_t			*softring;
703 
704 			ASSERT(tailmp[type]->b_next == NULL);
705 			switch (type) {
706 			case V4_TCP:
707 				softring = mac_srs->srs_tcp_soft_rings[0];
708 				break;
709 			case V4_UDP:
710 				softring = mac_srs->srs_udp_soft_rings[0];
711 				break;
712 			case OTH:
713 				softring = mac_srs->srs_oth_soft_rings[0];
714 			}
715 			mac_rx_soft_ring_process(mcip, softring,
716 			    headmp[type], tailmp[type], cnt[type], sz[type]);
717 		}
718 	}
719 }
720 
721 int	fanout_unalligned = 0;
722 
723 /*
724  * mac_rx_srs_long_fanout
725  *
726  * The fanout routine for IPv6
727  */
728 static int
729 mac_rx_srs_long_fanout(mac_soft_ring_set_t *mac_srs, mblk_t *mp,
730     uint32_t sap, size_t hdrsize, enum pkt_type *type, uint_t *indx)
731 {
732 	ip6_t		*ip6h;
733 	uint8_t		*whereptr;
734 	uint_t		hash;
735 	uint16_t	remlen;
736 	uint8_t		nexthdr;
737 	uint16_t	hdr_len;
738 
739 	if (sap == ETHERTYPE_IPV6) {
740 		boolean_t	modifiable = B_TRUE;
741 
742 		ASSERT(MBLKL(mp) >= hdrsize);
743 
744 		ip6h = (ip6_t *)(mp->b_rptr + hdrsize);
745 		if ((unsigned char *)ip6h == mp->b_wptr) {
746 			/*
747 			 * The first mblk_t only includes the mac header.
748 			 * Note that it is safe to change the mp pointer here,
749 			 * as the subsequent operation does not assume mp
750 			 * points to the start of the mac header.
751 			 */
752 			mp = mp->b_cont;
753 
754 			/*
755 			 * Make sure ip6h holds the full ip6_t structure.
756 			 */
757 			if (mp == NULL)
758 				return (-1);
759 
760 			if (MBLKL(mp) < IPV6_HDR_LEN) {
761 				modifiable = (DB_REF(mp) == 1);
762 
763 				if (modifiable &&
764 				    !pullupmsg(mp, IPV6_HDR_LEN)) {
765 					return (-1);
766 				}
767 			}
768 
769 			ip6h = (ip6_t *)mp->b_rptr;
770 		}
771 
772 		if (!modifiable || !(OK_32PTR((char *)ip6h)) ||
773 		    ((unsigned char *)ip6h + IPV6_HDR_LEN > mp->b_wptr)) {
774 			/*
775 			 * If either ip6h is not alligned, or ip6h does not
776 			 * hold the complete ip6_t structure (a pullupmsg()
777 			 * is not an option since it would result in an
778 			 * unalligned ip6h), fanout to the default ring. Note
779 			 * that this may cause packets reordering.
780 			 */
781 			*indx = 0;
782 			*type = OTH;
783 			fanout_unalligned++;
784 			return (0);
785 		}
786 
787 		remlen = ntohs(ip6h->ip6_plen);
788 		nexthdr = ip6h->ip6_nxt;
789 
790 		if (remlen < MIN_EHDR_LEN)
791 			return (-1);
792 		/*
793 		 * Do src based fanout if below tunable is set to B_TRUE or
794 		 * when mac_ip_hdr_length_v6() fails because of malformed
795 		 * packets or because mblk's need to be concatenated using
796 		 * pullupmsg().
797 		 */
798 		if (mac_src_ipv6_fanout || !mac_ip_hdr_length_v6(mp, ip6h,
799 		    &hdr_len, &nexthdr)) {
800 			goto src_based_fanout;
801 		}
802 		whereptr = (uint8_t *)ip6h + hdr_len;
803 
804 		/* If the transport is one of below, we do port based fanout */
805 		switch (nexthdr) {
806 		case IPPROTO_TCP:
807 		case IPPROTO_UDP:
808 		case IPPROTO_SCTP:
809 		case IPPROTO_ESP:
810 			/*
811 			 * If the ports in the transport header is not part of
812 			 * the mblk, do src_based_fanout, instead of calling
813 			 * pullupmsg().
814 			 */
815 			if (mp->b_cont != NULL &&
816 			    whereptr + PORTS_SIZE > mp->b_wptr) {
817 				goto src_based_fanout;
818 			}
819 			break;
820 		default:
821 			break;
822 		}
823 
824 		switch (nexthdr) {
825 		case IPPROTO_TCP:
826 			hash = HASH_ADDR(V4_PART_OF_V6(ip6h->ip6_src),
827 			    *(uint32_t *)whereptr);
828 			*indx = COMPUTE_INDEX(hash,
829 			    mac_srs->srs_tcp_ring_count);
830 			*type = OTH;
831 			break;
832 
833 		case IPPROTO_UDP:
834 		case IPPROTO_SCTP:
835 		case IPPROTO_ESP:
836 			if (mac_fanout_type == MAC_FANOUT_DEFAULT) {
837 				hash = HASH_ADDR(V4_PART_OF_V6(ip6h->ip6_src),
838 				    *(uint32_t *)whereptr);
839 				*indx = COMPUTE_INDEX(hash,
840 				    mac_srs->srs_udp_ring_count);
841 			} else {
842 				*indx = mac_srs->srs_ind %
843 				    mac_srs->srs_udp_ring_count;
844 				mac_srs->srs_ind++;
845 			}
846 			*type = OTH;
847 			break;
848 
849 			/* For all other protocol, do source based fanout */
850 		default:
851 			goto src_based_fanout;
852 		}
853 	} else {
854 		*indx = 0;
855 		*type = OTH;
856 	}
857 	return (0);
858 
859 src_based_fanout:
860 	hash = HASH_ADDR(V4_PART_OF_V6(ip6h->ip6_src), (uint32_t)0);
861 	*indx = COMPUTE_INDEX(hash, mac_srs->srs_oth_ring_count);
862 	*type = OTH;
863 	return (0);
864 }
865 
866 /*
867  * mac_rx_srs_fanout
868  *
869  * This routine delivers packets destined to an SRS into a soft ring member
870  * of the set.
871  *
872  * Given a chain of packets we need to split it up into multiple sub chains
873  * destined for one of the TCP, UDP or OTH soft rings. Instead of entering
874  * the soft ring one packet at a time, we want to enter it in the form of a
875  * chain otherwise we get this start/stop behaviour where the worker thread
876  * goes to sleep and then next packets comes in forcing it to wake up etc.
877  *
878  * Note:
879  * Since we know what is the maximum fanout possible, we create a 2D array
880  * of 'softring types * MAX_SR_FANOUT' for the head, tail, cnt and sz
881  * variables so that we can enter the softrings with chain. We need the
882  * MAX_SR_FANOUT so we can allocate the arrays on the stack (a kmem_alloc
883  * for each packet would be expensive). If we ever want to have the
884  * ability to have unlimited fanout, we should probably declare a head,
885  * tail, cnt, sz with each soft ring (a data struct which contains a softring
886  * along with these members) and create an array of this uber struct so we
887  * don't have to do kmem_alloc.
888  */
889 int	fanout_oth1 = 0;
890 int	fanout_oth2 = 0;
891 int	fanout_oth3 = 0;
892 int	fanout_oth4 = 0;
893 int	fanout_oth5 = 0;
894 
895 static void
896 mac_rx_srs_fanout(mac_soft_ring_set_t *mac_srs, mblk_t *head)
897 {
898 	struct ether_header		*ehp;
899 	struct ether_vlan_header	*evhp;
900 	uint32_t			sap;
901 	ipha_t				*ipha;
902 	uint8_t				*dstaddr;
903 	uint_t				indx;
904 	size_t				ports_offset;
905 	size_t				ipha_len;
906 	size_t				hdrsize;
907 	uint_t				hash;
908 	mblk_t				*mp;
909 	mblk_t				*headmp[MAX_SR_TYPES][MAX_SR_FANOUT];
910 	mblk_t				*tailmp[MAX_SR_TYPES][MAX_SR_FANOUT];
911 	int				cnt[MAX_SR_TYPES][MAX_SR_FANOUT];
912 	size_t				sz[MAX_SR_TYPES][MAX_SR_FANOUT];
913 	size_t				sz1;
914 	boolean_t			bw_ctl;
915 	boolean_t			hw_classified;
916 	boolean_t			dls_bypass;
917 	boolean_t			is_ether;
918 	boolean_t			is_unicast;
919 	int				fanout_cnt;
920 	enum pkt_type			type;
921 	mac_client_impl_t		*mcip = mac_srs->srs_mcip;
922 
923 	is_ether = (mcip->mci_mip->mi_info.mi_nativemedia == DL_ETHER);
924 	bw_ctl = ((mac_srs->srs_type & SRST_BW_CONTROL) != 0);
925 
926 	/*
927 	 * If we don't have a Rx ring, S/W classification would have done
928 	 * its job and its a packet meant for us. If we were polling on
929 	 * the default ring (i.e. there was a ring assigned to this SRS),
930 	 * then we need to make sure that the mac address really belongs
931 	 * to us.
932 	 */
933 	hw_classified = mac_srs->srs_ring != NULL &&
934 	    mac_srs->srs_ring->mr_classify_type == MAC_HW_CLASSIFIER;
935 
936 	/*
937 	 * Special clients (eg. VLAN, non ether, etc) need DLS
938 	 * processing in the Rx path. SRST_DLS_BYPASS will be clear for
939 	 * such SRSs. Another way of disabling bypass is to set the
940 	 * MCIS_RX_BYPASS_DISABLE flag.
941 	 */
942 	dls_bypass = ((mac_srs->srs_type & SRST_DLS_BYPASS) != 0) &&
943 	    ((mcip->mci_state_flags & MCIS_RX_BYPASS_DISABLE) == 0);
944 
945 	/*
946 	 * Since the softrings are never destroyed and we always
947 	 * create equal number of softrings for TCP, UDP and rest,
948 	 * its OK to check one of them for count and use it without
949 	 * any lock. In future, if soft rings get destroyed because
950 	 * of reduction in fanout, we will need to ensure that happens
951 	 * behind the SRS_PROC.
952 	 */
953 	fanout_cnt = mac_srs->srs_tcp_ring_count;
954 
955 	bzero(headmp, MAX_SR_TYPES * MAX_SR_FANOUT * sizeof (mblk_t *));
956 	bzero(tailmp, MAX_SR_TYPES * MAX_SR_FANOUT * sizeof (mblk_t *));
957 	bzero(cnt, MAX_SR_TYPES * MAX_SR_FANOUT * sizeof (int));
958 	bzero(sz, MAX_SR_TYPES * MAX_SR_FANOUT * sizeof (size_t));
959 
960 	/*
961 	 * We got a chain from SRS that we need to send to the soft rings.
962 	 * Since squeues for TCP & IPv4 sap poll their soft rings (for
963 	 * performance reasons), we need to separate out v4_tcp, v4_udp
964 	 * and the rest goes in other.
965 	 */
966 	while (head != NULL) {
967 		mp = head;
968 		head = head->b_next;
969 		mp->b_next = NULL;
970 
971 		type = OTH;
972 		sz1 = (mp->b_cont == NULL) ? MBLKL(mp) : msgdsize(mp);
973 
974 		if (is_ether) {
975 			/*
976 			 * At this point we can be sure the packet at least
977 			 * has an ether header.
978 			 */
979 			if (sz1 < sizeof (struct ether_header)) {
980 				mac_rx_drop_pkt(mac_srs, mp);
981 				continue;
982 			}
983 			ehp = (struct ether_header *)mp->b_rptr;
984 
985 			/*
986 			 * Determine if this is a VLAN or non-VLAN packet.
987 			 */
988 			if ((sap = ntohs(ehp->ether_type)) == VLAN_TPID) {
989 				evhp = (struct ether_vlan_header *)mp->b_rptr;
990 				sap = ntohs(evhp->ether_type);
991 				hdrsize = sizeof (struct ether_vlan_header);
992 				/*
993 				 * Check if the VID of the packet, if any,
994 				 * belongs to this client.
995 				 */
996 				if (!mac_client_check_flow_vid(mcip,
997 				    VLAN_ID(ntohs(evhp->ether_tci)))) {
998 					mac_rx_drop_pkt(mac_srs, mp);
999 					continue;
1000 				}
1001 			} else {
1002 				hdrsize = sizeof (struct ether_header);
1003 			}
1004 			is_unicast =
1005 			    ((((uint8_t *)&ehp->ether_dhost)[0] & 0x01) == 0);
1006 			dstaddr = (uint8_t *)&ehp->ether_dhost;
1007 		} else {
1008 			mac_header_info_t		mhi;
1009 
1010 			if (mac_header_info((mac_handle_t)mcip->mci_mip,
1011 			    mp, &mhi) != 0) {
1012 				mac_rx_drop_pkt(mac_srs, mp);
1013 				continue;
1014 			}
1015 			hdrsize = mhi.mhi_hdrsize;
1016 			sap = mhi.mhi_bindsap;
1017 			is_unicast = (mhi.mhi_dsttype == MAC_ADDRTYPE_UNICAST);
1018 			dstaddr = (uint8_t *)mhi.mhi_daddr;
1019 		}
1020 
1021 		if (!dls_bypass) {
1022 			if (mac_rx_srs_long_fanout(mac_srs, mp, sap,
1023 			    hdrsize, &type, &indx) == -1) {
1024 				mac_rx_drop_pkt(mac_srs, mp);
1025 				continue;
1026 			}
1027 
1028 			FANOUT_ENQUEUE_MP(headmp[type][indx],
1029 			    tailmp[type][indx], cnt[type][indx], bw_ctl,
1030 			    sz[type][indx], sz1, mp);
1031 			continue;
1032 		}
1033 
1034 
1035 		/*
1036 		 * If we are using the default Rx ring where H/W or S/W
1037 		 * classification has not happened, we need to verify if
1038 		 * this unicast packet really belongs to us.
1039 		 */
1040 		if (sap == ETHERTYPE_IP) {
1041 			/*
1042 			 * If we are H/W classified, but we have promisc
1043 			 * on, then we need to check for the unicast address.
1044 			 */
1045 			if (hw_classified && mcip->mci_promisc_list != NULL) {
1046 				mac_address_t		*map;
1047 
1048 				rw_enter(&mcip->mci_rw_lock, RW_READER);
1049 				map = mcip->mci_unicast;
1050 				if (bcmp(dstaddr, map->ma_addr,
1051 				    map->ma_len) == 0)
1052 					type = UNDEF;
1053 				rw_exit(&mcip->mci_rw_lock);
1054 			} else if (is_unicast) {
1055 				type = UNDEF;
1056 			}
1057 		}
1058 
1059 		/*
1060 		 * This needs to become a contract with the driver for
1061 		 * the fast path.
1062 		 */
1063 
1064 		ipha = (ipha_t *)(mp->b_rptr + hdrsize);
1065 		if ((type != OTH) && MBLK_RX_FANOUT_SLOWPATH(mp, ipha)) {
1066 			type = OTH;
1067 			fanout_oth1++;
1068 		}
1069 
1070 		if (type != OTH) {
1071 			uint16_t	frag_offset_flags;
1072 
1073 			switch (ipha->ipha_protocol) {
1074 			case IPPROTO_TCP:
1075 			case IPPROTO_UDP:
1076 			case IPPROTO_SCTP:
1077 			case IPPROTO_ESP:
1078 				ipha_len = IPH_HDR_LENGTH(ipha);
1079 				if ((uchar_t *)ipha + ipha_len + PORTS_SIZE >
1080 				    mp->b_wptr) {
1081 					type = OTH;
1082 					break;
1083 				}
1084 				frag_offset_flags =
1085 				    ntohs(ipha->ipha_fragment_offset_and_flags);
1086 				if ((frag_offset_flags &
1087 				    (IPH_MF | IPH_OFFSET)) != 0) {
1088 					type = OTH;
1089 					fanout_oth3++;
1090 					break;
1091 				}
1092 				ports_offset = hdrsize + ipha_len;
1093 				break;
1094 			default:
1095 				type = OTH;
1096 				fanout_oth4++;
1097 				break;
1098 			}
1099 		}
1100 
1101 		if (type == OTH) {
1102 			if (mac_rx_srs_long_fanout(mac_srs, mp, sap,
1103 			    hdrsize, &type, &indx) == -1) {
1104 				mac_rx_drop_pkt(mac_srs, mp);
1105 				continue;
1106 			}
1107 
1108 			FANOUT_ENQUEUE_MP(headmp[type][indx],
1109 			    tailmp[type][indx], cnt[type][indx], bw_ctl,
1110 			    sz[type][indx], sz1, mp);
1111 			continue;
1112 		}
1113 
1114 		ASSERT(type == UNDEF);
1115 
1116 		/*
1117 		 * XXX-Sunay: We should hold srs_lock since ring_count
1118 		 * below can change. But if we are always called from
1119 		 * mac_rx_srs_drain and SRS_PROC is set, then we can
1120 		 * enforce that ring_count can't be changed i.e.
1121 		 * to change fanout type or ring count, the calling
1122 		 * thread needs to be behind SRS_PROC.
1123 		 */
1124 		switch (ipha->ipha_protocol) {
1125 		case IPPROTO_TCP:
1126 			/*
1127 			 * Note that for ESP, we fanout on SPI and it is at the
1128 			 * same offset as the 2x16-bit ports. So it is clumped
1129 			 * along with TCP, UDP and SCTP.
1130 			 */
1131 			hash = HASH_ADDR(ipha->ipha_src,
1132 			    *(uint32_t *)(mp->b_rptr + ports_offset));
1133 			indx = COMPUTE_INDEX(hash, mac_srs->srs_tcp_ring_count);
1134 			type = V4_TCP;
1135 			mp->b_rptr += hdrsize;
1136 			break;
1137 		case IPPROTO_UDP:
1138 		case IPPROTO_SCTP:
1139 		case IPPROTO_ESP:
1140 			if (mac_fanout_type == MAC_FANOUT_DEFAULT) {
1141 				hash = HASH_ADDR(ipha->ipha_src,
1142 				    *(uint32_t *)(mp->b_rptr + ports_offset));
1143 				indx = COMPUTE_INDEX(hash,
1144 				    mac_srs->srs_udp_ring_count);
1145 			} else {
1146 				indx = mac_srs->srs_ind %
1147 				    mac_srs->srs_udp_ring_count;
1148 				mac_srs->srs_ind++;
1149 			}
1150 			type = V4_UDP;
1151 			mp->b_rptr += hdrsize;
1152 			break;
1153 		default:
1154 			indx = 0;
1155 			type = OTH;
1156 		}
1157 
1158 		FANOUT_ENQUEUE_MP(headmp[type][indx], tailmp[type][indx],
1159 		    cnt[type][indx], bw_ctl, sz[type][indx], sz1, mp);
1160 	}
1161 
1162 	for (type = V4_TCP; type < UNDEF; type++) {
1163 		int	i;
1164 
1165 		for (i = 0; i < fanout_cnt; i++) {
1166 			if (headmp[type][i] != NULL) {
1167 				mac_soft_ring_t	*softring;
1168 
1169 				ASSERT(tailmp[type][i]->b_next == NULL);
1170 				switch (type) {
1171 				case V4_TCP:
1172 					softring =
1173 					    mac_srs->srs_tcp_soft_rings[i];
1174 					break;
1175 				case V4_UDP:
1176 					softring =
1177 					    mac_srs->srs_udp_soft_rings[i];
1178 					break;
1179 				case OTH:
1180 					softring =
1181 					    mac_srs->srs_oth_soft_rings[i];
1182 					break;
1183 				}
1184 				mac_rx_soft_ring_process(mcip,
1185 				    softring, headmp[type][i], tailmp[type][i],
1186 				    cnt[type][i], sz[type][i]);
1187 			}
1188 		}
1189 	}
1190 }
1191 
1192 #define	SRS_BYTES_TO_PICKUP	150000
1193 ssize_t	max_bytes_to_pickup = SRS_BYTES_TO_PICKUP;
1194 
1195 /*
1196  * mac_rx_srs_poll_ring
1197  *
1198  * This SRS Poll thread uses this routine to poll the underlying hardware
1199  * Rx ring to get a chain of packets. It can inline process that chain
1200  * if mac_latency_optimize is set (default) or signal the SRS worker thread
1201  * to do the remaining processing.
1202  *
1203  * Since packets come in the system via interrupt or poll path, we also
1204  * update the stats and deal with promiscous clients here.
1205  */
1206 void
1207 mac_rx_srs_poll_ring(mac_soft_ring_set_t *mac_srs)
1208 {
1209 	kmutex_t 		*lock = &mac_srs->srs_lock;
1210 	kcondvar_t 		*async = &mac_srs->srs_cv;
1211 	mac_srs_rx_t		*srs_rx = &mac_srs->srs_rx;
1212 	mblk_t 			*head, *tail, *mp;
1213 	callb_cpr_t 		cprinfo;
1214 	ssize_t 		bytes_to_pickup;
1215 	size_t 			sz;
1216 	int			count;
1217 	mac_client_impl_t	*smcip;
1218 
1219 	CALLB_CPR_INIT(&cprinfo, lock, callb_generic_cpr, "mac_srs_poll");
1220 	mutex_enter(lock);
1221 
1222 start:
1223 	for (;;) {
1224 		if (mac_srs->srs_state & SRS_PAUSE)
1225 			goto done;
1226 
1227 		CALLB_CPR_SAFE_BEGIN(&cprinfo);
1228 		cv_wait(async, lock);
1229 		CALLB_CPR_SAFE_END(&cprinfo, lock);
1230 
1231 		if (mac_srs->srs_state & SRS_PAUSE)
1232 			goto done;
1233 
1234 check_again:
1235 		if (mac_srs->srs_type & SRST_BW_CONTROL) {
1236 			/*
1237 			 * We pick as many bytes as we are allowed to queue.
1238 			 * Its possible that we will exceed the total
1239 			 * packets queued in case this SRS is part of the
1240 			 * Rx ring group since > 1 poll thread can be pulling
1241 			 * upto the max allowed packets at the same time
1242 			 * but that should be OK.
1243 			 */
1244 			mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
1245 			bytes_to_pickup =
1246 			    mac_srs->srs_bw->mac_bw_drop_threshold -
1247 			    mac_srs->srs_bw->mac_bw_sz;
1248 			/*
1249 			 * We shouldn't have been signalled if we
1250 			 * have 0 or less bytes to pick but since
1251 			 * some of the bytes accounting is driver
1252 			 * dependant, we do the safety check.
1253 			 */
1254 			if (bytes_to_pickup < 0)
1255 				bytes_to_pickup = 0;
1256 			mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
1257 		} else {
1258 			/*
1259 			 * ToDO: Need to change the polling API
1260 			 * to add a packet count and a flag which
1261 			 * tells the driver whether we want packets
1262 			 * based on a count, or bytes, or all the
1263 			 * packets queued in the driver/HW. This
1264 			 * way, we never have to check the limits
1265 			 * on poll path. We truly let only as many
1266 			 * packets enter the system as we are willing
1267 			 * to process or queue.
1268 			 *
1269 			 * Something along the lines of
1270 			 * pkts_to_pickup = mac_soft_ring_max_q_cnt -
1271 			 *	mac_srs->srs_poll_pkt_cnt
1272 			 */
1273 
1274 			/*
1275 			 * Since we are not doing B/W control, pick
1276 			 * as many packets as allowed.
1277 			 */
1278 			bytes_to_pickup = max_bytes_to_pickup;
1279 		}
1280 
1281 		/* Poll the underlying Hardware */
1282 		mutex_exit(lock);
1283 		head = MAC_HWRING_POLL(mac_srs->srs_ring, (int)bytes_to_pickup);
1284 		mutex_enter(lock);
1285 
1286 		ASSERT((mac_srs->srs_state & SRS_POLL_THR_OWNER) ==
1287 		    SRS_POLL_THR_OWNER);
1288 
1289 		mp = tail = head;
1290 		count = 0;
1291 		sz = 0;
1292 		while (mp != NULL) {
1293 			tail = mp;
1294 			sz += msgdsize(mp);
1295 			mp = mp->b_next;
1296 			count++;
1297 		}
1298 
1299 		if (head != NULL) {
1300 			tail->b_next = NULL;
1301 			smcip = mac_srs->srs_mcip;
1302 
1303 			if ((mac_srs->srs_type & SRST_FLOW) ||
1304 			    (smcip == NULL)) {
1305 				FLOW_STAT_UPDATE(mac_srs->srs_flent,
1306 				    rbytes, sz);
1307 				FLOW_STAT_UPDATE(mac_srs->srs_flent,
1308 				    ipackets, count);
1309 			}
1310 
1311 			/*
1312 			 * If there are any promiscuous mode callbacks
1313 			 * defined for this MAC client, pass them a copy
1314 			 * if appropriate and also update the counters.
1315 			 */
1316 			if (smcip != NULL) {
1317 				smcip->mci_stat_ibytes += sz;
1318 				smcip->mci_stat_ipackets += count;
1319 
1320 				if (smcip->mci_mip->mi_promisc_list != NULL) {
1321 					mutex_exit(lock);
1322 					mac_promisc_dispatch(smcip->mci_mip,
1323 					    head, NULL);
1324 					mutex_enter(lock);
1325 				}
1326 			}
1327 			if (mac_srs->srs_type & SRST_BW_CONTROL) {
1328 				mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
1329 				mac_srs->srs_bw->mac_bw_polled += sz;
1330 				mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
1331 			}
1332 			srs_rx->sr_poll_count += count;
1333 			MAC_RX_SRS_ENQUEUE_CHAIN(mac_srs, head, tail,
1334 			    count, sz);
1335 			if (count <= 10)
1336 				srs_rx->sr_chain_cnt_undr10++;
1337 			else if (count > 10 && count <= 50)
1338 				srs_rx->sr_chain_cnt_10to50++;
1339 			else
1340 				srs_rx->sr_chain_cnt_over50++;
1341 		}
1342 
1343 		/*
1344 		 * We are guaranteed that SRS_PROC will be set if we
1345 		 * are here. Also, poll thread gets to run only if
1346 		 * the drain was being done by a worker thread although
1347 		 * its possible that worker thread is still running
1348 		 * and poll thread was sent down to keep the pipeline
1349 		 * going instead of doing a complete drain and then
1350 		 * trying to poll the NIC.
1351 		 *
1352 		 * So we need to check SRS_WORKER flag to make sure
1353 		 * that the worker thread is not processing the queue
1354 		 * in parallel to us. The flags and conditions are
1355 		 * protected by the srs_lock to prevent any race. We
1356 		 * ensure that we don't drop the srs_lock from now
1357 		 * till the end and similarly we don't drop the srs_lock
1358 		 * in mac_rx_srs_drain() till similar condition check
1359 		 * are complete. The mac_rx_srs_drain() needs to ensure
1360 		 * that SRS_WORKER flag remains set as long as its
1361 		 * processing the queue.
1362 		 */
1363 		if (!(mac_srs->srs_state & SRS_WORKER) &&
1364 		    (mac_srs->srs_first != NULL)) {
1365 			/*
1366 			 * We have packets to process and worker thread
1367 			 * is not running. Check to see if poll thread is
1368 			 * allowed to process.
1369 			 */
1370 			if (mac_srs->srs_state & SRS_LATENCY_OPT) {
1371 				mac_srs->srs_drain_func(mac_srs, SRS_POLL_PROC);
1372 				if (!(mac_srs->srs_state & SRS_PAUSE) &&
1373 				    srs_rx->sr_poll_pkt_cnt <=
1374 				    srs_rx->sr_lowat) {
1375 					srs_rx->sr_poll_again++;
1376 					goto check_again;
1377 				}
1378 				/*
1379 				 * We are already above low water mark
1380 				 * so stay in the polling mode but no
1381 				 * need to poll. Once we dip below
1382 				 * the polling threshold, the processing
1383 				 * thread (soft ring) will signal us
1384 				 * to poll again (MAC_UPDATE_SRS_COUNT)
1385 				 */
1386 				srs_rx->sr_poll_drain_no_poll++;
1387 				mac_srs->srs_state &= ~(SRS_PROC|SRS_GET_PKTS);
1388 				/*
1389 				 * In B/W control case, its possible
1390 				 * that the backlog built up due to
1391 				 * B/W limit being reached and packets
1392 				 * are queued only in SRS. In this case,
1393 				 * we should schedule worker thread
1394 				 * since no one else will wake us up.
1395 				 */
1396 				if ((mac_srs->srs_type & SRST_BW_CONTROL) &&
1397 				    (mac_srs->srs_tid == NULL)) {
1398 					mac_srs->srs_tid =
1399 					    timeout(mac_srs_fire, mac_srs, 1);
1400 					srs_rx->sr_poll_worker_wakeup++;
1401 				}
1402 			} else {
1403 				/*
1404 				 * Wakeup the worker thread for more processing.
1405 				 * We optimize for throughput in this case.
1406 				 */
1407 				mac_srs->srs_state &= ~(SRS_PROC|SRS_GET_PKTS);
1408 				MAC_SRS_WORKER_WAKEUP(mac_srs);
1409 				srs_rx->sr_poll_sig_worker++;
1410 			}
1411 		} else if ((mac_srs->srs_first == NULL) &&
1412 		    !(mac_srs->srs_state & SRS_WORKER)) {
1413 			/*
1414 			 * There is nothing queued in SRS and
1415 			 * no worker thread running. Plus we
1416 			 * didn't get anything from the H/W
1417 			 * as well (head == NULL);
1418 			 */
1419 			ASSERT(head == NULL);
1420 			mac_srs->srs_state &=
1421 			    ~(SRS_PROC|SRS_GET_PKTS);
1422 
1423 			/*
1424 			 * If we have a packets in soft ring, don't allow
1425 			 * more packets to come into this SRS by keeping the
1426 			 * interrupts off but not polling the H/W. The
1427 			 * poll thread will get signaled as soon as
1428 			 * srs_poll_pkt_cnt dips below poll threshold.
1429 			 */
1430 			if (srs_rx->sr_poll_pkt_cnt == 0) {
1431 				srs_rx->sr_poll_intr_enable++;
1432 				MAC_SRS_POLLING_OFF(mac_srs);
1433 			} else {
1434 				/*
1435 				 * We know nothing is queued in SRS
1436 				 * since we are here after checking
1437 				 * srs_first is NULL. The backlog
1438 				 * is entirely due to packets queued
1439 				 * in Soft ring which will wake us up
1440 				 * and get the interface out of polling
1441 				 * mode once the backlog dips below
1442 				 * sr_poll_thres.
1443 				 */
1444 				srs_rx->sr_poll_no_poll++;
1445 			}
1446 		} else {
1447 			/*
1448 			 * Worker thread is already running.
1449 			 * Nothing much to do. If the polling
1450 			 * was enabled, worker thread will deal
1451 			 * with that.
1452 			 */
1453 			mac_srs->srs_state &= ~SRS_GET_PKTS;
1454 			srs_rx->sr_poll_goto_sleep++;
1455 		}
1456 	}
1457 done:
1458 	mac_srs->srs_state |= SRS_POLL_THR_QUIESCED;
1459 	cv_signal(&mac_srs->srs_async);
1460 	/*
1461 	 * If this is a temporary quiesce then wait for the restart signal
1462 	 * from the srs worker. Then clear the flags and signal the srs worker
1463 	 * to ensure a positive handshake and go back to start.
1464 	 */
1465 	while (!(mac_srs->srs_state & (SRS_CONDEMNED | SRS_POLL_THR_RESTART)))
1466 		cv_wait(async, lock);
1467 	if (mac_srs->srs_state & SRS_POLL_THR_RESTART) {
1468 		ASSERT(!(mac_srs->srs_state & SRS_CONDEMNED));
1469 		mac_srs->srs_state &=
1470 		    ~(SRS_POLL_THR_QUIESCED | SRS_POLL_THR_RESTART);
1471 		cv_signal(&mac_srs->srs_async);
1472 		goto start;
1473 	} else {
1474 		mac_srs->srs_state |= SRS_POLL_THR_EXITED;
1475 		cv_signal(&mac_srs->srs_async);
1476 		CALLB_CPR_EXIT(&cprinfo);
1477 		thread_exit();
1478 	}
1479 }
1480 
1481 /*
1482  * mac_srs_pick_chain
1483  *
1484  * In Bandwidth control case, checks how many packets can be processed
1485  * and return them in a sub chain.
1486  */
1487 static mblk_t *
1488 mac_srs_pick_chain(mac_soft_ring_set_t *mac_srs, mblk_t **chain_tail,
1489     size_t *chain_sz, int *chain_cnt)
1490 {
1491 	mblk_t 			*head = NULL;
1492 	mblk_t 			*tail = NULL;
1493 	size_t			sz;
1494 	size_t 			tsz = 0;
1495 	int			cnt = 0;
1496 	mblk_t 			*mp;
1497 
1498 	ASSERT(MUTEX_HELD(&mac_srs->srs_lock));
1499 	mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
1500 	if (((mac_srs->srs_bw->mac_bw_used + mac_srs->srs_size) <=
1501 	    mac_srs->srs_bw->mac_bw_limit) ||
1502 	    (mac_srs->srs_bw->mac_bw_limit == 0)) {
1503 		mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
1504 		head = mac_srs->srs_first;
1505 		mac_srs->srs_first = NULL;
1506 		*chain_tail = mac_srs->srs_last;
1507 		mac_srs->srs_last = NULL;
1508 		*chain_sz = mac_srs->srs_size;
1509 		*chain_cnt = mac_srs->srs_count;
1510 		mac_srs->srs_count = 0;
1511 		mac_srs->srs_size = 0;
1512 		return (head);
1513 	}
1514 
1515 	/*
1516 	 * Can't clear the entire backlog.
1517 	 * Need to find how many packets to pick
1518 	 */
1519 	ASSERT(MUTEX_HELD(&mac_srs->srs_bw->mac_bw_lock));
1520 	while ((mp = mac_srs->srs_first) != NULL) {
1521 		sz = msgdsize(mp);
1522 		if ((tsz + sz + mac_srs->srs_bw->mac_bw_used) >
1523 		    mac_srs->srs_bw->mac_bw_limit) {
1524 			if (!(mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED))
1525 				mac_srs->srs_bw->mac_bw_state |=
1526 				    SRS_BW_ENFORCED;
1527 			break;
1528 		}
1529 
1530 		/*
1531 		 * The _size & cnt is  decremented from the softrings
1532 		 * when they send up the packet for polling to work
1533 		 * properly.
1534 		 */
1535 		tsz += sz;
1536 		cnt++;
1537 		mac_srs->srs_count--;
1538 		mac_srs->srs_size -= sz;
1539 		if (tail != NULL)
1540 			tail->b_next = mp;
1541 		else
1542 			head = mp;
1543 		tail = mp;
1544 		mac_srs->srs_first = mac_srs->srs_first->b_next;
1545 	}
1546 	mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
1547 	if (mac_srs->srs_first == NULL)
1548 		mac_srs->srs_last = NULL;
1549 
1550 	if (tail != NULL)
1551 		tail->b_next = NULL;
1552 	*chain_tail = tail;
1553 	*chain_cnt = cnt;
1554 	*chain_sz = tsz;
1555 
1556 	return (head);
1557 }
1558 
1559 /*
1560  * mac_rx_srs_drain
1561  *
1562  * The SRS drain routine. Gets to run to clear the queue. Any thread
1563  * (worker, interrupt, poll) can call this based on processing model.
1564  * The first thing we do is disable interrupts if possible and then
1565  * drain the queue. we also try to poll the underlying hardware if
1566  * there is a dedicated hardware Rx ring assigned to this SRS.
1567  *
1568  * There is a equivalent drain routine in bandwidth control mode
1569  * mac_rx_srs_drain_bw. There is some code duplication between the two
1570  * routines but they are highly performance sensitive and are easier
1571  * to read/debug if they stay separate. Any code changes here might
1572  * also apply to mac_rx_srs_drain_bw as well.
1573  */
1574 void
1575 mac_rx_srs_drain(mac_soft_ring_set_t *mac_srs, uint_t proc_type)
1576 {
1577 	mblk_t 			*head;
1578 	mblk_t			*tail;
1579 	timeout_id_t 		tid;
1580 	int			cnt = 0;
1581 	mac_client_impl_t	*mcip = mac_srs->srs_mcip;
1582 	mac_srs_rx_t		*srs_rx = &mac_srs->srs_rx;
1583 
1584 	ASSERT(MUTEX_HELD(&mac_srs->srs_lock));
1585 	ASSERT(!(mac_srs->srs_type & SRST_BW_CONTROL));
1586 
1587 	/* If we are blanked i.e. can't do upcalls, then we are done */
1588 	if (mac_srs->srs_state & (SRS_BLANK | SRS_PAUSE)) {
1589 		ASSERT((mac_srs->srs_type & SRST_NO_SOFT_RINGS) ||
1590 		    (mac_srs->srs_state & SRS_PAUSE));
1591 		goto out;
1592 	}
1593 
1594 	if (mac_srs->srs_first == NULL)
1595 		goto out;
1596 
1597 	if (!(mac_srs->srs_state & SRS_LATENCY_OPT) &&
1598 	    (srs_rx->sr_poll_pkt_cnt <= srs_rx->sr_lowat)) {
1599 		/*
1600 		 * In the normal case, the SRS worker thread does no
1601 		 * work and we wait for a backlog to build up before
1602 		 * we switch into polling mode. In case we are
1603 		 * optimizing for throughput, we use the worker thread
1604 		 * as well. The goal is to let worker thread process
1605 		 * the queue and poll thread to feed packets into
1606 		 * the queue. As such, we should signal the poll
1607 		 * thread to try and get more packets.
1608 		 *
1609 		 * We could have pulled this check in the POLL_RING
1610 		 * macro itself but keeping it explicit here makes
1611 		 * the architecture more human understandable.
1612 		 */
1613 		MAC_SRS_POLL_RING(mac_srs);
1614 	}
1615 
1616 again:
1617 	head = mac_srs->srs_first;
1618 	mac_srs->srs_first = NULL;
1619 	tail = mac_srs->srs_last;
1620 	mac_srs->srs_last = NULL;
1621 	cnt = mac_srs->srs_count;
1622 	mac_srs->srs_count = 0;
1623 
1624 	ASSERT(head != NULL);
1625 	ASSERT(tail != NULL);
1626 
1627 	if ((tid = mac_srs->srs_tid) != 0)
1628 		mac_srs->srs_tid = 0;
1629 
1630 	mac_srs->srs_state |= (SRS_PROC|proc_type);
1631 
1632 
1633 	/*
1634 	 * mcip is NULL for broadcast and multicast flows. The promisc
1635 	 * callbacks for broadcast and multicast packets are delivered from
1636 	 * mac_rx() and we don't need to worry about that case in this path
1637 	 */
1638 	if (mcip != NULL && mcip->mci_promisc_list != NULL) {
1639 		mutex_exit(&mac_srs->srs_lock);
1640 		mac_promisc_client_dispatch(mcip, head);
1641 		mutex_enter(&mac_srs->srs_lock);
1642 	}
1643 
1644 	/*
1645 	 * Check if SRS itself is doing the processing
1646 	 * This direct path does not apply when subflows are present. In this
1647 	 * case, packets need to be dispatched to a soft ring according to the
1648 	 * flow's bandwidth and other resources contraints.
1649 	 */
1650 	if (mac_srs->srs_type & SRST_NO_SOFT_RINGS) {
1651 		mac_direct_rx_t		proc;
1652 		void			*arg1;
1653 		mac_resource_handle_t	arg2;
1654 
1655 		/*
1656 		 * This is the case when a Rx is directly
1657 		 * assigned and we have a fully classified
1658 		 * protocol chain. We can deal with it in
1659 		 * one shot.
1660 		 */
1661 		proc = srs_rx->sr_func;
1662 		arg1 = srs_rx->sr_arg1;
1663 		arg2 = srs_rx->sr_arg2;
1664 
1665 		mac_srs->srs_state |= SRS_CLIENT_PROC;
1666 		mutex_exit(&mac_srs->srs_lock);
1667 		if (tid != 0) {
1668 			(void) untimeout(tid);
1669 			tid = 0;
1670 		}
1671 
1672 		proc(arg1, arg2, head, NULL);
1673 		/*
1674 		 * Decrement the size and count here itelf
1675 		 * since the packet has been processed.
1676 		 */
1677 		mutex_enter(&mac_srs->srs_lock);
1678 		MAC_UPDATE_SRS_COUNT_LOCKED(mac_srs, cnt);
1679 		if (mac_srs->srs_state & SRS_CLIENT_WAIT)
1680 			cv_signal(&mac_srs->srs_client_cv);
1681 		mac_srs->srs_state &= ~SRS_CLIENT_PROC;
1682 	} else {
1683 		/* Some kind of softrings based fanout is required */
1684 		mutex_exit(&mac_srs->srs_lock);
1685 		if (tid != 0) {
1686 			(void) untimeout(tid);
1687 			tid = 0;
1688 		}
1689 
1690 		/*
1691 		 * Since the fanout routines can deal with chains,
1692 		 * shoot the entire chain up.
1693 		 */
1694 		if (mac_srs->srs_type & SRST_FANOUT_SRC_IP)
1695 			mac_rx_srs_fanout(mac_srs, head);
1696 		else
1697 			mac_rx_srs_proto_fanout(mac_srs, head);
1698 		mutex_enter(&mac_srs->srs_lock);
1699 	}
1700 
1701 	if (!(mac_srs->srs_state & (SRS_BLANK|SRS_PAUSE)) &&
1702 	    (mac_srs->srs_first != NULL)) {
1703 		/*
1704 		 * More packets arrived while we were clearing the
1705 		 * SRS. This can be possible because of one of
1706 		 * three conditions below:
1707 		 * 1) The driver is using multiple worker threads
1708 		 *    to send the packets to us.
1709 		 * 2) The driver has a race in switching
1710 		 *    between interrupt and polling mode or
1711 		 * 3) Packets are arriving in this SRS via the
1712 		 *    S/W classification as well.
1713 		 *
1714 		 * We should switch to polling mode and see if we
1715 		 * need to send the poll thread down. Also, signal
1716 		 * the worker thread to process whats just arrived.
1717 		 */
1718 		MAC_SRS_POLLING_ON(mac_srs);
1719 		if (srs_rx->sr_poll_pkt_cnt <= srs_rx->sr_lowat) {
1720 			srs_rx->sr_drain_poll_sig++;
1721 			MAC_SRS_POLL_RING(mac_srs);
1722 		}
1723 
1724 		/*
1725 		 * If we didn't signal the poll thread, we need
1726 		 * to deal with the pending packets ourselves.
1727 		 */
1728 		if (proc_type == SRS_WORKER) {
1729 			srs_rx->sr_drain_again++;
1730 			goto again;
1731 		} else {
1732 			srs_rx->sr_drain_worker_sig++;
1733 			cv_signal(&mac_srs->srs_async);
1734 		}
1735 	}
1736 
1737 out:
1738 	if (mac_srs->srs_state & SRS_GET_PKTS) {
1739 		/*
1740 		 * Poll thread is already running. Leave the
1741 		 * SRS_RPOC set and hand over the control to
1742 		 * poll thread.
1743 		 */
1744 		mac_srs->srs_state &= ~proc_type;
1745 		srs_rx->sr_drain_poll_running++;
1746 		return;
1747 	}
1748 
1749 	/*
1750 	 * Even if there are no packets queued in SRS, we
1751 	 * need to make sure that the shared counter is
1752 	 * clear and any associated softrings have cleared
1753 	 * all the backlog. Otherwise, leave the interface
1754 	 * in polling mode and the poll thread will get
1755 	 * signalled once the count goes down to zero.
1756 	 *
1757 	 * If someone is already draining the queue (SRS_PROC is
1758 	 * set) when the srs_poll_pkt_cnt goes down to zero,
1759 	 * then it means that drain is already running and we
1760 	 * will turn off polling at that time if there is
1761 	 * no backlog.
1762 	 *
1763 	 * As long as there are packets queued either
1764 	 * in soft ring set or its soft rings, we will leave
1765 	 * the interface in polling mode (even if the drain
1766 	 * was done being the interrupt thread). We signal
1767 	 * the poll thread as well if we have dipped below
1768 	 * low water mark.
1769 	 *
1770 	 * NOTE: We can't use the MAC_SRS_POLLING_ON macro
1771 	 * since that turn polling on only for worker thread.
1772 	 * Its not worth turning polling on for interrupt
1773 	 * thread (since NIC will not issue another interrupt)
1774 	 * unless a backlog builds up.
1775 	 */
1776 	if ((srs_rx->sr_poll_pkt_cnt > 0) &&
1777 	    (mac_srs->srs_state & SRS_POLLING_CAPAB)) {
1778 		mac_srs->srs_state &= ~(SRS_PROC|proc_type);
1779 		srs_rx->sr_drain_keep_polling++;
1780 		MAC_SRS_POLLING_ON(mac_srs);
1781 		if (srs_rx->sr_poll_pkt_cnt <= srs_rx->sr_lowat)
1782 			MAC_SRS_POLL_RING(mac_srs);
1783 		return;
1784 	}
1785 
1786 	/* Nothing else to do. Get out of poll mode */
1787 	MAC_SRS_POLLING_OFF(mac_srs);
1788 	mac_srs->srs_state &= ~(SRS_PROC|proc_type);
1789 	srs_rx->sr_drain_finish_intr++;
1790 }
1791 
1792 /*
1793  * mac_rx_srs_drain_bw
1794  *
1795  * The SRS BW drain routine. Gets to run to clear the queue. Any thread
1796  * (worker, interrupt, poll) can call this based on processing model.
1797  * The first thing we do is disable interrupts if possible and then
1798  * drain the queue. we also try to poll the underlying hardware if
1799  * there is a dedicated hardware Rx ring assigned to this SRS.
1800  *
1801  * There is a equivalent drain routine in non bandwidth control mode
1802  * mac_rx_srs_drain. There is some code duplication between the two
1803  * routines but they are highly performance sensitive and are easier
1804  * to read/debug if they stay separate. Any code changes here might
1805  * also apply to mac_rx_srs_drain as well.
1806  */
1807 void
1808 mac_rx_srs_drain_bw(mac_soft_ring_set_t *mac_srs, uint_t proc_type)
1809 {
1810 	mblk_t 			*head;
1811 	mblk_t			*tail;
1812 	timeout_id_t 		tid;
1813 	size_t			sz = 0;
1814 	int			cnt = 0;
1815 	mac_client_impl_t	*mcip = mac_srs->srs_mcip;
1816 	mac_srs_rx_t		*srs_rx = &mac_srs->srs_rx;
1817 	clock_t			now;
1818 
1819 	ASSERT(MUTEX_HELD(&mac_srs->srs_lock));
1820 	ASSERT(mac_srs->srs_type & SRST_BW_CONTROL);
1821 again:
1822 	/* Check if we are doing B/W control */
1823 	mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
1824 	now = ddi_get_lbolt();
1825 	if (mac_srs->srs_bw->mac_bw_curr_time != now) {
1826 		mac_srs->srs_bw->mac_bw_curr_time = now;
1827 		mac_srs->srs_bw->mac_bw_used = 0;
1828 		if (mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED)
1829 			mac_srs->srs_bw->mac_bw_state &= ~SRS_BW_ENFORCED;
1830 	} else if (mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED) {
1831 		mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
1832 		goto done;
1833 	} else if (mac_srs->srs_bw->mac_bw_used >
1834 	    mac_srs->srs_bw->mac_bw_limit) {
1835 		mac_srs->srs_bw->mac_bw_state |= SRS_BW_ENFORCED;
1836 		mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
1837 		goto done;
1838 	}
1839 	mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
1840 
1841 	/* If we are blanked i.e. can't do upcalls, then we are done */
1842 	if (mac_srs->srs_state & (SRS_BLANK | SRS_PAUSE)) {
1843 		ASSERT((mac_srs->srs_type & SRST_NO_SOFT_RINGS) ||
1844 		    (mac_srs->srs_state & SRS_PAUSE));
1845 		goto done;
1846 	}
1847 
1848 	sz = 0;
1849 	cnt = 0;
1850 	if ((head = mac_srs_pick_chain(mac_srs, &tail, &sz, &cnt)) == NULL) {
1851 		/*
1852 		 * We couldn't pick up a single packet.
1853 		 */
1854 		mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
1855 		if ((mac_srs->srs_bw->mac_bw_used == 0) &&
1856 		    (mac_srs->srs_size != 0) &&
1857 		    !(mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED)) {
1858 			/*
1859 			 * Seems like configured B/W doesn't
1860 			 * even allow processing of 1 packet
1861 			 * per tick.
1862 			 *
1863 			 * XXX: raise the limit to processing
1864 			 * at least 1 packet per tick.
1865 			 */
1866 			mac_srs->srs_bw->mac_bw_limit +=
1867 			    mac_srs->srs_bw->mac_bw_limit;
1868 			mac_srs->srs_bw->mac_bw_drop_threshold +=
1869 			    mac_srs->srs_bw->mac_bw_drop_threshold;
1870 			cmn_err(CE_NOTE, "mac_rx_srs_drain: srs(%p) "
1871 			    "raised B/W limit to %d since not even a "
1872 			    "single packet can be processed per "
1873 			    "tick %d\n", (void *)mac_srs,
1874 			    (int)mac_srs->srs_bw->mac_bw_limit,
1875 			    (int)msgdsize(mac_srs->srs_first));
1876 		}
1877 		mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
1878 		goto done;
1879 	}
1880 
1881 	ASSERT(head != NULL);
1882 	ASSERT(tail != NULL);
1883 
1884 	/* zero bandwidth: drop all and return to interrupt mode */
1885 	mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
1886 	if (mac_srs->srs_bw->mac_bw_limit == 0) {
1887 		srs_rx->sr_drop_count += cnt;
1888 		ASSERT(mac_srs->srs_bw->mac_bw_sz >= sz);
1889 		mac_srs->srs_bw->mac_bw_sz -= sz;
1890 		mac_srs->srs_bw->mac_bw_drop_bytes += sz;
1891 		mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
1892 		mac_pkt_drop(NULL, NULL, head, B_FALSE);
1893 		goto leave_poll;
1894 	} else {
1895 		mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
1896 	}
1897 
1898 	if ((tid = mac_srs->srs_tid) != 0)
1899 		mac_srs->srs_tid = 0;
1900 
1901 	mac_srs->srs_state |= (SRS_PROC|proc_type);
1902 	MAC_SRS_WORKER_POLLING_ON(mac_srs);
1903 
1904 	/*
1905 	 * mcip is NULL for broadcast and multicast flows. The promisc
1906 	 * callbacks for broadcast and multicast packets are delivered from
1907 	 * mac_rx() and we don't need to worry about that case in this path
1908 	 */
1909 	if (mcip != NULL && mcip->mci_promisc_list != NULL) {
1910 		mutex_exit(&mac_srs->srs_lock);
1911 		mac_promisc_client_dispatch(mcip, head);
1912 		mutex_enter(&mac_srs->srs_lock);
1913 	}
1914 
1915 	/*
1916 	 * Check if SRS itself is doing the processing
1917 	 * This direct path does not apply when subflows are present. In this
1918 	 * case, packets need to be dispatched to a soft ring according to the
1919 	 * flow's bandwidth and other resources contraints.
1920 	 */
1921 	if (mac_srs->srs_type & SRST_NO_SOFT_RINGS) {
1922 		mac_direct_rx_t		proc;
1923 		void			*arg1;
1924 		mac_resource_handle_t	arg2;
1925 
1926 		/*
1927 		 * This is the case when a Rx is directly
1928 		 * assigned and we have a fully classified
1929 		 * protocol chain. We can deal with it in
1930 		 * one shot.
1931 		 */
1932 		proc = srs_rx->sr_func;
1933 		arg1 = srs_rx->sr_arg1;
1934 		arg2 = srs_rx->sr_arg2;
1935 
1936 		mac_srs->srs_state |= SRS_CLIENT_PROC;
1937 		mutex_exit(&mac_srs->srs_lock);
1938 		if (tid != 0) {
1939 			(void) untimeout(tid);
1940 			tid = 0;
1941 		}
1942 
1943 		proc(arg1, arg2, head, NULL);
1944 		/*
1945 		 * Decrement the size and count here itelf
1946 		 * since the packet has been processed.
1947 		 */
1948 		mutex_enter(&mac_srs->srs_lock);
1949 		MAC_UPDATE_SRS_COUNT_LOCKED(mac_srs, cnt);
1950 		MAC_UPDATE_SRS_SIZE_LOCKED(mac_srs, sz);
1951 
1952 		if (mac_srs->srs_state & SRS_CLIENT_WAIT)
1953 			cv_signal(&mac_srs->srs_client_cv);
1954 		mac_srs->srs_state &= ~SRS_CLIENT_PROC;
1955 	} else {
1956 		/* Some kind of softrings based fanout is required */
1957 		mutex_exit(&mac_srs->srs_lock);
1958 		if (tid != 0) {
1959 			(void) untimeout(tid);
1960 			tid = 0;
1961 		}
1962 
1963 		/*
1964 		 * Since the fanout routines can deal with chains,
1965 		 * shoot the entire chain up.
1966 		 */
1967 		if (mac_srs->srs_type & SRST_FANOUT_SRC_IP)
1968 			mac_rx_srs_fanout(mac_srs, head);
1969 		else
1970 			mac_rx_srs_proto_fanout(mac_srs, head);
1971 		mutex_enter(&mac_srs->srs_lock);
1972 	}
1973 
1974 	/*
1975 	 * Send the poll thread to pick up any packets arrived
1976 	 * so far. This also serves as the last check in case
1977 	 * nothing else is queued in the SRS. The poll thread
1978 	 * is signalled only in the case the drain was done
1979 	 * by the worker thread and SRS_WORKER is set. The
1980 	 * worker thread can run in parallel as long as the
1981 	 * SRS_WORKER flag is set. We we have nothing else to
1982 	 * process, we can exit while leaving SRS_PROC set
1983 	 * which gives the poll thread control to process and
1984 	 * cleanup once it returns from the NIC.
1985 	 *
1986 	 * If we have nothing else to process, we need to
1987 	 * ensure that we keep holding the srs_lock till
1988 	 * all the checks below are done and control is
1989 	 * handed to the poll thread if it was running.
1990 	 */
1991 	mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
1992 	if (!(mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED)) {
1993 		if (mac_srs->srs_first != NULL) {
1994 			if (proc_type == SRS_WORKER) {
1995 				mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
1996 				if (srs_rx->sr_poll_pkt_cnt <=
1997 				    srs_rx->sr_lowat)
1998 					MAC_SRS_POLL_RING(mac_srs);
1999 				goto again;
2000 			} else {
2001 				cv_signal(&mac_srs->srs_async);
2002 			}
2003 		}
2004 	}
2005 	mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2006 
2007 done:
2008 
2009 	if (mac_srs->srs_state & SRS_GET_PKTS) {
2010 		/*
2011 		 * Poll thread is already running. Leave the
2012 		 * SRS_RPOC set and hand over the control to
2013 		 * poll thread.
2014 		 */
2015 		mac_srs->srs_state &= ~proc_type;
2016 		return;
2017 	}
2018 
2019 	/*
2020 	 * If we can't process packets because we have exceeded
2021 	 * B/W limit for this tick, just set the timeout
2022 	 * and leave.
2023 	 *
2024 	 * Even if there are no packets queued in SRS, we
2025 	 * need to make sure that the shared counter is
2026 	 * clear and any associated softrings have cleared
2027 	 * all the backlog. Otherwise, leave the interface
2028 	 * in polling mode and the poll thread will get
2029 	 * signalled once the count goes down to zero.
2030 	 *
2031 	 * If someone is already draining the queue (SRS_PROC is
2032 	 * set) when the srs_poll_pkt_cnt goes down to zero,
2033 	 * then it means that drain is already running and we
2034 	 * will turn off polling at that time if there is
2035 	 * no backlog. As long as there are packets queued either
2036 	 * is soft ring set or its soft rings, we will leave
2037 	 * the interface in polling mode.
2038 	 */
2039 	mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
2040 	if ((mac_srs->srs_state & SRS_POLLING_CAPAB) &&
2041 	    ((mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED) ||
2042 	    (srs_rx->sr_poll_pkt_cnt > 0))) {
2043 		MAC_SRS_POLLING_ON(mac_srs);
2044 		mac_srs->srs_state &= ~(SRS_PROC|proc_type);
2045 		if ((mac_srs->srs_first != NULL) &&
2046 		    (mac_srs->srs_tid == NULL))
2047 			mac_srs->srs_tid = timeout(mac_srs_fire,
2048 			    mac_srs, 1);
2049 		mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2050 		return;
2051 	}
2052 	mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2053 
2054 leave_poll:
2055 
2056 	/* Nothing else to do. Get out of poll mode */
2057 	MAC_SRS_POLLING_OFF(mac_srs);
2058 	mac_srs->srs_state &= ~(SRS_PROC|proc_type);
2059 }
2060 
2061 /*
2062  * mac_srs_worker
2063  *
2064  * The SRS worker routine. Drains the queue when no one else is
2065  * processing it.
2066  */
2067 void
2068 mac_srs_worker(mac_soft_ring_set_t *mac_srs)
2069 {
2070 	kmutex_t 		*lock = &mac_srs->srs_lock;
2071 	kcondvar_t 		*async = &mac_srs->srs_async;
2072 	callb_cpr_t		cprinfo;
2073 	boolean_t		bw_ctl_flag;
2074 
2075 	CALLB_CPR_INIT(&cprinfo, lock, callb_generic_cpr, "srs_worker");
2076 	mutex_enter(lock);
2077 
2078 start:
2079 	for (;;) {
2080 		bw_ctl_flag = B_FALSE;
2081 		if (mac_srs->srs_type & SRST_BW_CONTROL) {
2082 			MAC_SRS_BW_LOCK(mac_srs);
2083 			MAC_SRS_CHECK_BW_CONTROL(mac_srs);
2084 			if (mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED)
2085 				bw_ctl_flag = B_TRUE;
2086 			MAC_SRS_BW_UNLOCK(mac_srs);
2087 		}
2088 		/*
2089 		 * The SRS_BW_ENFORCED flag may change since we have dropped
2090 		 * the mac_bw_lock. However the drain function can handle both
2091 		 * a drainable SRS or a bandwidth controlled SRS, and the
2092 		 * effect of scheduling a timeout is to wakeup the worker
2093 		 * thread which in turn will call the drain function. Since
2094 		 * we release the srs_lock atomically only in the cv_wait there
2095 		 * isn't a fear of waiting for ever.
2096 		 */
2097 		while (((mac_srs->srs_state & SRS_PROC) ||
2098 		    (mac_srs->srs_first == NULL) || bw_ctl_flag ||
2099 		    (mac_srs->srs_state & SRS_TX_BLOCKED)) &&
2100 		    !(mac_srs->srs_state & SRS_PAUSE)) {
2101 			/*
2102 			 * If we have packets queued and we are here
2103 			 * because B/W control is in place, we better
2104 			 * schedule the worker wakeup after 1 tick
2105 			 * to see if bandwidth control can be relaxed.
2106 			 */
2107 			if (bw_ctl_flag && mac_srs->srs_tid == NULL) {
2108 				/*
2109 				 * We need to ensure that a timer  is already
2110 				 * scheduled or we force  schedule one for
2111 				 * later so that we can continue processing
2112 				 * after this  quanta is over.
2113 				 */
2114 				mac_srs->srs_tid = timeout(mac_srs_fire,
2115 				    mac_srs, 1);
2116 			}
2117 wait:
2118 			CALLB_CPR_SAFE_BEGIN(&cprinfo);
2119 			cv_wait(async, lock);
2120 			CALLB_CPR_SAFE_END(&cprinfo, lock);
2121 
2122 			if (mac_srs->srs_state & SRS_PAUSE)
2123 				goto done;
2124 			if (mac_srs->srs_state & SRS_PROC)
2125 				goto wait;
2126 
2127 			if (mac_srs->srs_first != NULL &&
2128 			    mac_srs->srs_type & SRST_BW_CONTROL) {
2129 				MAC_SRS_BW_LOCK(mac_srs);
2130 				if (mac_srs->srs_bw->mac_bw_state &
2131 				    SRS_BW_ENFORCED) {
2132 					MAC_SRS_CHECK_BW_CONTROL(mac_srs);
2133 				}
2134 				bw_ctl_flag = mac_srs->srs_bw->mac_bw_state &
2135 				    SRS_BW_ENFORCED;
2136 				MAC_SRS_BW_UNLOCK(mac_srs);
2137 			}
2138 		}
2139 
2140 		if (mac_srs->srs_state & SRS_PAUSE)
2141 			goto done;
2142 		mac_srs->srs_drain_func(mac_srs, SRS_WORKER);
2143 	}
2144 done:
2145 	/*
2146 	 * The Rx SRS quiesce logic first cuts off packet supply to the SRS
2147 	 * from both hard and soft classifications and waits for such threads
2148 	 * to finish before signaling the worker. So at this point the only
2149 	 * thread left that could be competing with the worker is the poll
2150 	 * thread. In the case of Tx, there shouldn't be any thread holding
2151 	 * SRS_PROC at this point.
2152 	 */
2153 	if (!(mac_srs->srs_state & SRS_PROC)) {
2154 		mac_srs->srs_state |= SRS_PROC;
2155 	} else {
2156 		ASSERT((mac_srs->srs_type & SRST_TX) == 0);
2157 		/*
2158 		 * Poll thread still owns the SRS and is still running
2159 		 */
2160 		ASSERT((mac_srs->srs_poll_thr == NULL) ||
2161 		    ((mac_srs->srs_state & SRS_POLL_THR_OWNER) ==
2162 		    SRS_POLL_THR_OWNER));
2163 	}
2164 	mac_srs_worker_quiesce(mac_srs);
2165 	/*
2166 	 * Wait for the SRS_RESTART or SRS_CONDEMNED signal from the initiator
2167 	 * of the quiesce operation
2168 	 */
2169 	while (!(mac_srs->srs_state & (SRS_CONDEMNED | SRS_RESTART)))
2170 		cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock);
2171 
2172 	if (mac_srs->srs_state & SRS_RESTART) {
2173 		ASSERT(!(mac_srs->srs_state & SRS_CONDEMNED));
2174 		mac_srs_worker_restart(mac_srs);
2175 		mac_srs->srs_state &= ~SRS_PROC;
2176 		goto start;
2177 	}
2178 
2179 	if (!(mac_srs->srs_state & SRS_CONDEMNED_DONE))
2180 		mac_srs_worker_quiesce(mac_srs);
2181 
2182 	mac_srs->srs_state &= ~SRS_PROC;
2183 	/* The macro drops the srs_lock */
2184 	CALLB_CPR_EXIT(&cprinfo);
2185 	thread_exit();
2186 }
2187 
2188 /*
2189  * mac_rx_srs_subflow_process
2190  *
2191  * Receive side routine called from interrupt path when there are
2192  * sub flows present on this SRS.
2193  */
2194 /* ARGSUSED */
2195 void
2196 mac_rx_srs_subflow_process(void *arg, mac_resource_handle_t srs,
2197     mblk_t *mp_chain, boolean_t loopback)
2198 {
2199 	flow_entry_t		*flent = NULL;
2200 	flow_entry_t		*prev_flent = NULL;
2201 	mblk_t			*mp = NULL;
2202 	mblk_t			*tail = NULL;
2203 	mac_soft_ring_set_t	*mac_srs = (mac_soft_ring_set_t *)srs;
2204 	mac_client_impl_t	*mcip;
2205 
2206 	mcip = mac_srs->srs_mcip;
2207 	ASSERT(mcip != NULL);
2208 
2209 	/*
2210 	 * We need to determine the SRS for every packet
2211 	 * by walking the flow table, if we don't get any,
2212 	 * then we proceed using the SRS we came with.
2213 	 */
2214 	mp = tail = mp_chain;
2215 	while (mp != NULL) {
2216 
2217 		/*
2218 		 * We will increment the stats for the mactching subflow.
2219 		 * when we get the bytes/pkt count for the classified packets
2220 		 * later in mac_rx_srs_process.
2221 		 */
2222 		(void) mac_flow_lookup(mcip->mci_subflow_tab, mp,
2223 		    FLOW_INBOUND, &flent);
2224 
2225 		if (mp == mp_chain || flent == prev_flent) {
2226 			if (prev_flent != NULL)
2227 				FLOW_REFRELE(prev_flent);
2228 			prev_flent = flent;
2229 			flent = NULL;
2230 			tail = mp;
2231 			mp = mp->b_next;
2232 			continue;
2233 		}
2234 		tail->b_next = NULL;
2235 		/*
2236 		 * A null indicates, this is for the mac_srs itself.
2237 		 * XXX-venu : probably assert for fe_rx_srs_cnt == 0.
2238 		 */
2239 		if (prev_flent == NULL || prev_flent->fe_rx_srs_cnt == 0) {
2240 			mac_rx_srs_process(arg,
2241 			    (mac_resource_handle_t)mac_srs, mp_chain,
2242 			    loopback);
2243 		} else {
2244 			(prev_flent->fe_cb_fn)(prev_flent->fe_cb_arg1,
2245 			    prev_flent->fe_cb_arg2, mp_chain, loopback);
2246 			FLOW_REFRELE(prev_flent);
2247 		}
2248 		prev_flent = flent;
2249 		flent = NULL;
2250 		mp_chain = mp;
2251 		tail = mp;
2252 		mp = mp->b_next;
2253 	}
2254 	/* Last chain */
2255 	ASSERT(mp_chain != NULL);
2256 	if (prev_flent == NULL || prev_flent->fe_rx_srs_cnt == 0) {
2257 		mac_rx_srs_process(arg,
2258 		    (mac_resource_handle_t)mac_srs, mp_chain, loopback);
2259 	} else {
2260 		(prev_flent->fe_cb_fn)(prev_flent->fe_cb_arg1,
2261 		    prev_flent->fe_cb_arg2, mp_chain, loopback);
2262 		FLOW_REFRELE(prev_flent);
2263 	}
2264 }
2265 
2266 /*
2267  * mac_rx_srs_process
2268  *
2269  * Receive side routine called from the interrupt path.
2270  *
2271  * loopback is set to force a context switch on the loopback
2272  * path between MAC clients.
2273  */
2274 /* ARGSUSED */
2275 void
2276 mac_rx_srs_process(void *arg, mac_resource_handle_t srs, mblk_t *mp_chain,
2277     boolean_t loopback)
2278 {
2279 	mac_soft_ring_set_t	*mac_srs = (mac_soft_ring_set_t *)srs;
2280 	mblk_t			*mp, *tail, *head;
2281 	int			count = 0;
2282 	int			count1;
2283 	size_t			sz = 0;
2284 	size_t			chain_sz, sz1;
2285 	mac_bw_ctl_t		*mac_bw;
2286 	mac_client_impl_t	*smcip;
2287 	mac_srs_rx_t		*srs_rx = &mac_srs->srs_rx;
2288 
2289 	/*
2290 	 * Set the tail, count and sz. We set the sz irrespective
2291 	 * of whether we are doing B/W control or not for the
2292 	 * purpose of updating the stats.
2293 	 */
2294 	mp = tail = mp_chain;
2295 	while (mp != NULL) {
2296 		tail = mp;
2297 		count++;
2298 		sz += msgdsize(mp);
2299 		mp = mp->b_next;
2300 	}
2301 
2302 	mutex_enter(&mac_srs->srs_lock);
2303 	smcip = mac_srs->srs_mcip;
2304 
2305 	if (mac_srs->srs_type & SRST_FLOW || smcip == NULL) {
2306 		FLOW_STAT_UPDATE(mac_srs->srs_flent, rbytes, sz);
2307 		FLOW_STAT_UPDATE(mac_srs->srs_flent, ipackets, count);
2308 	}
2309 	if (smcip != NULL) {
2310 		smcip->mci_stat_ibytes += sz;
2311 		smcip->mci_stat_ipackets += count;
2312 	}
2313 
2314 	/*
2315 	 * If the SRS in already being processed; has been blanked;
2316 	 * can be processed by worker thread only; or the B/W limit
2317 	 * has been reached, then queue the chain and check if
2318 	 * worker thread needs to be awakend.
2319 	 */
2320 	if (mac_srs->srs_type & SRST_BW_CONTROL) {
2321 		mac_bw = mac_srs->srs_bw;
2322 		ASSERT(mac_bw != NULL);
2323 		mutex_enter(&mac_bw->mac_bw_lock);
2324 		/* Count the packets and bytes via interrupt */
2325 		srs_rx->sr_intr_count += count;
2326 		mac_bw->mac_bw_intr += sz;
2327 		if (mac_bw->mac_bw_limit == 0) {
2328 			/* zero bandwidth: drop all */
2329 			srs_rx->sr_drop_count += count;
2330 			mac_bw->mac_bw_drop_bytes += sz;
2331 			mutex_exit(&mac_bw->mac_bw_lock);
2332 			mutex_exit(&mac_srs->srs_lock);
2333 			mac_pkt_drop(NULL, NULL, mp_chain, B_FALSE);
2334 			return;
2335 		} else {
2336 			if ((mac_bw->mac_bw_sz + sz) <=
2337 			    mac_bw->mac_bw_drop_threshold) {
2338 				mutex_exit(&mac_bw->mac_bw_lock);
2339 				MAC_RX_SRS_ENQUEUE_CHAIN(mac_srs, mp_chain,
2340 				    tail, count, sz);
2341 			} else {
2342 				mp = mp_chain;
2343 				chain_sz = 0;
2344 				count1 = 0;
2345 				tail = NULL;
2346 				head = NULL;
2347 				while (mp != NULL) {
2348 					sz1 = msgdsize(mp);
2349 					if (mac_bw->mac_bw_sz + chain_sz + sz1 >
2350 					    mac_bw->mac_bw_drop_threshold)
2351 						break;
2352 					chain_sz += sz1;
2353 					count1++;
2354 					tail = mp;
2355 					mp = mp->b_next;
2356 				}
2357 				mutex_exit(&mac_bw->mac_bw_lock);
2358 				if (tail != NULL) {
2359 					head = tail->b_next;
2360 					tail->b_next = NULL;
2361 					MAC_RX_SRS_ENQUEUE_CHAIN(mac_srs,
2362 					    mp_chain, tail, count1, chain_sz);
2363 					sz -= chain_sz;
2364 					count -= count1;
2365 				} else {
2366 					/* Can't pick up any */
2367 					head = mp_chain;
2368 				}
2369 				if (head != NULL) {
2370 					/* Drop any packet over the threshold */
2371 					srs_rx->sr_drop_count += count;
2372 					mutex_enter(&mac_bw->mac_bw_lock);
2373 					mac_bw->mac_bw_drop_bytes += sz;
2374 					mutex_exit(&mac_bw->mac_bw_lock);
2375 					freemsgchain(head);
2376 				}
2377 			}
2378 			MAC_SRS_WORKER_WAKEUP(mac_srs);
2379 			mutex_exit(&mac_srs->srs_lock);
2380 			return;
2381 		}
2382 	}
2383 
2384 	/*
2385 	 * If the total number of packets queued in the SRS and
2386 	 * its associated soft rings exceeds the max allowed,
2387 	 * then drop the chain. If we are polling capable, this
2388 	 * shouldn't be happening.
2389 	 */
2390 	if (!(mac_srs->srs_type & SRST_BW_CONTROL) &&
2391 	    (srs_rx->sr_poll_pkt_cnt > srs_rx->sr_hiwat)) {
2392 		mac_bw = mac_srs->srs_bw;
2393 		srs_rx->sr_drop_count += count;
2394 		mutex_enter(&mac_bw->mac_bw_lock);
2395 		mac_bw->mac_bw_drop_bytes += sz;
2396 		mutex_exit(&mac_bw->mac_bw_lock);
2397 		freemsgchain(mp_chain);
2398 		mutex_exit(&mac_srs->srs_lock);
2399 		return;
2400 	}
2401 
2402 	MAC_RX_SRS_ENQUEUE_CHAIN(mac_srs, mp_chain, tail, count, sz);
2403 	/* Count the packets entering via interrupt path */
2404 	srs_rx->sr_intr_count += count;
2405 
2406 	if (!(mac_srs->srs_state & SRS_PROC)) {
2407 		/*
2408 		 * If we are coming via loopback or if we are not
2409 		 * optimizing for latency, we should signal the
2410 		 * worker thread.
2411 		 */
2412 		if (loopback || !(mac_srs->srs_state & SRS_LATENCY_OPT)) {
2413 			/*
2414 			 * For loopback, We need to let the worker take
2415 			 * over as we don't want to continue in the same
2416 			 * thread even if we can. This could lead to stack
2417 			 * overflows and may also end up using
2418 			 * resources (cpu) incorrectly.
2419 			 */
2420 			cv_signal(&mac_srs->srs_async);
2421 		} else {
2422 			/*
2423 			 * Seems like no one is processing the SRS and
2424 			 * there is no backlog. We also inline process
2425 			 * our packet if its a single packet in non
2426 			 * latency optimized case (in latency optimized
2427 			 * case, we inline process chains of any size).
2428 			 */
2429 			mac_srs->srs_drain_func(mac_srs, SRS_PROC_FAST);
2430 		}
2431 	}
2432 	mutex_exit(&mac_srs->srs_lock);
2433 }
2434 
2435 /* TX SIDE ROUTINES (RUNTIME) */
2436 
2437 /*
2438  * mac_tx_srs_no_desc
2439  *
2440  * This routine is called by Tx single ring default mode
2441  * when Tx ring runs out of descs.
2442  */
2443 mac_tx_cookie_t
2444 mac_tx_srs_no_desc(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
2445     uint16_t flag, mblk_t **ret_mp)
2446 {
2447 	mac_tx_cookie_t cookie = NULL;
2448 	mac_srs_tx_t *srs_tx = &mac_srs->srs_tx;
2449 	boolean_t wakeup_worker = B_TRUE;
2450 	uint32_t tx_mode = srs_tx->st_mode;
2451 	int cnt, sz;
2452 	mblk_t *tail;
2453 
2454 	ASSERT(tx_mode == SRS_TX_DEFAULT || tx_mode == SRS_TX_BW);
2455 	if (flag & MAC_DROP_ON_NO_DESC) {
2456 		MAC_TX_SRS_DROP_MESSAGE(mac_srs, mp_chain, cookie);
2457 	} else {
2458 		if (mac_srs->srs_first != NULL)
2459 			wakeup_worker = B_FALSE;
2460 		MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz);
2461 		if (flag & MAC_TX_NO_ENQUEUE) {
2462 			/*
2463 			 * If TX_QUEUED is not set, queue the
2464 			 * packet and let mac_tx_srs_drain()
2465 			 * set the TX_BLOCKED bit for the
2466 			 * reasons explained above. Otherwise,
2467 			 * return the mblks.
2468 			 */
2469 			if (wakeup_worker) {
2470 				MAC_TX_SRS_ENQUEUE_CHAIN(mac_srs,
2471 				    mp_chain, tail, cnt, sz);
2472 			} else {
2473 				MAC_TX_SET_NO_ENQUEUE(mac_srs,
2474 				    mp_chain, ret_mp, cookie);
2475 			}
2476 		} else {
2477 			MAC_TX_SRS_TEST_HIWAT(mac_srs, mp_chain,
2478 			    tail, cnt, sz, cookie);
2479 		}
2480 		if (wakeup_worker)
2481 			cv_signal(&mac_srs->srs_async);
2482 	}
2483 	return (cookie);
2484 }
2485 
2486 /*
2487  * mac_tx_srs_enqueue
2488  *
2489  * This routine is called when Tx SRS is operating in either serializer
2490  * or bandwidth mode. In serializer mode, a packet will get enqueued
2491  * when a thread cannot enter SRS exclusively. In bandwidth mode,
2492  * packets gets queued if allowed byte-count limit for a tick is
2493  * exceeded. The action that gets taken when MAC_DROP_ON_NO_DESC and
2494  * MAC_TX_NO_ENQUEUE is set is different than when operaing in either
2495  * the default mode or fanout mode. Here packets get dropped or
2496  * returned back to the caller only after hi-watermark worth of data
2497  * is queued.
2498  */
2499 static mac_tx_cookie_t
2500 mac_tx_srs_enqueue(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
2501     uint16_t flag, uintptr_t fanout_hint, mblk_t **ret_mp)
2502 {
2503 	mac_tx_cookie_t cookie = NULL;
2504 	int cnt, sz;
2505 	mblk_t *tail;
2506 	boolean_t wakeup_worker = B_TRUE;
2507 
2508 	/*
2509 	 * Ignore fanout hint if we don't have multiple tx rings.
2510 	 */
2511 	if (!TX_MULTI_RING_MODE(mac_srs))
2512 		fanout_hint = 0;
2513 
2514 	if (mac_srs->srs_first != NULL)
2515 		wakeup_worker = B_FALSE;
2516 	MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz);
2517 	if (flag & MAC_DROP_ON_NO_DESC) {
2518 		if (mac_srs->srs_count > mac_srs->srs_tx.st_hiwat) {
2519 			MAC_TX_SRS_DROP_MESSAGE(mac_srs, mp_chain, cookie);
2520 		} else {
2521 			MAC_TX_SRS_ENQUEUE_CHAIN(mac_srs,
2522 			    mp_chain, tail, cnt, sz);
2523 		}
2524 	} else if (flag & MAC_TX_NO_ENQUEUE) {
2525 		if ((mac_srs->srs_count > mac_srs->srs_tx.st_hiwat) ||
2526 		    (mac_srs->srs_state & SRS_TX_WAKEUP_CLIENT)) {
2527 			MAC_TX_SET_NO_ENQUEUE(mac_srs, mp_chain,
2528 			    ret_mp, cookie);
2529 		} else {
2530 			mp_chain->b_prev = (mblk_t *)fanout_hint;
2531 			MAC_TX_SRS_ENQUEUE_CHAIN(mac_srs,
2532 			    mp_chain, tail, cnt, sz);
2533 		}
2534 	} else {
2535 		/*
2536 		 * If you are BW_ENFORCED, just enqueue the
2537 		 * packet. srs_worker will drain it at the
2538 		 * prescribed rate. Before enqueueing, save
2539 		 * the fanout hint.
2540 		 */
2541 		mp_chain->b_prev = (mblk_t *)fanout_hint;
2542 		MAC_TX_SRS_TEST_HIWAT(mac_srs, mp_chain,
2543 		    tail, cnt, sz, cookie);
2544 	}
2545 	if (wakeup_worker)
2546 		cv_signal(&mac_srs->srs_async);
2547 	return (cookie);
2548 }
2549 
2550 /*
2551  * There are five tx modes:
2552  *
2553  * 1) Default mode (SRS_TX_DEFAULT)
2554  * 2) Serialization mode (SRS_TX_SERIALIZE)
2555  * 3) Fanout mode (SRS_TX_FANOUT)
2556  * 4) Bandwdith mode (SRS_TX_BW)
2557  * 5) Fanout and Bandwidth mode (SRS_TX_BW_FANOUT)
2558  *
2559  * The tx mode in which an SRS operates is decided in mac_tx_srs_setup()
2560  * based on the number of Tx rings requested for an SRS and whether
2561  * bandwidth control is requested or not.
2562  *
2563  * In the default mode (i.e., no fanout/no bandwidth), the SRS acts as a
2564  * pass-thru. Packets will go directly to mac_tx_send(). When the underlying
2565  * Tx ring runs out of Tx descs, it starts queueing up packets in SRS.
2566  * When flow-control is relieved, the srs_worker drains the queued
2567  * packets and informs blocked clients to restart sending packets.
2568  *
2569  * In the SRS_TX_SERIALIZE mode, all calls to mac_tx() are serialized.
2570  *
2571  * In the SRS_TX_FANOUT mode, packets will be fanned out to multiple
2572  * Tx rings. Each Tx ring will have a soft ring associated with it.
2573  * These soft rings will be hung off the Tx SRS. Queueing if it happens
2574  * due to lack of Tx desc will be in individual soft ring (and not srs)
2575  * associated with Tx ring.
2576  *
2577  * In the TX_BW mode, tx srs will allow packets to go down to Tx ring
2578  * only if bw is available. Otherwise the packets will be queued in
2579  * SRS. If fanout to multiple Tx rings is configured, the packets will
2580  * be fanned out among the soft rings associated with the Tx rings.
2581  *
2582  * Four flags are used in srs_state for indicating flow control
2583  * conditions : SRS_TX_BLOCKED, SRS_TX_HIWAT, SRS_TX_WAKEUP_CLIENT.
2584  * SRS_TX_BLOCKED indicates out of Tx descs. SRS expects a wakeup from the
2585  * driver below.
2586  * SRS_TX_HIWAT indicates packet count enqueued in Tx SRS exceeded Tx hiwat
2587  * and flow-control pressure is applied back to clients. The clients expect
2588  * wakeup when flow-control is relieved.
2589  * SRS_TX_WAKEUP_CLIENT get set when (flag == MAC_TX_NO_ENQUEUE) and mblk
2590  * got returned back to client either due to lack of Tx descs or due to bw
2591  * control reasons. The clients expect a wakeup when condition is relieved.
2592  *
2593  * The fourth argument to mac_tx() is the flag. Normally it will be 0 but
2594  * some clients set the following values too: MAC_DROP_ON_NO_DESC,
2595  * MAC_TX_NO_ENQUEUE
2596  * Mac clients that do not want packets to be enqueued in the mac layer set
2597  * MAC_DROP_ON_NO_DESC value. The packets won't be queued in the Tx SRS or
2598  * Tx soft rings but instead get dropped when the NIC runs out of desc. The
2599  * behaviour of this flag is different when the Tx is running in serializer
2600  * or bandwidth mode. Under these (Serializer, bandwidth) modes, the packet
2601  * get dropped when Tx high watermark is reached.
2602  * There are some mac clients like vsw, aggr that want the mblks to be
2603  * returned back to clients instead of being queued in Tx SRS (or Tx soft
2604  * rings) under flow-control (i.e., out of desc or exceeding bw limits)
2605  * conditions. These clients call mac_tx() with MAC_TX_NO_ENQUEUE flag set.
2606  * In the default and Tx fanout mode, the un-transmitted mblks will be
2607  * returned back to the clients when the driver runs out of Tx descs.
2608  * SRS_TX_WAKEUP_CLIENT (or S_RING_WAKEUP_CLIENT) will be set in SRS (or
2609  * soft ring) so that the clients can be woken up when Tx desc become
2610  * available. When running in serializer or bandwidth mode mode,
2611  * SRS_TX_WAKEUP_CLIENT will be set when tx hi-watermark is reached.
2612  */
2613 
2614 mac_tx_func_t
2615 mac_tx_get_func(uint32_t mode)
2616 {
2617 	return (mac_tx_mode_list[mode].mac_tx_func);
2618 }
2619 
2620 /* ARGSUSED */
2621 static mac_tx_cookie_t
2622 mac_tx_single_ring_mode(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
2623     uintptr_t fanout_hint, uint16_t flag, mblk_t **ret_mp)
2624 {
2625 	mac_srs_tx_t		*srs_tx = &mac_srs->srs_tx;
2626 	boolean_t		is_subflow;
2627 	mac_tx_stats_t		stats;
2628 	mac_tx_cookie_t		cookie = NULL;
2629 
2630 	ASSERT(srs_tx->st_mode == SRS_TX_DEFAULT);
2631 
2632 	/* Regular case with a single Tx ring */
2633 	/*
2634 	 * SRS_TX_BLOCKED is set when underlying NIC runs
2635 	 * out of Tx descs and messages start getting
2636 	 * queued. It won't get reset until
2637 	 * tx_srs_drain() completely drains out the
2638 	 * messages.
2639 	 */
2640 	if ((mac_srs->srs_state & SRS_ENQUEUED) != 0) {
2641 		/* Tx descs/resources not available */
2642 		mutex_enter(&mac_srs->srs_lock);
2643 		if ((mac_srs->srs_state & SRS_ENQUEUED) != 0) {
2644 			cookie = mac_tx_srs_no_desc(mac_srs, mp_chain,
2645 			    flag, ret_mp);
2646 			mutex_exit(&mac_srs->srs_lock);
2647 			return (cookie);
2648 		}
2649 		/*
2650 		 * While we were computing mblk count, the
2651 		 * flow control condition got relieved.
2652 		 * Continue with the transmission.
2653 		 */
2654 		mutex_exit(&mac_srs->srs_lock);
2655 	}
2656 
2657 	is_subflow = ((mac_srs->srs_type & SRST_FLOW) != 0);
2658 
2659 	mp_chain = mac_tx_send(srs_tx->st_arg1, srs_tx->st_arg2,
2660 	    mp_chain, (is_subflow ? &stats : NULL));
2661 
2662 	/*
2663 	 * Multiple threads could be here sending packets.
2664 	 * Under such conditions, it is not possible to
2665 	 * automically set SRS_TX_BLOCKED bit to indicate
2666 	 * out of tx desc condition. To atomically set
2667 	 * this, we queue the returned packet and do
2668 	 * the setting of SRS_TX_BLOCKED in
2669 	 * mac_tx_srs_drain().
2670 	 */
2671 	if (mp_chain != NULL) {
2672 		mutex_enter(&mac_srs->srs_lock);
2673 		cookie = mac_tx_srs_no_desc(mac_srs, mp_chain, flag, ret_mp);
2674 		mutex_exit(&mac_srs->srs_lock);
2675 		return (cookie);
2676 	}
2677 
2678 	if (is_subflow)
2679 		FLOW_TX_STATS_UPDATE(mac_srs->srs_flent, &stats);
2680 
2681 	return (NULL);
2682 }
2683 
2684 /*
2685  * mac_tx_serialize_mode
2686  *
2687  * This is an experimental mode implemented as per the request of PAE.
2688  * In this mode, all callers attempting to send a packet to the NIC
2689  * will get serialized. Only one thread at any time will access the
2690  * NIC to send the packet out.
2691  */
2692 /* ARGSUSED */
2693 static mac_tx_cookie_t
2694 mac_tx_serializer_mode(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
2695     uintptr_t fanout_hint, uint16_t flag, mblk_t **ret_mp)
2696 {
2697 	boolean_t		is_subflow;
2698 	mac_tx_stats_t		stats;
2699 	mac_tx_cookie_t		cookie = NULL;
2700 	mac_srs_tx_t		*srs_tx = &mac_srs->srs_tx;
2701 
2702 	/* Single ring, serialize below */
2703 	ASSERT(srs_tx->st_mode == SRS_TX_SERIALIZE);
2704 	mutex_enter(&mac_srs->srs_lock);
2705 	if ((mac_srs->srs_first != NULL) ||
2706 	    (mac_srs->srs_state & SRS_PROC)) {
2707 		/*
2708 		 * In serialization mode, queue all packets until
2709 		 * TX_HIWAT is set.
2710 		 * If drop bit is set, drop if TX_HIWAT is set.
2711 		 * If no_enqueue is set, still enqueue until hiwat
2712 		 * is set and return mblks after TX_HIWAT is set.
2713 		 */
2714 		cookie = mac_tx_srs_enqueue(mac_srs, mp_chain,
2715 		    flag, NULL, ret_mp);
2716 		mutex_exit(&mac_srs->srs_lock);
2717 		return (cookie);
2718 	}
2719 	/*
2720 	 * No packets queued, nothing on proc and no flow
2721 	 * control condition. Fast-path, ok. Do inline
2722 	 * processing.
2723 	 */
2724 	mac_srs->srs_state |= SRS_PROC;
2725 	mutex_exit(&mac_srs->srs_lock);
2726 
2727 	is_subflow = ((mac_srs->srs_type & SRST_FLOW) != 0);
2728 
2729 	mp_chain = mac_tx_send(srs_tx->st_arg1, srs_tx->st_arg2,
2730 	    mp_chain, (is_subflow ? &stats : NULL));
2731 
2732 	mutex_enter(&mac_srs->srs_lock);
2733 	mac_srs->srs_state &= ~SRS_PROC;
2734 	if (mp_chain != NULL) {
2735 		cookie = mac_tx_srs_enqueue(mac_srs,
2736 		    mp_chain, flag, NULL, ret_mp);
2737 	}
2738 	if (mac_srs->srs_first != NULL) {
2739 		/*
2740 		 * We processed inline our packet and a new
2741 		 * packet/s got queued while we were
2742 		 * processing. Wakeup srs worker
2743 		 */
2744 		cv_signal(&mac_srs->srs_async);
2745 	}
2746 	mutex_exit(&mac_srs->srs_lock);
2747 
2748 	if (is_subflow && cookie == NULL)
2749 		FLOW_TX_STATS_UPDATE(mac_srs->srs_flent, &stats);
2750 
2751 	return (cookie);
2752 }
2753 
2754 /*
2755  * mac_tx_fanout_mode
2756  *
2757  * In this mode, the SRS will have access to multiple Tx rings to send
2758  * the packet out. The fanout hint that is passed as an argument is
2759  * used to find an appropriate ring to fanout the traffic. Each Tx
2760  * ring, in turn,  will have a soft ring associated with it. If a Tx
2761  * ring runs out of Tx desc's the returned packet will be queued in
2762  * the soft ring associated with that Tx ring. The srs itself will not
2763  * queue any packets.
2764  */
2765 
2766 #define	MAC_TX_SOFT_RING_PROCESS(chain) {		       		\
2767 	index = COMPUTE_INDEX(hash, mac_srs->srs_oth_ring_count),	\
2768 	softring = mac_srs->srs_oth_soft_rings[index];			\
2769 	cookie = mac_tx_soft_ring_process(softring, chain, flag, ret_mp); \
2770 	DTRACE_PROBE2(tx__fanout, uint64_t, hash, uint_t, index);	\
2771 }
2772 
2773 static mac_tx_cookie_t
2774 mac_tx_fanout_mode(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
2775     uintptr_t fanout_hint, uint16_t flag, mblk_t **ret_mp)
2776 {
2777 	mac_soft_ring_t		*softring;
2778 	uint64_t		hash;
2779 	uint_t			index;
2780 	mac_tx_cookie_t		cookie = NULL;
2781 
2782 	ASSERT(mac_srs->srs_tx.st_mode == SRS_TX_FANOUT);
2783 	if (fanout_hint != 0) {
2784 		/*
2785 		 * The hint is specified by the caller, simply pass the
2786 		 * whole chain to the soft ring.
2787 		 */
2788 		hash = HASH_HINT(fanout_hint);
2789 		MAC_TX_SOFT_RING_PROCESS(mp_chain);
2790 	} else {
2791 		mblk_t *last_mp, *cur_mp, *sub_chain;
2792 		uint64_t last_hash = 0;
2793 		uint_t media = mac_srs->srs_mcip->mci_mip->mi_info.mi_media;
2794 
2795 		/*
2796 		 * Compute the hash from the contents (headers) of the
2797 		 * packets of the mblk chain. Split the chains into
2798 		 * subchains of the same conversation.
2799 		 *
2800 		 * Since there may be more than one ring used for
2801 		 * sub-chains of the same call, and since the caller
2802 		 * does not maintain per conversation state since it
2803 		 * passed a zero hint, unsent subchains will be
2804 		 * dropped.
2805 		 */
2806 
2807 		flag |= MAC_DROP_ON_NO_DESC;
2808 		ret_mp = NULL;
2809 
2810 		ASSERT(ret_mp == NULL);
2811 
2812 		sub_chain = NULL;
2813 		last_mp = NULL;
2814 
2815 		for (cur_mp = mp_chain; cur_mp != NULL;
2816 		    cur_mp = cur_mp->b_next) {
2817 			hash = mac_pkt_hash(media, cur_mp, MAC_PKT_HASH_L4,
2818 			    B_TRUE);
2819 			if (last_hash != 0 && hash != last_hash) {
2820 				/*
2821 				 * Starting a different subchain, send current
2822 				 * chain out.
2823 				 */
2824 				ASSERT(last_mp != NULL);
2825 				last_mp->b_next = NULL;
2826 				MAC_TX_SOFT_RING_PROCESS(sub_chain);
2827 				sub_chain = NULL;
2828 			}
2829 
2830 			/* add packet to subchain */
2831 			if (sub_chain == NULL)
2832 				sub_chain = cur_mp;
2833 			last_mp = cur_mp;
2834 			last_hash = hash;
2835 		}
2836 
2837 		if (sub_chain != NULL) {
2838 			/* send last subchain */
2839 			ASSERT(last_mp != NULL);
2840 			last_mp->b_next = NULL;
2841 			MAC_TX_SOFT_RING_PROCESS(sub_chain);
2842 		}
2843 
2844 		cookie = NULL;
2845 	}
2846 
2847 	return (cookie);
2848 }
2849 
2850 /*
2851  * mac_tx_bw_mode
2852  *
2853  * In the bandwidth mode, Tx srs will allow packets to go down to Tx ring
2854  * only if bw is available. Otherwise the packets will be queued in
2855  * SRS. If the SRS has multiple Tx rings, then packets will get fanned
2856  * out to a Tx rings.
2857  */
2858 static mac_tx_cookie_t
2859 mac_tx_bw_mode(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
2860     uintptr_t fanout_hint, uint16_t flag, mblk_t **ret_mp)
2861 {
2862 	int			cnt, sz;
2863 	mblk_t			*tail;
2864 	mac_tx_cookie_t		cookie = NULL;
2865 	mac_srs_tx_t		*srs_tx = &mac_srs->srs_tx;
2866 	clock_t			now;
2867 
2868 	ASSERT(TX_BANDWIDTH_MODE(mac_srs));
2869 	ASSERT(mac_srs->srs_type & SRST_BW_CONTROL);
2870 	mutex_enter(&mac_srs->srs_lock);
2871 	if (mac_srs->srs_bw->mac_bw_limit == 0) {
2872 		/*
2873 		 * zero bandwidth, no traffic is sent: drop the packets,
2874 		 * or return the whole chain if the caller requests all
2875 		 * unsent packets back.
2876 		 */
2877 		if (flag & MAC_TX_NO_ENQUEUE) {
2878 			cookie = (mac_tx_cookie_t)mac_srs;
2879 			*ret_mp = mp_chain;
2880 		} else {
2881 			MAC_TX_SRS_DROP_MESSAGE(mac_srs, mp_chain, cookie);
2882 		}
2883 		mutex_exit(&mac_srs->srs_lock);
2884 		return (cookie);
2885 	} else if ((mac_srs->srs_first != NULL) ||
2886 	    (mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED)) {
2887 		cookie = mac_tx_srs_enqueue(mac_srs, mp_chain, flag,
2888 		    fanout_hint, ret_mp);
2889 		mutex_exit(&mac_srs->srs_lock);
2890 		return (cookie);
2891 	}
2892 	MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz);
2893 	now = ddi_get_lbolt();
2894 	if (mac_srs->srs_bw->mac_bw_curr_time != now) {
2895 		mac_srs->srs_bw->mac_bw_curr_time = now;
2896 		mac_srs->srs_bw->mac_bw_used = 0;
2897 	} else if (mac_srs->srs_bw->mac_bw_used >
2898 	    mac_srs->srs_bw->mac_bw_limit) {
2899 		mac_srs->srs_bw->mac_bw_state |= SRS_BW_ENFORCED;
2900 		MAC_TX_SRS_ENQUEUE_CHAIN(mac_srs,
2901 		    mp_chain, tail, cnt, sz);
2902 		/*
2903 		 * Wakeup worker thread. Note that worker
2904 		 * thread has to be woken up so that it
2905 		 * can fire up the timer to be woken up
2906 		 * on the next tick. Also once
2907 		 * BW_ENFORCED is set, it can only be
2908 		 * reset by srs_worker thread. Until then
2909 		 * all packets will get queued up in SRS
2910 		 * and hence this this code path won't be
2911 		 * entered until BW_ENFORCED is reset.
2912 		 */
2913 		cv_signal(&mac_srs->srs_async);
2914 		mutex_exit(&mac_srs->srs_lock);
2915 		return (cookie);
2916 	}
2917 
2918 	mac_srs->srs_bw->mac_bw_used += sz;
2919 	mutex_exit(&mac_srs->srs_lock);
2920 
2921 	if (srs_tx->st_mode == SRS_TX_BW_FANOUT) {
2922 		mac_soft_ring_t *softring;
2923 		uint_t indx, hash;
2924 
2925 		hash = HASH_HINT(fanout_hint);
2926 		indx = COMPUTE_INDEX(hash,
2927 		    mac_srs->srs_oth_ring_count);
2928 		softring = mac_srs->srs_oth_soft_rings[indx];
2929 		return (mac_tx_soft_ring_process(softring, mp_chain, flag,
2930 		    ret_mp));
2931 	} else {
2932 		boolean_t		is_subflow;
2933 		mac_tx_stats_t		stats;
2934 
2935 		is_subflow = ((mac_srs->srs_type & SRST_FLOW) != 0);
2936 
2937 		mp_chain = mac_tx_send(srs_tx->st_arg1, srs_tx->st_arg2,
2938 		    mp_chain, (is_subflow ? &stats : NULL));
2939 
2940 		if (mp_chain != NULL) {
2941 			mutex_enter(&mac_srs->srs_lock);
2942 			MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz);
2943 			if (mac_srs->srs_bw->mac_bw_used > sz)
2944 				mac_srs->srs_bw->mac_bw_used -= sz;
2945 			else
2946 				mac_srs->srs_bw->mac_bw_used = 0;
2947 			cookie = mac_tx_srs_enqueue(mac_srs, mp_chain, flag,
2948 			    fanout_hint, ret_mp);
2949 			mutex_exit(&mac_srs->srs_lock);
2950 			return (cookie);
2951 		}
2952 		if (is_subflow)
2953 			FLOW_TX_STATS_UPDATE(mac_srs->srs_flent, &stats);
2954 
2955 		return (NULL);
2956 	}
2957 }
2958 
2959 /* ARGSUSED */
2960 void
2961 mac_tx_srs_drain(mac_soft_ring_set_t *mac_srs, uint_t proc_type)
2962 {
2963 	mblk_t			*head, *tail;
2964 	size_t			sz;
2965 	uint32_t		tx_mode;
2966 	uint_t			saved_pkt_count;
2967 	boolean_t		is_subflow;
2968 	mac_tx_stats_t		stats;
2969 	mac_srs_tx_t		*srs_tx = &mac_srs->srs_tx;
2970 	clock_t			now;
2971 
2972 	saved_pkt_count = 0;
2973 	ASSERT(mutex_owned(&mac_srs->srs_lock));
2974 	ASSERT(!(mac_srs->srs_state & SRS_PROC));
2975 
2976 	mac_srs->srs_state |= SRS_PROC;
2977 
2978 	is_subflow = ((mac_srs->srs_type & SRST_FLOW) != 0);
2979 	tx_mode = srs_tx->st_mode;
2980 	if (tx_mode == SRS_TX_DEFAULT || tx_mode == SRS_TX_SERIALIZE) {
2981 		if (mac_srs->srs_first != NULL) {
2982 			head = mac_srs->srs_first;
2983 			tail = mac_srs->srs_last;
2984 			saved_pkt_count = mac_srs->srs_count;
2985 			mac_srs->srs_first = NULL;
2986 			mac_srs->srs_last = NULL;
2987 			mac_srs->srs_count = 0;
2988 			mutex_exit(&mac_srs->srs_lock);
2989 
2990 			head = mac_tx_send(srs_tx->st_arg1, srs_tx->st_arg2,
2991 			    head, &stats);
2992 
2993 			mutex_enter(&mac_srs->srs_lock);
2994 			if (head != NULL) {
2995 				/* Device out of tx desc, set block */
2996 				if (head->b_next == NULL)
2997 					VERIFY(head == tail);
2998 				tail->b_next = mac_srs->srs_first;
2999 				mac_srs->srs_first = head;
3000 				mac_srs->srs_count +=
3001 				    (saved_pkt_count - stats.ts_opackets);
3002 				if (mac_srs->srs_last == NULL)
3003 					mac_srs->srs_last = tail;
3004 				MAC_TX_SRS_BLOCK(mac_srs, head);
3005 			} else {
3006 				srs_tx->st_woken_up = B_FALSE;
3007 				if (is_subflow) {
3008 					FLOW_TX_STATS_UPDATE(
3009 					    mac_srs->srs_flent, &stats);
3010 				}
3011 			}
3012 		}
3013 	} else if (tx_mode == SRS_TX_BW) {
3014 		/*
3015 		 * We are here because the timer fired and we have some data
3016 		 * to tranmit. Also mac_tx_srs_worker should have reset
3017 		 * SRS_BW_ENFORCED flag
3018 		 */
3019 		ASSERT(!(mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED));
3020 		head = tail = mac_srs->srs_first;
3021 		while (mac_srs->srs_first != NULL) {
3022 			tail = mac_srs->srs_first;
3023 			tail->b_prev = NULL;
3024 			mac_srs->srs_first = tail->b_next;
3025 			if (mac_srs->srs_first == NULL)
3026 				mac_srs->srs_last = NULL;
3027 			mac_srs->srs_count--;
3028 			sz = msgdsize(tail);
3029 			mac_srs->srs_size -= sz;
3030 			saved_pkt_count++;
3031 			MAC_TX_UPDATE_BW_INFO(mac_srs, sz);
3032 
3033 			if (mac_srs->srs_bw->mac_bw_used <
3034 			    mac_srs->srs_bw->mac_bw_limit)
3035 				continue;
3036 
3037 			now = ddi_get_lbolt();
3038 			if (mac_srs->srs_bw->mac_bw_curr_time != now) {
3039 				mac_srs->srs_bw->mac_bw_curr_time = now;
3040 				mac_srs->srs_bw->mac_bw_used = sz;
3041 				continue;
3042 			}
3043 			mac_srs->srs_bw->mac_bw_state |= SRS_BW_ENFORCED;
3044 			break;
3045 		}
3046 
3047 		ASSERT((head == NULL && tail == NULL) ||
3048 		    (head != NULL && tail != NULL));
3049 		if (tail != NULL) {
3050 			tail->b_next = NULL;
3051 			mutex_exit(&mac_srs->srs_lock);
3052 
3053 			head = mac_tx_send(srs_tx->st_arg1, srs_tx->st_arg2,
3054 			    head, &stats);
3055 
3056 			mutex_enter(&mac_srs->srs_lock);
3057 			if (head != NULL) {
3058 				uint_t size_sent;
3059 
3060 				/* Device out of tx desc, set block */
3061 				if (head->b_next == NULL)
3062 					VERIFY(head == tail);
3063 				tail->b_next = mac_srs->srs_first;
3064 				mac_srs->srs_first = head;
3065 				mac_srs->srs_count +=
3066 				    (saved_pkt_count - stats.ts_opackets);
3067 				if (mac_srs->srs_last == NULL)
3068 					mac_srs->srs_last = tail;
3069 				size_sent = sz - stats.ts_obytes;
3070 				mac_srs->srs_size += size_sent;
3071 				mac_srs->srs_bw->mac_bw_sz += size_sent;
3072 				if (mac_srs->srs_bw->mac_bw_used > size_sent) {
3073 					mac_srs->srs_bw->mac_bw_used -=
3074 					    size_sent;
3075 				} else {
3076 					mac_srs->srs_bw->mac_bw_used = 0;
3077 				}
3078 				MAC_TX_SRS_BLOCK(mac_srs, head);
3079 			} else {
3080 				srs_tx->st_woken_up = B_FALSE;
3081 				if (is_subflow) {
3082 					FLOW_TX_STATS_UPDATE(
3083 					    mac_srs->srs_flent, &stats);
3084 				}
3085 			}
3086 		}
3087 	} else if (tx_mode == SRS_TX_BW_FANOUT) {
3088 		mblk_t *prev;
3089 		mac_soft_ring_t *softring;
3090 		uint64_t hint;
3091 
3092 		/*
3093 		 * We are here because the timer fired and we
3094 		 * have some quota to tranmit.
3095 		 */
3096 		prev = NULL;
3097 		head = tail = mac_srs->srs_first;
3098 		while (mac_srs->srs_first != NULL) {
3099 			tail = mac_srs->srs_first;
3100 			mac_srs->srs_first = tail->b_next;
3101 			if (mac_srs->srs_first == NULL)
3102 				mac_srs->srs_last = NULL;
3103 			mac_srs->srs_count--;
3104 			sz = msgdsize(tail);
3105 			mac_srs->srs_size -= sz;
3106 			mac_srs->srs_bw->mac_bw_used += sz;
3107 			if (prev == NULL)
3108 				hint = (ulong_t)tail->b_prev;
3109 			if (hint != (ulong_t)tail->b_prev) {
3110 				prev->b_next = NULL;
3111 				mutex_exit(&mac_srs->srs_lock);
3112 				TX_SRS_TO_SOFT_RING(mac_srs, head, hint);
3113 				head = tail;
3114 				hint = (ulong_t)tail->b_prev;
3115 				mutex_enter(&mac_srs->srs_lock);
3116 			}
3117 
3118 			prev = tail;
3119 			tail->b_prev = NULL;
3120 			if (mac_srs->srs_bw->mac_bw_used <
3121 			    mac_srs->srs_bw->mac_bw_limit)
3122 				continue;
3123 
3124 			now = ddi_get_lbolt();
3125 			if (mac_srs->srs_bw->mac_bw_curr_time != now) {
3126 				mac_srs->srs_bw->mac_bw_curr_time = now;
3127 				mac_srs->srs_bw->mac_bw_used = 0;
3128 				continue;
3129 			}
3130 			mac_srs->srs_bw->mac_bw_state |= SRS_BW_ENFORCED;
3131 			break;
3132 		}
3133 		ASSERT((head == NULL && tail == NULL) ||
3134 		    (head != NULL && tail != NULL));
3135 		if (tail != NULL) {
3136 			tail->b_next = NULL;
3137 			mutex_exit(&mac_srs->srs_lock);
3138 			TX_SRS_TO_SOFT_RING(mac_srs, head, hint);
3139 			mutex_enter(&mac_srs->srs_lock);
3140 		}
3141 	}
3142 	/*
3143 	 * SRS_TX_FANOUT case not considered here because packets
3144 	 * won't be queued in the SRS for this case. Packets will
3145 	 * be sent directly to soft rings underneath and if there
3146 	 * is any queueing at all, it would be in Tx side soft
3147 	 * rings.
3148 	 */
3149 
3150 	/*
3151 	 * When srs_count becomes 0, reset SRS_TX_HIWAT and
3152 	 * SRS_TX_WAKEUP_CLIENT and wakeup registered clients.
3153 	 */
3154 	if (mac_srs->srs_count == 0 && (mac_srs->srs_state &
3155 	    (SRS_TX_HIWAT | SRS_TX_WAKEUP_CLIENT | SRS_ENQUEUED))) {
3156 		mac_tx_notify_cb_t *mtnfp;
3157 		mac_cb_t *mcb;
3158 		mac_client_impl_t *mcip = mac_srs->srs_mcip;
3159 		boolean_t wakeup_required = B_FALSE;
3160 
3161 		if (mac_srs->srs_state &
3162 		    (SRS_TX_HIWAT|SRS_TX_WAKEUP_CLIENT)) {
3163 			wakeup_required = B_TRUE;
3164 		}
3165 		mac_srs->srs_state &= ~(SRS_TX_HIWAT |
3166 		    SRS_TX_WAKEUP_CLIENT | SRS_ENQUEUED);
3167 		mutex_exit(&mac_srs->srs_lock);
3168 		if (wakeup_required) {
3169 			/* Wakeup callback registered clients */
3170 			MAC_CALLBACK_WALKER_INC(&mcip->mci_tx_notify_cb_info);
3171 			for (mcb = mcip->mci_tx_notify_cb_list; mcb != NULL;
3172 			    mcb = mcb->mcb_nextp) {
3173 				mtnfp = (mac_tx_notify_cb_t *)mcb->mcb_objp;
3174 				mtnfp->mtnf_fn(mtnfp->mtnf_arg,
3175 				    (mac_tx_cookie_t)mac_srs);
3176 			}
3177 			MAC_CALLBACK_WALKER_DCR(&mcip->mci_tx_notify_cb_info,
3178 			    &mcip->mci_tx_notify_cb_list);
3179 			/*
3180 			 * If the client is not the primary MAC client, then we
3181 			 * need to send the notification to the clients upper
3182 			 * MAC, i.e. mci_upper_mip.
3183 			 */
3184 			mac_tx_notify(mcip->mci_upper_mip != NULL ?
3185 			    mcip->mci_upper_mip : mcip->mci_mip);
3186 		}
3187 		mutex_enter(&mac_srs->srs_lock);
3188 	}
3189 	mac_srs->srs_state &= ~SRS_PROC;
3190 }
3191 
3192 /*
3193  * Given a packet, get the flow_entry that identifies the flow
3194  * to which that packet belongs. The flow_entry will contain
3195  * the transmit function to be used to send the packet. If the
3196  * function returns NULL, the packet should be sent using the
3197  * underlying NIC.
3198  */
3199 static flow_entry_t *
3200 mac_tx_classify(mac_impl_t *mip, mblk_t *mp)
3201 {
3202 	flow_entry_t		*flent = NULL;
3203 	mac_client_impl_t	*mcip;
3204 	int	err;
3205 
3206 	/*
3207 	 * Do classification on the packet.
3208 	 */
3209 	err = mac_flow_lookup(mip->mi_flow_tab, mp, FLOW_OUTBOUND, &flent);
3210 	if (err != 0)
3211 		return (NULL);
3212 
3213 	/*
3214 	 * This flent might just be an additional one on the MAC client,
3215 	 * i.e. for classification purposes (different fdesc), however
3216 	 * the resources, SRS et. al., are in the mci_flent, so if
3217 	 * this isn't the mci_flent, we need to get it.
3218 	 */
3219 	if ((mcip = flent->fe_mcip) != NULL && mcip->mci_flent != flent) {
3220 		FLOW_REFRELE(flent);
3221 		flent = mcip->mci_flent;
3222 		FLOW_TRY_REFHOLD(flent, err);
3223 		if (err != 0)
3224 			return (NULL);
3225 	}
3226 
3227 	return (flent);
3228 }
3229 
3230 /*
3231  * This macro is only meant to be used by mac_tx_send().
3232  */
3233 #define	CHECK_VID_AND_ADD_TAG(mp) {			\
3234 	if (vid_check) {				\
3235 		int err = 0;				\
3236 							\
3237 		MAC_VID_CHECK(src_mcip, (mp), err);	\
3238 		if (err != 0) {				\
3239 			freemsg((mp));			\
3240 			(mp) = next;			\
3241 			oerrors++;			\
3242 			continue;			\
3243 		}					\
3244 	}						\
3245 	if (add_tag) {					\
3246 		(mp) = mac_add_vlan_tag((mp), 0, vid);	\
3247 		if ((mp) == NULL) {			\
3248 			(mp) = next;			\
3249 			oerrors++;			\
3250 			continue;			\
3251 		}					\
3252 	}						\
3253 }
3254 
3255 mblk_t *
3256 mac_tx_send(mac_client_handle_t mch, mac_ring_handle_t ring, mblk_t *mp_chain,
3257     mac_tx_stats_t *stats)
3258 {
3259 	mac_client_impl_t *src_mcip = (mac_client_impl_t *)mch;
3260 	mac_impl_t *mip = src_mcip->mci_mip;
3261 	uint_t obytes = 0, opackets = 0, oerrors = 0;
3262 	mblk_t *mp = NULL, *next;
3263 	boolean_t vid_check, add_tag;
3264 	uint16_t vid = 0;
3265 
3266 	if (mip->mi_nclients > 1) {
3267 		vid_check = MAC_VID_CHECK_NEEDED(src_mcip);
3268 		add_tag = MAC_TAG_NEEDED(src_mcip);
3269 		if (add_tag)
3270 			vid = mac_client_vid(mch);
3271 	} else {
3272 		ASSERT(mip->mi_nclients == 1);
3273 		vid_check = add_tag = B_FALSE;
3274 	}
3275 
3276 	/*
3277 	 * Fastpath: if there's only one client, and there's no
3278 	 * multicast listeners, we simply send the packet down to the
3279 	 * underlying NIC.
3280 	 */
3281 	if (mip->mi_nactiveclients == 1 && mip->mi_promisc_list == NULL)  {
3282 		DTRACE_PROBE2(fastpath,
3283 		    mac_client_impl_t *, src_mcip, mblk_t *, mp_chain);
3284 
3285 		mp = mp_chain;
3286 		while (mp != NULL) {
3287 			next = mp->b_next;
3288 			mp->b_next = NULL;
3289 			opackets++;
3290 			obytes += (mp->b_cont == NULL ? MBLKL(mp) :
3291 			    msgdsize(mp));
3292 
3293 			CHECK_VID_AND_ADD_TAG(mp);
3294 			MAC_TX(mip, ring, mp,
3295 			    ((src_mcip->mci_state_flags & MCIS_SHARE_BOUND) !=
3296 			    0));
3297 
3298 			/*
3299 			 * If the driver is out of descriptors and does a
3300 			 * partial send it will return a chain of unsent
3301 			 * mblks. Adjust the accounting stats.
3302 			 */
3303 			if (mp != NULL) {
3304 				opackets--;
3305 				obytes -= msgdsize(mp);
3306 				mp->b_next = next;
3307 				break;
3308 			}
3309 			mp = next;
3310 		}
3311 		goto done;
3312 	}
3313 
3314 	/*
3315 	 * No fastpath, we either have more than one MAC client
3316 	 * defined on top of the same MAC, or one or more MAC
3317 	 * client promiscuous callbacks.
3318 	 */
3319 	DTRACE_PROBE3(slowpath, mac_client_impl_t *,
3320 	    src_mcip, int, mip->mi_nclients, mblk_t *, mp_chain);
3321 
3322 	mp = mp_chain;
3323 	while (mp != NULL) {
3324 		flow_entry_t *dst_flow_ent;
3325 		void *flow_cookie;
3326 		size_t	pkt_size;
3327 		mblk_t *mp1;
3328 
3329 		next = mp->b_next;
3330 		mp->b_next = NULL;
3331 		opackets++;
3332 		pkt_size = (mp->b_cont == NULL ? MBLKL(mp) : msgdsize(mp));
3333 		obytes += pkt_size;
3334 		CHECK_VID_AND_ADD_TAG(mp);
3335 
3336 		/*
3337 		 * Check if there are promiscuous mode callbacks defined.
3338 		 */
3339 		if (mip->mi_promisc_list != NULL)
3340 			mac_promisc_dispatch(mip, mp, src_mcip);
3341 
3342 		/*
3343 		 * Find the destination.
3344 		 */
3345 		dst_flow_ent = mac_tx_classify(mip, mp);
3346 
3347 		if (dst_flow_ent != NULL) {
3348 			size_t	hdrsize;
3349 			int	err = 0;
3350 
3351 			if (mip->mi_info.mi_nativemedia == DL_ETHER) {
3352 				struct ether_vlan_header *evhp =
3353 				    (struct ether_vlan_header *)mp->b_rptr;
3354 
3355 				if (ntohs(evhp->ether_tpid) == ETHERTYPE_VLAN)
3356 					hdrsize = sizeof (*evhp);
3357 				else
3358 					hdrsize = sizeof (struct ether_header);
3359 			} else {
3360 				mac_header_info_t	mhi;
3361 
3362 				err = mac_header_info((mac_handle_t)mip,
3363 				    mp, &mhi);
3364 				if (err == 0)
3365 					hdrsize = mhi.mhi_hdrsize;
3366 			}
3367 
3368 			/*
3369 			 * Got a matching flow. It's either another
3370 			 * MAC client, or a broadcast/multicast flow.
3371 			 * Make sure the packet size is within the
3372 			 * allowed size. If not drop the packet and
3373 			 * move to next packet.
3374 			 */
3375 			if (err != 0 ||
3376 			    (pkt_size - hdrsize) > mip->mi_sdu_max) {
3377 				oerrors++;
3378 				DTRACE_PROBE2(loopback__drop, size_t, pkt_size,
3379 				    mblk_t *, mp);
3380 				freemsg(mp);
3381 				mp = next;
3382 				FLOW_REFRELE(dst_flow_ent);
3383 				continue;
3384 			}
3385 			flow_cookie = mac_flow_get_client_cookie(dst_flow_ent);
3386 			if (flow_cookie != NULL) {
3387 				/*
3388 				 * The vnic_bcast_send function expects
3389 				 * to receive the sender MAC client
3390 				 * as value for arg2.
3391 				 */
3392 				mac_bcast_send(flow_cookie, src_mcip, mp,
3393 				    B_TRUE);
3394 			} else {
3395 				/*
3396 				 * loopback the packet to a
3397 				 * local MAC client. We force a context
3398 				 * switch if both source and destination
3399 				 * MAC clients are used by IP, i.e. bypass
3400 				 * is set.
3401 				 */
3402 				boolean_t do_switch;
3403 				mac_client_impl_t *dst_mcip =
3404 				    dst_flow_ent->fe_mcip;
3405 
3406 				do_switch = ((src_mcip->mci_state_flags &
3407 				    dst_mcip->mci_state_flags &
3408 				    MCIS_CLIENT_POLL_CAPABLE) != 0);
3409 
3410 				if ((mp1 = mac_fix_cksum(mp)) != NULL) {
3411 					(dst_flow_ent->fe_cb_fn)(
3412 					    dst_flow_ent->fe_cb_arg1,
3413 					    dst_flow_ent->fe_cb_arg2,
3414 					    mp1, do_switch);
3415 				}
3416 			}
3417 			FLOW_REFRELE(dst_flow_ent);
3418 		} else {
3419 			/*
3420 			 * Unknown destination, send via the underlying
3421 			 * NIC.
3422 			 */
3423 			MAC_TX(mip, ring, mp,
3424 			    ((src_mcip->mci_state_flags & MCIS_SHARE_BOUND) !=
3425 			    0));
3426 			if (mp != NULL) {
3427 				/*
3428 				 * Adjust for the last packet that
3429 				 * could not be transmitted
3430 				 */
3431 				opackets--;
3432 				obytes -= pkt_size;
3433 				mp->b_next = next;
3434 				break;
3435 			}
3436 		}
3437 		mp = next;
3438 	}
3439 
3440 done:
3441 	src_mcip->mci_stat_obytes += obytes;
3442 	src_mcip->mci_stat_opackets += opackets;
3443 	src_mcip->mci_stat_oerrors += oerrors;
3444 
3445 	if (stats != NULL) {
3446 		stats->ts_opackets = opackets;
3447 		stats->ts_obytes = obytes;
3448 		stats->ts_oerrors = oerrors;
3449 	}
3450 	return (mp);
3451 }
3452 
3453 /*
3454  * mac_tx_srs_ring_present
3455  *
3456  * Returns whether the specified ring is part of the specified SRS.
3457  */
3458 boolean_t
3459 mac_tx_srs_ring_present(mac_soft_ring_set_t *srs, mac_ring_t *tx_ring)
3460 {
3461 	int i;
3462 	mac_soft_ring_t *soft_ring;
3463 
3464 	if (srs->srs_tx.st_arg2 == tx_ring)
3465 		return (B_TRUE);
3466 
3467 	for (i = 0; i < srs->srs_oth_ring_count; i++) {
3468 		soft_ring =  srs->srs_oth_soft_rings[i];
3469 		if (soft_ring->s_ring_tx_arg2 == tx_ring)
3470 			return (B_TRUE);
3471 	}
3472 
3473 	return (B_FALSE);
3474 }
3475 
3476 /*
3477  * mac_tx_srs_wakeup
3478  *
3479  * Called when Tx desc become available. Wakeup the appropriate worker
3480  * thread after resetting the SRS_TX_BLOCKED/S_RING_BLOCK bit in the
3481  * state field.
3482  */
3483 void
3484 mac_tx_srs_wakeup(mac_soft_ring_set_t *mac_srs, mac_ring_handle_t ring)
3485 {
3486 	int i;
3487 	mac_soft_ring_t *sringp;
3488 	mac_srs_tx_t *srs_tx = &mac_srs->srs_tx;
3489 
3490 	mutex_enter(&mac_srs->srs_lock);
3491 	if (TX_SINGLE_RING_MODE(mac_srs)) {
3492 		if (srs_tx->st_arg2 == ring &&
3493 		    mac_srs->srs_state & SRS_TX_BLOCKED) {
3494 			mac_srs->srs_state &= ~SRS_TX_BLOCKED;
3495 			srs_tx->st_unblocked_cnt++;
3496 			cv_signal(&mac_srs->srs_async);
3497 		}
3498 		/*
3499 		 * A wakeup can come before tx_srs_drain() could
3500 		 * grab srs lock and set SRS_TX_BLOCKED. So
3501 		 * always set woken_up flag when we come here.
3502 		 */
3503 		srs_tx->st_woken_up = B_TRUE;
3504 		mutex_exit(&mac_srs->srs_lock);
3505 		return;
3506 	}
3507 
3508 	/* If you are here, it is for FANOUT or BW_FANOUT case */
3509 	ASSERT(TX_MULTI_RING_MODE(mac_srs));
3510 	for (i = 0; i < mac_srs->srs_oth_ring_count; i++) {
3511 		sringp = mac_srs->srs_oth_soft_rings[i];
3512 		mutex_enter(&sringp->s_ring_lock);
3513 		if (sringp->s_ring_tx_arg2 == ring) {
3514 			if (sringp->s_ring_state & S_RING_BLOCK) {
3515 				sringp->s_ring_state &= ~S_RING_BLOCK;
3516 				sringp->s_ring_unblocked_cnt++;
3517 				cv_signal(&sringp->s_ring_async);
3518 			}
3519 			sringp->s_ring_tx_woken_up = B_TRUE;
3520 		}
3521 		mutex_exit(&sringp->s_ring_lock);
3522 	}
3523 	mutex_exit(&mac_srs->srs_lock);
3524 }
3525 
3526 /*
3527  * Once the driver is done draining, send a MAC_NOTE_TX notification to unleash
3528  * the blocked clients again.
3529  */
3530 void
3531 mac_tx_notify(mac_impl_t *mip)
3532 {
3533 	i_mac_notify(mip, MAC_NOTE_TX);
3534 }
3535 
3536 /*
3537  * RX SOFTRING RELATED FUNCTIONS
3538  *
3539  * These functions really belong in mac_soft_ring.c and here for
3540  * a short period.
3541  */
3542 
3543 #define	SOFT_RING_ENQUEUE_CHAIN(ringp, mp, tail, cnt, sz) {	       	\
3544 	/*								\
3545 	 * Enqueue our mblk chain.					\
3546 	 */								\
3547 	ASSERT(MUTEX_HELD(&(ringp)->s_ring_lock));			\
3548 									\
3549 	if ((ringp)->s_ring_last != NULL)				\
3550 		(ringp)->s_ring_last->b_next = (mp);			\
3551 	else								\
3552 		(ringp)->s_ring_first = (mp);				\
3553 	(ringp)->s_ring_last = (tail);					\
3554 	(ringp)->s_ring_count += (cnt);					\
3555 	ASSERT((ringp)->s_ring_count > 0);				\
3556 	if ((ringp)->s_ring_type & ST_RING_BW_CTL) {			\
3557 		(ringp)->s_ring_size += sz;				\
3558 	}								\
3559 }
3560 
3561 /*
3562  * Default entry point to deliver a packet chain to a MAC client.
3563  * If the MAC client has flows, do the classification with these
3564  * flows as well.
3565  */
3566 /* ARGSUSED */
3567 void
3568 mac_rx_deliver(void *arg1, mac_resource_handle_t mrh, mblk_t *mp_chain,
3569     mac_header_info_t *arg3)
3570 {
3571 	mac_client_impl_t *mcip = arg1;
3572 
3573 	if (mcip->mci_nvids == 1 &&
3574 	    !(mcip->mci_state_flags & MCIS_STRIP_DISABLE)) {
3575 		/*
3576 		 * If the client has exactly one VID associated with it
3577 		 * and striping of VLAN header is not disabled,
3578 		 * remove the VLAN tag from the packet before
3579 		 * passing it on to the client's receive callback.
3580 		 * Note that this needs to be done after we dispatch
3581 		 * the packet to the promiscuous listeners of the
3582 		 * client, since they expect to see the whole
3583 		 * frame including the VLAN headers.
3584 		 */
3585 		mp_chain = mac_strip_vlan_tag_chain(mp_chain);
3586 	}
3587 
3588 	mcip->mci_rx_fn(mcip->mci_rx_arg, mrh, mp_chain, B_FALSE);
3589 }
3590 
3591 /*
3592  * mac_rx_soft_ring_process
3593  *
3594  * process a chain for a given soft ring. The number of packets queued
3595  * in the SRS and its associated soft rings (including this one) is
3596  * very small (tracked by srs_poll_pkt_cnt), then allow the entering
3597  * thread (interrupt or poll thread) to do inline processing. This
3598  * helps keep the latency down under low load.
3599  *
3600  * The proc and arg for each mblk is already stored in the mblk in
3601  * appropriate places.
3602  */
3603 /* ARGSUSED */
3604 void
3605 mac_rx_soft_ring_process(mac_client_impl_t *mcip, mac_soft_ring_t *ringp,
3606     mblk_t *mp_chain, mblk_t *tail, int cnt, size_t sz)
3607 {
3608 	mac_direct_rx_t		proc;
3609 	void			*arg1;
3610 	mac_resource_handle_t	arg2;
3611 	mac_soft_ring_set_t	*mac_srs = ringp->s_ring_set;
3612 
3613 	ASSERT(ringp != NULL);
3614 	ASSERT(mp_chain != NULL);
3615 	ASSERT(tail != NULL);
3616 	ASSERT(MUTEX_NOT_HELD(&ringp->s_ring_lock));
3617 
3618 	mutex_enter(&ringp->s_ring_lock);
3619 	ringp->s_ring_total_inpkt += cnt;
3620 	if ((mac_srs->srs_rx.sr_poll_pkt_cnt <= 1) &&
3621 	    !(ringp->s_ring_type & ST_RING_WORKER_ONLY)) {
3622 		/* If on processor or blanking on, then enqueue and return */
3623 		if (ringp->s_ring_state & S_RING_BLANK ||
3624 		    ringp->s_ring_state & S_RING_PROC) {
3625 			SOFT_RING_ENQUEUE_CHAIN(ringp, mp_chain, tail, cnt, sz);
3626 			mutex_exit(&ringp->s_ring_lock);
3627 			return;
3628 		}
3629 		proc = ringp->s_ring_rx_func;
3630 		arg1 = ringp->s_ring_rx_arg1;
3631 		arg2 = ringp->s_ring_rx_arg2;
3632 		/*
3633 		 * See if anything is already queued. If we are the
3634 		 * first packet, do inline processing else queue the
3635 		 * packet and do the drain.
3636 		 */
3637 		if (ringp->s_ring_first == NULL) {
3638 			/*
3639 			 * Fast-path, ok to process and nothing queued.
3640 			 */
3641 			ringp->s_ring_run = curthread;
3642 			ringp->s_ring_state |= (S_RING_PROC);
3643 
3644 			mutex_exit(&ringp->s_ring_lock);
3645 
3646 			/*
3647 			 * We are the chain of 1 packet so
3648 			 * go through this fast path.
3649 			 */
3650 			ASSERT(mp_chain->b_next == NULL);
3651 
3652 			(*proc)(arg1, arg2, mp_chain, NULL);
3653 
3654 			ASSERT(MUTEX_NOT_HELD(&ringp->s_ring_lock));
3655 			/*
3656 			 * If we have a soft ring set which is doing
3657 			 * bandwidth control, we need to decrement
3658 			 * srs_size and count so it the SRS can have a
3659 			 * accurate idea of what is the real data
3660 			 * queued between SRS and its soft rings. We
3661 			 * decrement the counters only when the packet
3662 			 * gets processed by both SRS and the soft ring.
3663 			 */
3664 			mutex_enter(&mac_srs->srs_lock);
3665 			MAC_UPDATE_SRS_COUNT_LOCKED(mac_srs, cnt);
3666 			MAC_UPDATE_SRS_SIZE_LOCKED(mac_srs, sz);
3667 			mutex_exit(&mac_srs->srs_lock);
3668 
3669 			mutex_enter(&ringp->s_ring_lock);
3670 			ringp->s_ring_run = NULL;
3671 			ringp->s_ring_state &= ~S_RING_PROC;
3672 			if (ringp->s_ring_state & S_RING_CLIENT_WAIT)
3673 				cv_signal(&ringp->s_ring_client_cv);
3674 
3675 			if ((ringp->s_ring_first == NULL) ||
3676 			    (ringp->s_ring_state & S_RING_BLANK)) {
3677 				/*
3678 				 * We processed inline our packet and
3679 				 * nothing new has arrived or our
3680 				 * receiver doesn't want to receive
3681 				 * any packets. We are done.
3682 				 */
3683 				mutex_exit(&ringp->s_ring_lock);
3684 				return;
3685 			}
3686 		} else {
3687 			SOFT_RING_ENQUEUE_CHAIN(ringp,
3688 			    mp_chain, tail, cnt, sz);
3689 		}
3690 
3691 		/*
3692 		 * We are here because either we couldn't do inline
3693 		 * processing (because something was already
3694 		 * queued), or we had a chain of more than one
3695 		 * packet, or something else arrived after we were
3696 		 * done with inline processing.
3697 		 */
3698 		ASSERT(MUTEX_HELD(&ringp->s_ring_lock));
3699 		ASSERT(ringp->s_ring_first != NULL);
3700 
3701 		ringp->s_ring_drain_func(ringp);
3702 		mutex_exit(&ringp->s_ring_lock);
3703 		return;
3704 	} else {
3705 		/* ST_RING_WORKER_ONLY case */
3706 		SOFT_RING_ENQUEUE_CHAIN(ringp, mp_chain, tail, cnt, sz);
3707 		mac_soft_ring_worker_wakeup(ringp);
3708 		mutex_exit(&ringp->s_ring_lock);
3709 	}
3710 }
3711 
3712 /*
3713  * TX SOFTRING RELATED FUNCTIONS
3714  *
3715  * These functions really belong in mac_soft_ring.c and here for
3716  * a short period.
3717  */
3718 
3719 #define	TX_SOFT_RING_ENQUEUE_CHAIN(ringp, mp, tail, cnt, sz) {	       	\
3720 	ASSERT(MUTEX_HELD(&ringp->s_ring_lock));			\
3721 	ringp->s_ring_state |= S_RING_ENQUEUED;				\
3722 	SOFT_RING_ENQUEUE_CHAIN(ringp, mp_chain, tail, cnt, sz);	\
3723 }
3724 
3725 /*
3726  * mac_tx_sring_queued
3727  *
3728  * When we are out of transmit descriptors and we already have a
3729  * queue that exceeds hiwat (or the client called us with
3730  * MAC_TX_NO_ENQUEUE or MAC_DROP_ON_NO_DESC flag), return the
3731  * soft ring pointer as the opaque cookie for the client enable
3732  * flow control.
3733  */
3734 static mac_tx_cookie_t
3735 mac_tx_sring_enqueue(mac_soft_ring_t *ringp, mblk_t *mp_chain, uint16_t flag,
3736     mblk_t **ret_mp)
3737 {
3738 	int cnt;
3739 	size_t sz;
3740 	mblk_t *tail;
3741 	mac_soft_ring_set_t *mac_srs = ringp->s_ring_set;
3742 	mac_tx_cookie_t cookie = NULL;
3743 	boolean_t wakeup_worker = B_TRUE;
3744 
3745 	ASSERT(MUTEX_HELD(&ringp->s_ring_lock));
3746 	MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz);
3747 	if (flag & MAC_DROP_ON_NO_DESC) {
3748 		mac_pkt_drop(NULL, NULL, mp_chain, B_FALSE);
3749 		/* increment freed stats */
3750 		ringp->s_ring_drops += cnt;
3751 		cookie = (mac_tx_cookie_t)ringp;
3752 	} else {
3753 		if (ringp->s_ring_first != NULL)
3754 			wakeup_worker = B_FALSE;
3755 
3756 		if (flag & MAC_TX_NO_ENQUEUE) {
3757 			/*
3758 			 * If QUEUED is not set, queue the packet
3759 			 * and let mac_tx_soft_ring_drain() set
3760 			 * the TX_BLOCKED bit for the reasons
3761 			 * explained above. Otherwise, return the
3762 			 * mblks.
3763 			 */
3764 			if (wakeup_worker) {
3765 				TX_SOFT_RING_ENQUEUE_CHAIN(ringp,
3766 				    mp_chain, tail, cnt, sz);
3767 			} else {
3768 				ringp->s_ring_state |= S_RING_WAKEUP_CLIENT;
3769 				cookie = (mac_tx_cookie_t)ringp;
3770 				*ret_mp = mp_chain;
3771 			}
3772 		} else {
3773 			boolean_t enqueue = B_TRUE;
3774 
3775 			if (ringp->s_ring_count > ringp->s_ring_tx_hiwat) {
3776 				/*
3777 				 * flow-controlled. Store ringp in cookie
3778 				 * so that it can be returned as
3779 				 * mac_tx_cookie_t to client
3780 				 */
3781 				ringp->s_ring_state |= S_RING_TX_HIWAT;
3782 				cookie = (mac_tx_cookie_t)ringp;
3783 				ringp->s_ring_hiwat_cnt++;
3784 				if (ringp->s_ring_count >
3785 				    ringp->s_ring_tx_max_q_cnt) {
3786 					/* increment freed stats */
3787 					ringp->s_ring_drops += cnt;
3788 					/*
3789 					 * b_prev may be set to the fanout hint
3790 					 * hence can't use freemsg directly
3791 					 */
3792 					mac_pkt_drop(NULL, NULL,
3793 					    mp_chain, B_FALSE);
3794 					DTRACE_PROBE1(tx_queued_hiwat,
3795 					    mac_soft_ring_t *, ringp);
3796 					enqueue = B_FALSE;
3797 				}
3798 			}
3799 			if (enqueue) {
3800 				TX_SOFT_RING_ENQUEUE_CHAIN(ringp, mp_chain,
3801 				    tail, cnt, sz);
3802 			}
3803 		}
3804 		if (wakeup_worker)
3805 			cv_signal(&ringp->s_ring_async);
3806 	}
3807 	return (cookie);
3808 }
3809 
3810 
3811 /*
3812  * mac_tx_soft_ring_process
3813  *
3814  * This routine is called when fanning out outgoing traffic among
3815  * multipe Tx rings.
3816  * Note that a soft ring is associated with a h/w Tx ring.
3817  */
3818 mac_tx_cookie_t
3819 mac_tx_soft_ring_process(mac_soft_ring_t *ringp, mblk_t *mp_chain,
3820     uint16_t flag, mblk_t **ret_mp)
3821 {
3822 	mac_soft_ring_set_t *mac_srs = ringp->s_ring_set;
3823 	int	cnt;
3824 	size_t	sz;
3825 	mblk_t	*tail;
3826 	mac_tx_cookie_t cookie = NULL;
3827 
3828 	ASSERT(ringp != NULL);
3829 	ASSERT(mp_chain != NULL);
3830 	ASSERT(MUTEX_NOT_HELD(&ringp->s_ring_lock));
3831 	/*
3832 	 * Only two modes can come here; either it can be
3833 	 * SRS_TX_BW_FANOUT or SRS_TX_FANOUT
3834 	 */
3835 	ASSERT(mac_srs->srs_tx.st_mode == SRS_TX_FANOUT ||
3836 	    mac_srs->srs_tx.st_mode == SRS_TX_BW_FANOUT);
3837 
3838 	if (ringp->s_ring_type & ST_RING_WORKER_ONLY) {
3839 		/* Serialization mode */
3840 
3841 		mutex_enter(&ringp->s_ring_lock);
3842 		if (ringp->s_ring_count > ringp->s_ring_tx_hiwat) {
3843 			cookie = mac_tx_sring_enqueue(ringp, mp_chain,
3844 			    flag, ret_mp);
3845 			mutex_exit(&ringp->s_ring_lock);
3846 			return (cookie);
3847 		}
3848 		MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz);
3849 		TX_SOFT_RING_ENQUEUE_CHAIN(ringp, mp_chain, tail, cnt, sz);
3850 		if (ringp->s_ring_state & (S_RING_BLOCK | S_RING_PROC)) {
3851 			/*
3852 			 * If ring is blocked due to lack of Tx
3853 			 * descs, just return. Worker thread
3854 			 * will get scheduled when Tx desc's
3855 			 * become available.
3856 			 */
3857 			mutex_exit(&ringp->s_ring_lock);
3858 			return (cookie);
3859 		}
3860 		mac_soft_ring_worker_wakeup(ringp);
3861 		mutex_exit(&ringp->s_ring_lock);
3862 		return (cookie);
3863 	} else {
3864 		/* Default fanout mode */
3865 		/*
3866 		 * S_RING_BLOCKED is set when underlying NIC runs
3867 		 * out of Tx descs and messages start getting
3868 		 * queued. It won't get reset until
3869 		 * tx_srs_drain() completely drains out the
3870 		 * messages.
3871 		 */
3872 		boolean_t		is_subflow;
3873 		mac_tx_stats_t		stats;
3874 
3875 		if (ringp->s_ring_state & S_RING_ENQUEUED) {
3876 			/* Tx descs/resources not available */
3877 			mutex_enter(&ringp->s_ring_lock);
3878 			if (ringp->s_ring_state & S_RING_ENQUEUED) {
3879 				cookie = mac_tx_sring_enqueue(ringp, mp_chain,
3880 				    flag, ret_mp);
3881 				mutex_exit(&ringp->s_ring_lock);
3882 				return (cookie);
3883 			}
3884 			/*
3885 			 * While we were computing mblk count, the
3886 			 * flow control condition got relieved.
3887 			 * Continue with the transmission.
3888 			 */
3889 			mutex_exit(&ringp->s_ring_lock);
3890 		}
3891 		is_subflow = ((mac_srs->srs_type & SRST_FLOW) != 0);
3892 
3893 		mp_chain = mac_tx_send(ringp->s_ring_tx_arg1,
3894 		    ringp->s_ring_tx_arg2, mp_chain,
3895 		    (is_subflow ? &stats : NULL));
3896 
3897 		/*
3898 		 * Multiple threads could be here sending packets.
3899 		 * Under such conditions, it is not possible to
3900 		 * automically set S_RING_BLOCKED bit to indicate
3901 		 * out of tx desc condition. To atomically set
3902 		 * this, we queue the returned packet and do
3903 		 * the setting of S_RING_BLOCKED in
3904 		 * mac_tx_soft_ring_drain().
3905 		 */
3906 		if (mp_chain != NULL) {
3907 			mutex_enter(&ringp->s_ring_lock);
3908 			cookie =
3909 			    mac_tx_sring_enqueue(ringp, mp_chain, flag, ret_mp);
3910 			mutex_exit(&ringp->s_ring_lock);
3911 			return (cookie);
3912 		}
3913 		if (is_subflow) {
3914 			FLOW_TX_STATS_UPDATE(mac_srs->srs_flent, &stats);
3915 		}
3916 		return (NULL);
3917 	}
3918 }
3919