xref: /titanic_41/usr/src/uts/common/io/mac/mac_sched.c (revision 0a0f0095b1aef5b3ab0b1fabd1c8e39599b20910)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  * Copyright 2015 Joyent, Inc.
25  * Copyright 2013 Nexenta Systems, Inc. All rights reserved.
26  */
27 
28 /*
29  * MAC data path
30  *
31  * The MAC data path is concerned with the flow of traffic from mac clients --
32  * DLS, IP, etc. -- to various GLDv3 device drivers -- e1000g, vnic, aggr,
33  * ixgbe, etc. -- and from the GLDv3 device drivers back to clients.
34  *
35  * -----------
36  * Terminology
37  * -----------
38  *
39  * MAC uses a lot of different, but related terms that are associated with the
40  * design and structure of the data path. Before we cover other aspects, first
41  * let's review the terminology that MAC uses.
42  *
43  * MAC
44  *
45  * 	This driver. It interfaces with device drivers and provides abstractions
46  * 	that the rest of the system consumes. All data links -- things managed
47  * 	with dladm(1M), are accessed through MAC.
48  *
49  * GLDv3 DEVICE DRIVER
50  *
51  * 	A GLDv3 device driver refers to a driver, both for pseudo-devices and
52  * 	real devices, which implement the GLDv3 driver API. Common examples of
53  * 	these are igb and ixgbe, which are drivers for various Intel networking
54  * 	cards. These devices may or may not have various features, such as
55  * 	hardware rings and checksum offloading. For MAC, a GLDv3 device is the
56  * 	final point for the transmission of a packet and the starting point for
57  * 	the receipt of a packet.
58  *
59  * FLOWS
60  *
61  * 	At a high level, a flow refers to a series of packets that are related.
62  * 	Often times the term is used in the context of TCP to indicate a unique
63  * 	TCP connection and the traffic over it. However, a flow can exist at
64  * 	other levels of the system as well. MAC has a notion of a default flow
65  * 	which is used for all unicast traffic addressed to the address of a MAC
66  * 	device. For example, when a VNIC is created, a default flow is created
67  * 	for the VNIC's MAC address. In addition, flows are created for broadcast
68  * 	groups and a user may create a flow with flowadm(1M).
69  *
70  * CLASSIFICATION
71  *
72  * 	Classification refers to the notion of identifying an incoming frame
73  * 	based on its destination address and optionally its source addresses and
74  * 	doing different processing based on that information. Classification can
75  * 	be done in both hardware and software. In general, we usually only
76  * 	classify based on the layer two destination, eg. for Ethernet, the
77  * 	destination MAC address.
78  *
79  * 	The system also will do classification based on layer three and layer
80  * 	four properties. This is used to support things like flowadm(1M), which
81  * 	allows setting QoS and other properties on a per-flow basis.
82  *
83  * RING
84  *
85  * 	Conceptually, a ring represents a series of framed messages, often in a
86  * 	contiguous chunk of memory that acts as a circular buffer. Rings come in
87  * 	a couple of forms. Generally they are either a hardware construct (hw
88  * 	ring) or they are a software construct (sw ring) maintained by MAC.
89  *
90  * HW RING
91  *
92  * 	A hardware ring is a set of resources provided by a GLDv3 device driver
93  * 	(even if it is a pseudo-device). A hardware ring comes in two different
94  * 	forms: receive (rx) rings and transmit (tx) rings. An rx hw ring is
95  * 	something that has a unique DMA (direct memory access) region and
96  * 	generally supports some form of classification (though it isn't always
97  * 	used), as well as a means of generating an interrupt specific to that
98  * 	ring. For example, the device may generate a specific MSI-X for a PCI
99  * 	express device. A tx ring is similar, except that it is dedicated to
100  * 	transmission. It may also be a vector for enabling features such as VLAN
101  * 	tagging and large transmit offloading. It usually has its own dedicated
102  * 	interrupts for transmit being completed.
103  *
104  * SW RING
105  *
106  * 	A software ring is a construction of MAC. It represents the same thing
107  * 	that a hardware ring generally does, a collection of frames. However,
108  * 	instead of being in a contiguous ring of memory, they're instead linked
109  * 	by using the mblk_t's b_next pointer. Each frame may itself be multiple
110  * 	mblk_t's linked together by the b_cont pointer. A software ring always
111  * 	represents a collection of classified packets; however, it varies as to
112  * 	whether it uses only layer two information, or a combination of that and
113  * 	additional layer three and layer four data.
114  *
115  * FANOUT
116  *
117  * 	Fanout is the idea of spreading out the load of processing frames based
118  * 	on the source and destination information contained in the layer two,
119  * 	three, and four headers, such that the data can then be processed in
120  * 	parallel using multiple hardware threads.
121  *
122  * 	A fanout algorithm hashes the headers and uses that to place different
123  * 	flows into a bucket. The most important thing is that packets that are
124  * 	in the same flow end up in the same bucket. If they do not, performance
125  * 	can be adversely affected. Consider the case of TCP.  TCP severely
126  * 	penalizes a connection if the data arrives out of order. If a given flow
127  * 	is processed on different CPUs, then the data will appear out of order,
128  * 	hence the invariant that fanout always hash a given flow to the same
129  * 	bucket and thus get processed on the same CPU.
130  *
131  * RECEIVE SIDE SCALING (RSS)
132  *
133  *
134  * 	Receive side scaling is a term that isn't common in illumos, but is used
135  * 	by vendors and was popularized by Microsoft. It refers to the idea of
136  * 	spreading the incoming receive load out across multiple interrupts which
137  * 	can be directed to different CPUs. This allows a device to leverage
138  * 	hardware rings even when it doesn't support hardware classification. The
139  * 	hardware uses an algorithm to perform fanout that ensures the flow
140  * 	invariant is maintained.
141  *
142  * SOFT RING SET
143  *
144  * 	A soft ring set, commonly abbreviated SRS, is a collection of rings and
145  * 	is used for both transmitting and receiving. It is maintained in the
146  * 	structure mac_soft_ring_set_t. A soft ring set is usually associated
147  * 	with flows, and coordinates both the use of hardware and software rings.
148  * 	Because the use of hardware rings can change as devices such as VNICs
149  * 	come and go, we always ensure that the set has software classification
150  * 	rules that correspond to the hardware classification rules from rings.
151  *
152  * 	Soft ring sets are also used for the enforcement of various QoS
153  * 	properties. For example, if a bandwidth limit has been placed on a
154  * 	specific flow or device, then that will be enforced by the soft ring
155  * 	set.
156  *
157  * SERVICE ATTACHMENT POINT (SAP)
158  *
159  * 	The service attachment point is a DLPI (Data Link Provider Interface)
160  * 	concept; however, it comes up quite often in MAC. Most MAC devices speak
161  * 	a protocol that has some notion of different channels or message type
162  * 	identifiers. For example, Ethernet defines an EtherType which is a part
163  * 	of the Ethernet header and defines the particular protocol of the data
164  * 	payload. If the EtherType is set to 0x0800, then it defines that the
165  * 	contents of that Ethernet frame is IPv4 traffic. For Ethernet, the
166  * 	EtherType is the SAP.
167  *
168  * 	In DLPI, a given consumer attaches to a specific SAP. In illumos, the ip
169  * 	and arp drivers attach to the EtherTypes for IPv4, IPv6, and ARP. Using
170  * 	libdlpi(3LIB) user software can attach to arbitrary SAPs. With the
171  * 	exception of 802.1Q VLAN tagged traffic, MAC itself does not directly
172  * 	consume the SAP; however, it uses that information as part of hashing
173  * 	and it may be used as part of the construction of flows.
174  *
175  * PRIMARY MAC CLIENT
176  *
177  * 	The primary mac client refers to a mac client whose unicast address
178  * 	matches the address of the device itself. For example, if the system has
179  * 	instance of the e1000g driver such as e1000g0, e1000g1, etc., the
180  * 	primary mac client is the one named after the device itself. VNICs that
181  * 	are created on top of such devices are not the primary client.
182  *
183  * TRANSMIT DESCRIPTORS
184  *
185  * 	Transmit descriptors are a resource that most GLDv3 device drivers have.
186  * 	Generally, a GLDv3 device driver takes a frame that's meant to be output
187  * 	and puts a copy of it into a region of memory. Each region of memory
188  * 	usually has an associated descriptor that the device uses to manage
189  * 	properties of the frames. Devices have a limited number of such
190  * 	descriptors. They get reclaimed once the device finishes putting the
191  * 	frame on the wire.
192  *
193  * 	If the driver runs out of transmit descriptors, for example, the OS is
194  * 	generating more frames than it can put on the wire, then it will return
195  * 	them back to the MAC layer.
196  *
197  * ---------------------------------
198  * Rings, Classification, and Fanout
199  * ---------------------------------
200  *
201  * The heart of MAC is made up of rings, and not those that Elven-kings wear.
202  * When receiving a packet, MAC breaks the work into two different, though
203  * interrelated phases. The first phase is generally classification and then the
204  * second phase is generally fanout. When a frame comes in from a GLDv3 Device,
205  * MAC needs to determine where that frame should be delivered. If it's a
206  * unicast frame (say a normal TCP/IP packet), then it will be delivered to a
207  * single MAC client; however, if it's a broadcast or multicast frame, then MAC
208  * may need to deliver it to multiple MAC clients.
209  *
210  * On transmit, classification isn't quite as important, but may still be used.
211  * Unlike with the receive path, the classification is not used to determine
212  * devices that should transmit something, but rather is used for special
213  * properties of a flow, eg. bandwidth limits for a given IP address, device, or
214  * connection.
215  *
216  * MAC employs a software classifier and leverages hardware classification as
217  * well. The software classifier can leverage the full layer two information,
218  * source, destination, VLAN, and SAP. If the SAP indicates that IP traffic is
219  * being sent, it can classify based on the IP header, and finally, it also
220  * knows how to classify based on the local and remote ports of TCP, UDP, and
221  * SCTP.
222  *
223  * Hardware classifiers vary in capability. Generally all hardware classifiers
224  * provide the capability to classify based on the destination MAC address. Some
225  * hardware has additional filters built in for performing more in-depth
226  * classification; however, it often has much more limited resources for these
227  * activities as compared to the layer two destination address classification.
228  *
229  * The modus operandi in MAC is to always ensure that we have software-based
230  * capabilities and rules in place and then to supplement that with hardware
231  * resources when available. In general, simple layer two classification is
232  * sufficient and nothing else is used, unless a specific flow is created with
233  * tools such as flowadm(1M) or bandwidth limits are set on a device with
234  * dladm(1M).
235  *
236  * RINGS AND GROUPS
237  *
238  * To get into how rings and classification play together, it's first important
239  * to understand how hardware devices commonly associate rings and allow them to
240  * be programmed. Recall that a hardware ring should be thought of as a DMA
241  * buffer and an interrupt resource. Rings are then collected into groups. A
242  * group itself has a series of classification rules. One or more MAC addresses
243  * are assigned to a group.
244  *
245  * Hardware devices vary in terms of what capabilities they provide. Sometimes
246  * they allow for a dynamic assignment of rings to a group and sometimes they
247  * have a static assignment of rings to a group. For example, the ixgbe driver
248  * has a static assignment of rings to groups such that every group has exactly
249  * one ring and the number of groups is equal to the number of rings.
250  *
251  * Classification and receive side scaling both come into play with how a device
252  * advertises itself to MAC and how MAC uses it. If a device supports layer two
253  * classification of frames, then MAC will assign MAC addresses to a group as a
254  * form of primary classification. If a single MAC address is assigned to a
255  * group, a common case, then MAC will consider packets that come in from rings
256  * on that group to be fully classified and will not need to do any software
257  * classification unless a specific flow has been created.
258  *
259  * If a device supports receive side scaling, then it may advertise or support
260  * groups with multiple rings. In those cases, then receive side scaling will
261  * come into play and MAC will use that as a means of fanning out received
262  * frames across multiple CPUs. This can also be combined with groups that
263  * support layer two classification.
264  *
265  * If a device supports dynamic assignments of rings to groups, then MAC will
266  * change around the way that rings are assigned to various groups as devices
267  * come and go from the system. For example, when a VNIC is created, a new flow
268  * will be created for the VNIC's MAC address. If a hardware ring is available,
269  * MAC may opt to reassign it from one group to another.
270  *
271  * ASSIGNMENT OF HARDWARE RINGS
272  *
273  * This is a bit of a complicated subject that varies depending on the device,
274  * the use of aggregations, the special nature of the primary mac client. This
275  * section deserves being fleshed out.
276  *
277  * FANOUT
278  *
279  * illumos uses fanout to help spread out the incoming processing load of chains
280  * of frames away from a single CPU. If a device supports receive side scaling,
281  * then that provides an initial form of fanout; however, what we're concerned
282  * with all happens after the context of a given set of frames being classified
283  * to a soft ring set.
284  *
285  * After frames reach a soft ring set and account for any potential bandwidth
286  * related accounting, they may be fanned out based on one of the following
287  * three modes:
288  *
289  *     o No Fanout
290  *     o Protocol level fanout
291  *     o Full software ring protocol fanout
292  *
293  * MAC makes the determination as to which of these modes a given soft ring set
294  * obtains based on parameters such as whether or not it's the primary mac
295  * client, whether it's on a 10 GbE or faster device, user controlled dladm(1M)
296  * properties, and the nature of the hardware and the resources that it has.
297  *
298  * When there is no fanout, MAC does not create any soft rings for a device and
299  * the device has frames delivered directly to the MAC client.
300  *
301  * Otherwise, all fanout is performed by software. MAC divides incoming frames
302  * into one of three buckets -- IPv4 TCP traffic, IPv4 UDP traffic, and
303  * everything else. Note, VLAN tagged traffic is considered other, regardless of
304  * the interior EtherType. Regardless of the type of fanout, these three
305  * categories or buckets are always used.
306  *
307  * The difference between protocol level fanout and full software ring protocol
308  * fanout is the number of software rings that end up getting created. The
309  * system always uses the same number of software rings per protocol bucket. So
310  * in the first case when we're just doing protocol level fanout, we just create
311  * one software ring each for IPv4 TCP traffic, IPv4 UDP traffic, and everything
312  * else.
313  *
314  * In the case where we do full software ring protocol fanout, we generally use
315  * mac_compute_soft_ring_count() to determine the number of rings. There are
316  * other combinations of properties and devices that may send us down other
317  * paths, but this is a common starting point. If it's a non-bandwidth enforced
318  * device and we're on at least a 10 GbE link, then we'll use eight soft rings
319  * per protocol bucket as a starting point. See mac_compute_soft_ring_count()
320  * for more information on the total number.
321  *
322  * For each of these rings, we create a mac_soft_ring_t and an associated worker
323  * thread. Particularly when doing full software ring protocol fanout, we bind
324  * each of the worker threads to individual CPUs.
325  *
326  * The other advantage of these software rings is that it allows upper layers to
327  * optionally poll on them. For example, TCP can leverage an squeue to poll on
328  * the software ring, see squeue.c for more information.
329  *
330  * DLS BYPASS
331  *
332  * DLS is the data link services module. It interfaces with DLPI, which is the
333  * primary way that other parts of the system such as IP interface with the MAC
334  * layer. While DLS is traditionally a STREAMS-based interface, it allows for
335  * certain modules such as IP to negotiate various more modern interfaces to be
336  * used, which are useful for higher performance and allow it to use direct
337  * function calls to DLS instead of using STREAMS.
338  *
339  * When we have IPv4 TCP or UDP software rings, then traffic on those rings is
340  * eligible for what we call the dls bypass. In those cases, rather than going
341  * out mac_rx_deliver() to DLS, DLS instead registers them to go directly via
342  * the direct callback registered with DLS, generally ip_input().
343  *
344  * HARDWARE RING POLLING
345  *
346  * GLDv3 devices with hardware rings generally deliver chains of messages
347  * (mblk_t chain) during the context of a single interrupt. However, interrupts
348  * are not the only way that these devices may be used. As part of implementing
349  * ring support, a GLDv3 device driver must have a way to disable the generation
350  * of that interrupt and allow for the operating system to poll on that ring.
351  *
352  * To implement this, every soft ring set has a worker thread and a polling
353  * thread. If a sufficient packet rate comes into the system, MAC will 'blank'
354  * (disable) interrupts on that specific ring and the polling thread will start
355  * consuming packets from the hardware device and deliver them to the soft ring
356  * set, where the worker thread will take over.
357  *
358  * Once the rate of packet intake drops down below a certain threshold, then
359  * polling on the hardware ring will be quiesced and interrupts will be
360  * re-enabled for the given ring. This effectively allows the system to shift
361  * how it handles a ring based on its load. At high packet rates, polling on the
362  * device as opposed to relying on interrupts can actually reduce overall system
363  * load due to the minimization of interrupt activity.
364  *
365  * Note the importance of each ring having its own interrupt source. The whole
366  * idea here is that we do not disable interrupts on the device as a whole, but
367  * rather each ring can be independently toggled.
368  *
369  * USE OF WORKER THREADS
370  *
371  * Both the soft ring set and individual soft rings have a worker thread
372  * associated with them that may be bound to a specific CPU in the system. Any
373  * such assignment will get reassessed as part of dynamic reconfiguration events
374  * in the system such as the onlining and offlining of CPUs and the creation of
375  * CPU partitions.
376  *
377  * In many cases, while in an interrupt, we try to deliver a frame all the way
378  * through the stack in the context of the interrupt itself. However, if the
379  * amount of queued frames has exceeded a threshold, then we instead defer to
380  * the worker thread to do this work and signal it. This is particularly useful
381  * when you have the soft ring set delivering frames into multiple software
382  * rings. If it was only delivering frames into a single software ring then
383  * there'd be no need to have another thread take over. However, if it's
384  * delivering chains of frames to multiple rings, then it's worthwhile to have
385  * the worker for the software ring take over so that the different software
386  * rings can be processed in parallel.
387  *
388  * In a similar fashion to the hardware polling thread, if we don't have a
389  * backlog or there's nothing to do, then the worker thread will go back to
390  * sleep and frames can be delivered all the way from an interrupt. This
391  * behavior is useful as it's designed to minimize latency and the default
392  * disposition of MAC is to optimize for latency.
393  *
394  * MAINTAINING CHAINS
395  *
396  * Another useful idea that MAC uses is to try and maintain frames in chains for
397  * as long as possible. The idea is that all of MAC can handle chains of frames
398  * structured as a series of mblk_t structures linked with the b_next pointer.
399  * When performing software classification and software fanout, MAC does not
400  * simply determine the destination and send the frame along. Instead, in the
401  * case of classification, it tries to maintain a chain for as long as possible
402  * before passing it along and performing additional processing.
403  *
404  * In the case of fanout, MAC first determines what the target software ring is
405  * for every frame in the original chain and constructs a new chain for each
406  * target. MAC then delivers the new chain to each software ring in succession.
407  *
408  * The whole rationale for doing this is that we want to try and maintain the
409  * pipe as much as possible and deliver as many frames through the stack at once
410  * that we can, rather than just pushing a single frame through. This can often
411  * help bring down latency and allows MAC to get a better sense of the overall
412  * activity in the system and properly engage worker threads.
413  *
414  * --------------------
415  * Bandwidth Management
416  * --------------------
417  *
418  * Bandwidth management is something that's built into the soft ring set itself.
419  * When bandwidth limits are placed on a flow, a corresponding soft ring set is
420  * toggled into bandwidth mode. This changes how we transmit and receive the
421  * frames in question.
422  *
423  * Bandwidth management is done on a per-tick basis. We translate the user's
424  * requested bandwidth from a quantity per-second into a quantity per-tick. MAC
425  * cannot process a frame across more than one tick, thus it sets a lower bound
426  * for the bandwidth cap to be a single MTU. This also means that when
427  * hires ticks are enabled (hz is set to 1000), that the minimum amount of
428  * bandwidth is higher, because the number of ticks has increased and MAC has to
429  * go from accepting 100 packets / sec to 1000 / sec.
430  *
431  * The bandwidth counter is reset by either the soft ring set's worker thread or
432  * a thread that is doing an inline transmit or receive if they discover that
433  * the current tick is in the future from the recorded tick.
434  *
435  * Whenever we're receiving or transmitting data, we end up leaving most of the
436  * work to the soft ring set's worker thread. This forces data inserted into the
437  * soft ring set to be effectively serialized and allows us to exhume bandwidth
438  * at a reasonable rate. If there is nothing in the soft ring set at the moment
439  * and the set has available bandwidth, then it may processed inline.
440  * Otherwise, the worker is responsible for taking care of the soft ring set.
441  *
442  * ---------------------
443  * The Receive Data Path
444  * ---------------------
445  *
446  * The following series of ASCII art images breaks apart the way that a frame
447  * comes in and is processed in MAC.
448  *
449  * Part 1 -- Initial frame receipt, SRS classification
450  *
451  * Here, a frame is received by a GLDv3 driver, generally in the context of an
452  * interrupt, and it ends up in mac_rx_common(). A driver calls either mac_rx or
453  * mac_rx_ring, depending on whether or not it supports rings and can identify
454  * the interrupt as having come from a specific ring. Here we determine whether
455  * or not it's fully classified and perform software classification as
456  * appropriate. From here, everything always ends up going to either entry [A]
457  * or entry [B] based on whether or not they have subflow processing needed. We
458  * leave via fanout or delivery.
459  *
460  *           +===========+
461  *           v hardware  v
462  *           v interrupt v
463  *           +===========+
464  *                 |
465  *                 * . . appropriate
466  *                 |     upcall made
467  *                 |     by GLDv3 driver  . . always
468  *                 |                      .
469  *  +--------+     |     +----------+     .    +---------------+
470  *  | GLDv3  |     +---->| mac_rx   |-----*--->| mac_rx_common |
471  *  | Driver |-->--+     +----------+          +---------------+
472  *  +--------+     |        ^                         |
473  *      |          |        ^                         v
474  *      ^          |        * . . always   +----------------------+
475  *      |          |        |              | mac_promisc_dispatch |
476  *      |          |    +-------------+    +----------------------+
477  *      |          +--->| mac_rx_ring |               |
478  *      |               +-------------+               * . . hw classified
479  *      |                                             v     or single flow?
480  *      |                                             |
481  *      |                                   +--------++--------------+
482  *      |                                   |        |               * hw class,
483  *      |                                   |        * hw classified | subflows
484  *      |                 no hw class and . *        | or single     | exist
485  *      |                 subflows          |        | flow          |
486  *      |                                   |        v               v
487  *      |                                   |   +-----------+   +-----------+
488  *      |                                   |   |   goto    |   |  goto     |
489  *      |                                   |   | entry [A] |   | entry [B] |
490  *      |                                   |   +-----------+   +-----------+
491  *      |                                   v          ^
492  *      |                            +-------------+   |
493  *      |                            | mac_rx_flow |   * SRS and flow found,
494  *      |                            +-------------+   | call flow cb
495  *      |                                   |          +------+
496  *      |                                   v                 |
497  *      v                             +==========+    +-----------------+
498  *      |                             v For each v--->| mac_rx_classify |
499  * +----------+                       v  mblk_t  v    +-----------------+
500  * |   srs    |                       +==========+
501  * | pollling |
502  * |  thread  |->------------------------------------------+
503  * +----------+                                            |
504  *                                                         v       . inline
505  *            +--------------------+   +----------+   +---------+  .
506  *    [A]---->| mac_rx_srs_process |-->| check bw |-->| enqueue |--*---------+
507  *            +--------------------+   |  limits  |   | frames  |            |
508  *               ^                     +----------+   | to SRS  |            |
509  *               |                                    +---------+            |
510  *               |  send chain              +--------+    |                  |
511  *               *  when clasified          | signal |    * BW limits,       |
512  *               |  flow changes            |  srs   |<---+ loopback,        |
513  *               |                          | worker |      stack too        |
514  *               |                          +--------+      deep             |
515  *      +-----------------+        +--------+                                |
516  *      | mac_flow_lookup |        |  srs   |     +---------------------+    |
517  *      +-----------------+        | worker |---->| mac_rx_srs_drain    |<---+
518  *               ^                 | thread |     | mac_rx_srs_drain_bw |
519  *               |                 +--------+     +---------------------+
520  *               |                                          |
521  *         +----------------------------+                   * software rings
522  *   [B]-->| mac_rx_srs_subflow_process |                   | for fanout?
523  *         +----------------------------+                   |
524  *                                               +----------+-----------+
525  *                                               |                      |
526  *                                               v                      v
527  *                                          +--------+             +--------+
528  *                                          |  goto  |             |  goto  |
529  *                                          | Part 2 |             | Part 3 |
530  *                                          +--------+             +--------+
531  *
532  * Part 2 -- Fanout
533  *
534  * This part is concerned with using software fanout to assign frames to
535  * software rings and then deliver them to MAC clients or allow those rings to
536  * be polled upon. While there are two different primary fanout entry points,
537  * mac_rx_fanout and mac_rx_proto_fanout, they behave in similar ways, and aside
538  * from some of the individual hashing techniques used, most of the general
539  * flow is the same.
540  *
541  *  +--------+              +-------------------+
542  *  |  From  |---+--------->| mac_rx_srs_fanout |----+
543  *  | Part 1 |   |          +-------------------+    |    +=================+
544  *  +--------+   |                                   |    v for each mblk_t v
545  *               * . . protocol only                 +--->v assign to new   v
546  *               |     fanout                        |    v chain based on  v
547  *               |                                   |    v hash % nrings   v
548  *               |    +-------------------------+    |    +=================+
549  *               +--->| mac_rx_srs_proto_fanout |----+             |
550  *                    +-------------------------+                  |
551  *                                                                 v
552  *    +------------+    +--------------------------+       +================+
553  *    | enqueue in |<---| mac_rx_soft_ring_process |<------v for each chain v
554  *    | soft ring  |    +--------------------------+       +================+
555  *    +------------+
556  *         |                                    +-----------+
557  *         * soft ring set                      | soft ring |
558  *         | empty and no                       |  worker   |
559  *         | worker?                            |  thread   |
560  *         |                                    +-----------+
561  *         +------*----------------+                  |
562  *         |      .                |                  v
563  *    No . *      . Yes            |       +------------------------+
564  *         |                       +----<--| mac_rx_soft_ring_drain |
565  *         |                       |       +------------------------+
566  *         v                       |
567  *   +-----------+                 v
568  *   |   signal  |         +---------------+
569  *   | soft ring |         | Deliver chain |
570  *   |   worker  |         | goto Part 3   |
571  *   +-----------+         +---------------+
572  *
573  *
574  * Part 3 -- Packet Delivery
575  *
576  * Here, we go through and deliver the mblk_t chain directly to a given
577  * processing function. In a lot of cases this is mac_rx_deliver(). In the case
578  * of DLS bypass being used, then instead we end up going ahead and deliver it
579  * to the direct callback registered with DLS, generally ip_input.
580  *
581  *
582  *   +---------+            +----------------+    +------------------+
583  *   |  From   |---+------->| mac_rx_deliver |--->| Off to DLS, or   |
584  *   | Parts 1 |   |        +----------------+    | other MAC client |
585  *   |  and 2  |   * DLS bypass                   +------------------+
586  *   +---------+   | enabled   +----------+    +-------------+
587  *                 +---------->| ip_input |--->|    To IP    |
588  *                             +----------+    | and beyond! |
589  *                                             +-------------+
590  *
591  * ----------------------
592  * The Transmit Data Path
593  * ----------------------
594  *
595  * Before we go into the images, it's worth talking about a problem that is a
596  * bit different from the receive data path. GLDv3 device drivers have a finite
597  * amount of transmit descriptors. When they run out, they return unused frames
598  * back to MAC. MAC, at this point has several options about what it will do,
599  * which vary based upon the settings that the client uses.
600  *
601  * When a device runs out of descriptors, the next thing that MAC does is
602  * enqueue them off of the soft ring set or a software ring, depending on the
603  * configuration of the soft ring set. MAC will enqueue up to a high watermark
604  * of mblk_t chains, at which point it will indicate flow control back to the
605  * client. Once this condition is reached, any mblk_t chains that were not
606  * enqueued will be returned to the caller and they will have to decide what to
607  * do with them. There are various flags that control this behavior that a
608  * client may pass, which are discussed below.
609  *
610  * When this condition is hit, MAC also returns a cookie to the client in
611  * addition to unconsumed frames. Clients can poll on that cookie and register a
612  * callback with MAC to be notified when they are no longer subject to flow
613  * control, at which point they may continue to call mac_tx(). This flow control
614  * actually manages to work itself all the way up the stack, back through dls,
615  * to ip, through the various protocols, and to sockfs.
616  *
617  * While the behavior described above is the default, this behavior can be
618  * modified. There are two alternate modes, described below, which are
619  * controlled with flags.
620  *
621  * DROP MODE
622  *
623  * This mode is controlled by having the client pass the MAC_DROP_ON_NO_DESC
624  * flag. When this is passed, if a device driver runs out of transmit
625  * descriptors, then the MAC layer will drop any unsent traffic. The client in
626  * this case will never have any frames returned to it.
627  *
628  * DON'T ENQUEUE
629  *
630  * This mode is controlled by having the client pass the MAC_TX_NO_ENQUEUE flag.
631  * If the MAC_DROP_ON_NO_DESC flag is also passed, it takes precedence. In this
632  * mode, when we hit a case where a driver runs out of transmit descriptors,
633  * then instead of enqueuing packets in a soft ring set or software ring, we
634  * instead return the mblk_t chain back to the caller and immediately put the
635  * soft ring set into flow control mode.
636  *
637  * The following series of ASCII art images describe the transmit data path that
638  * MAC clients enter into based on calling into mac_tx(). A soft ring set has a
639  * transmission function associated with it. There are seven possible
640  * transmission modes, some of which share function entry points. The one that a
641  * soft ring set gets depends on properties such as whether there are
642  * transmission rings for fanout, whether the device involves aggregations,
643  * whether any bandwidth limits exist, etc.
644  *
645  *
646  * Part 1 -- Initial checks
647  *
648  *      * . called by
649  *      |   MAC clients
650  *      v                     . . No
651  *  +--------+  +-----------+ .   +-------------------+  +====================+
652  *  | mac_tx |->| device    |-*-->| mac_protect_check |->v Is this the simple v
653  *  +--------+  | quiesced? |     +-------------------+  v case? See [1]      v
654  *              +-----------+            |               +====================+
655  *                  * . Yes              * failed                 |
656  *                  v                    | frames                 |
657  *             +--------------+          |                +-------+---------+
658  *             | freemsgchain |<---------+          Yes . *            No . *
659  *             +--------------+                           v                 v
660  *                                                  +-----------+     +--------+
661  *                                                  |   goto    |     |  goto  |
662  *                                                  |  Part 2   |     | SRS TX |
663  *                                                  | Entry [A] |     |  func  |
664  *                                                  +-----------+     +--------+
665  *                                                        |                 |
666  *                                                        |                 v
667  *                                                        |           +--------+
668  *                                                        +---------->| return |
669  *                                                                    | cookie |
670  *                                                                    +--------+
671  *
672  * [1] The simple case refers to the SRS being configured with the
673  * SRS_TX_DEFAULT transmission mode, having a single mblk_t (not a chain), their
674  * being only a single active client, and not having a backlog in the srs.
675  *
676  *
677  * Part 2 -- The SRS transmission functions
678  *
679  * This part is a bit more complicated. The different transmission paths often
680  * leverage one another. In this case, we'll draw out the more common ones
681  * before the parts that depend upon them. Here, we're going to start with the
682  * workings of mac_tx_send() a common function that most of the others end up
683  * calling.
684  *
685  *      +-------------+
686  *      | mac_tx_send |
687  *      +-------------+
688  *            |
689  *            v
690  *      +=============+    +==============+
691  *      v  more than  v--->v    check     v
692  *      v one client? v    v VLAN and add v
693  *      +=============+    v  VLAN tags   v
694  *            |            +==============+
695  *            |                  |
696  *            +------------------+
697  *            |
698  *            |                 [A]
699  *            v                  |
700  *       +============+ . No     v
701  *       v more than  v .     +==========+     +--------------------------+
702  *       v one active v-*---->v for each v---->| mac_promisc_dispatch_one |---+
703  *       v  client?   v       v mblk_t   v     +--------------------------+   |
704  *       +============+       +==========+        ^                           |
705  *            |                                   |       +==========+        |
706  *            * . Yes                             |       v hardware v<-------+
707  *            v                      +------------+       v  rings?  v
708  *       +==========+                |                    +==========+
709  *       v for each v       No . . . *                         |
710  *       v mblk_t   v       specific |                         |
711  *       +==========+       flow     |                   +-----+-----+
712  *            |                      |                   |           |
713  *            v                      |                   v           v
714  *    +-----------------+            |               +-------+  +---------+
715  *    | mac_tx_classify |------------+               | GLDv3 |  |  GLDv3  |
716  *    +-----------------+                            |TX func|  | ring tx |
717  *            |                                      +-------+  |  func   |
718  *            * Specific flow, generally                 |      +---------+
719  *            | bcast, mcast, loopback                   |           |
720  *            v                                          +-----+-----+
721  *      +==========+       +---------+                         |
722  *      v valid L2 v--*--->| freemsg |                         v
723  *      v  header  v  . No +---------+               +-------------------+
724  *      +==========+                                 | return unconsumed |
725  *            * . Yes                                |   frames to the   |
726  *            v                                      |      caller       |
727  *      +===========+                                +-------------------+
728  *      v braodcast v      +----------------+                  ^
729  *      v   flow?   v--*-->| mac_bcast_send |------------------+
730  *      +===========+  .   +----------------+                  |
731  *            |        . . Yes                                 |
732  *       No . *                                                v
733  *            |  +---------------------+  +---------------+  +----------+
734  *            +->|mac_promisc_dispatch |->| mac_fix_cksum |->|   flow   |
735  *               +---------------------+  +---------------+  | callback |
736  *                                                           +----------+
737  *
738  *
739  * In addition, many but not all of the routines, all rely on
740  * mac_tx_softring_process as an entry point.
741  *
742  *
743  *                                           . No             . No
744  * +--------------------------+   +========+ .  +===========+ .  +-------------+
745  * | mac_tx_soft_ring_process |-->v worker v-*->v out of tx v-*->|    goto     |
746  * +--------------------------+   v only?  v    v  descr.?  v    | mac_tx_send |
747  *                                +========+    +===========+    +-------------+
748  *                              Yes . *               * . Yes           |
749  *                   . No             v               |                 v
750  *     v=========+   .          +===========+ . Yes   |     Yes .  +==========+
751  *     v apppend v<--*----------v out of tx v-*-------+---------*--v returned v
752  *     v mblk_t  v              v  descr.?  v         |            v frames?  v
753  *     v chain   v              +===========+         |            +==========+
754  *     +=========+                                    |                 *. No
755  *         |                                          |                 v
756  *         v                                          v           +------------+
757  * +===================+           +----------------------+       |   done     |
758  * v worker scheduled? v           | mac_tx_sring_enqueue |       | processing |
759  * v Out of tx descr?  v           +----------------------+       +------------+
760  * +===================+                      |
761  *    |           |           . Yes           v
762  *    * Yes       * No        .         +============+
763  *    |           v         +-*---------v drop on no v
764  *    |      +========+     v           v  TX desc?  v
765  *    |      v  wake  v  +----------+   +============+
766  *    |      v worker v  | mac_pkt_ |         * . No
767  *    |      +========+  | drop     |         |         . Yes         . No
768  *    |           |      +----------+         v         .             .
769  *    |           |         v   ^     +===============+ .  +========+ .
770  *    +--+--------+---------+   |     v Don't enqueue v-*->v ring   v-*----+
771  *       |                      |     v     Set?      v    v empty? v      |
772  *       |      +---------------+     +===============+    +========+      |
773  *       |      |                            |                |            |
774  *       |      |        +-------------------+                |            |
775  *       |      *. Yes   |                          +---------+            |
776  *       |      |        v                          v                      v
777  *       |      |  +===========+               +========+      +--------------+
778  *       |      +<-v At hiwat? v               v append v      |    return    |
779  *       |         +===========+               v mblk_t v      | mblk_t chain |
780  *       |                  * No               v chain  v      |   and flow   |
781  *       |                  v                  +========+      |    control   |
782  *       |               +=========+                |          |    cookie    |
783  *       |               v  append v                v          +--------------+
784  *       |               v  mblk_t v           +========+
785  *       |               v  chain  v           v  wake  v   +------------+
786  *       |               +=========+           v worker v-->|    done    |
787  *       |                    |                +========+   | processing |
788  *       |                    v       .. Yes                +------------+
789  *       |               +=========+  .   +========+
790  *       |               v  first  v--*-->v  wake  v
791  *       |               v append? v      v worker v
792  *       |               +=========+      +========+
793  *       |                   |                |
794  *       |              No . *                |
795  *       |                   v                |
796  *       |       +--------------+             |
797  *       +------>|   Return     |             |
798  *               | flow control |<------------+
799  *               |   cookie     |
800  *               +--------------+
801  *
802  *
803  * The remaining images are all specific to each of the different transmission
804  * modes.
805  *
806  * SRS TX DEFAULT
807  *
808  *      [ From Part 1 ]
809  *             |
810  *             v
811  * +-------------------------+
812  * | mac_tx_single_ring_mode |
813  * +-------------------------+
814  *            |
815  *            |       . Yes
816  *            v       .
817  *       +==========+ .  +============+
818  *       v   SRS    v-*->v   Try to   v---->---------------------+
819  *       v backlog? v    v enqueue in v                          |
820  *       +==========+    v     SRS    v-->------+                * . . Queue too
821  *            |          +============+         * don't enqueue  |     deep or
822  *            * . No         ^     |            | flag or at     |     drop flag
823  *            |              |     v            | hiwat,         |
824  *            v              |     |            | return    +---------+
825  *     +-------------+       |     |            | cookie    | freemsg |
826  *     |    goto     |-*-----+     |            |           +---------+
827  *     | mac_tx_send | . returned  |            |                |
828  *     +-------------+   mblk_t    |            |                |
829  *            |                    |            |                |
830  *            |                    |            |                |
831  *            * . . all mblk_t     * queued,    |                |
832  *            v     consumed       | may return |                |
833  *     +-------------+             | tx cookie  |                |
834  *     | SRS TX func |<------------+------------+----------------+
835  *     |  completed  |
836  *     +-------------+
837  *
838  * SRS_TX_SERIALIZE
839  *
840  *   +------------------------+
841  *   | mac_tx_serializer_mode |
842  *   +------------------------+
843  *               |
844  *               |        . No
845  *               v        .
846  *         +============+ .  +============+    +-------------+   +============+
847  *         v srs being  v-*->v  set SRS   v--->|    goto     |-->v remove SRS v
848  *         v processed? v    v proc flags v    | mac_tx_send |   v proc flag  v
849  *         +============+    +============+    +-------------+   +============+
850  *               |                                                     |
851  *               * Yes                                                 |
852  *               v                                       . No          v
853  *      +--------------------+                           .        +==========+
854  *      | mac_tx_srs_enqueue |  +------------------------*-----<--v returned v
855  *      +--------------------+  |                                 v frames?  v
856  *               |              |   . Yes                         +==========+
857  *               |              |   .                                  |
858  *               |              |   . +=========+                      v
859  *               v              +-<-*-v queued  v     +--------------------+
860  *        +-------------+       |     v frames? v<----| mac_tx_srs_enqueue |
861  *        | SRS TX func |       |     +=========+     +--------------------+
862  *        | completed,  |<------+         * . Yes
863  *        | may return  |       |         v
864  *        |   cookie    |       |     +========+
865  *        +-------------+       +-<---v  wake  v
866  *                                    v worker v
867  *                                    +========+
868  *
869  *
870  * SRS_TX_FANOUT
871  *
872  *                                             . Yes
873  *   +--------------------+    +=============+ .   +--------------------------+
874  *   | mac_tx_fanout_mode |--->v Have fanout v-*-->|           goto           |
875  *   +--------------------+    v   hint?     v     | mac_rx_soft_ring_process |
876  *                             +=============+     +--------------------------+
877  *                                   * . No                    |
878  *                                   v                         ^
879  *                             +===========+                   |
880  *                        +--->v for each  v           +===============+
881  *                        |    v   mblk_t  v           v pick softring v
882  *                 same   *    +===========+           v   from hash   v
883  *                 hash   |          |                 +===============+
884  *                        |          v                         |
885  *                        |   +--------------+                 |
886  *                        +---| mac_pkt_hash |--->*------------+
887  *                            +--------------+    . different
888  *                                                  hash or
889  *                                                  done proc.
890  * SRS_TX_AGGR                                      chain
891  *
892  *   +------------------+    +================================+
893  *   | mac_tx_aggr_mode |--->v Use aggr capab function to     v
894  *   +------------------+    v find appropriate tx ring.      v
895  *                           v Applies hash based on aggr     v
896  *                           v policy, see mac_tx_aggr_mode() v
897  *                           +================================+
898  *                                          |
899  *                                          v
900  *                           +-------------------------------+
901  *                           |            goto               |
902  *                           |  mac_rx_srs_soft_ring_process |
903  *                           +-------------------------------+
904  *
905  *
906  * SRS_TX_BW, SRS_TX_BW_FANOUT, SRS_TX_BW_AGGR
907  *
908  * Note, all three of these tx functions start from the same place --
909  * mac_tx_bw_mode().
910  *
911  *  +----------------+
912  *  | mac_tx_bw_mode |
913  *  +----------------+
914  *         |
915  *         v          . No               . No               . Yes
916  *  +==============+  .  +============+  .  +=============+ .  +=========+
917  *  v  Out of BW?  v--*->v SRS empty? v--*->v  reset BW   v-*->v Bump BW v
918  *  +==============+     +============+     v tick count? v    v Usage   v
919  *         |                   |            +=============+    +=========+
920  *         |         +---------+                   |                |
921  *         |         |        +--------------------+                |
922  *         |         |        |              +----------------------+
923  *         v         |        v              v
924  * +===============+ |  +==========+   +==========+      +------------------+
925  * v Don't enqueue v |  v  set bw  v   v Is aggr? v--*-->|       goto       |
926  * v   flag set?   v |  v enforced v   +==========+  .   | mac_tx_aggr_mode |-+
927  * +===============+ |  +==========+         |       .   +------------------+ |
928  *   |    Yes .*     |        |         No . *       .                        |
929  *   |         |     |        |              |       . Yes                    |
930  *   * . No    |     |        v              |                                |
931  *   |  +---------+  |   +========+          v              +======+          |
932  *   |  | freemsg |  |   v append v   +============+  . Yes v pick v          |
933  *   |  +---------+  |   v mblk_t v   v Is fanout? v--*---->v ring v          |
934  *   |      |        |   v chain  v   +============+        +======+          |
935  *   +------+        |   +========+          |                  |             |
936  *          v        |        |              v                  v             |
937  *    +---------+    |        v       +-------------+ +--------------------+  |
938  *    | return  |    |   +========+   |    goto     | |       goto         |  |
939  *    |  flow   |    |   v wakeup v   | mac_tx_send | | mac_tx_fanout_mode |  |
940  *    | control |    |   v worker v   +-------------+ +--------------------+  |
941  *    | cookie  |    |   +========+          |                  |             |
942  *    +---------+    |        |              |                  +------+------+
943  *                   |        v              |                         |
944  *                   |   +---------+         |                         v
945  *                   |   | return  |   +============+           +------------+
946  *                   |   |  flow   |   v unconsumed v-------+   |   done     |
947  *                   |   | control |   v   frames?  v       |   | processing |
948  *                   |   | cookie  |   +============+       |   +------------+
949  *                   |   +---------+         |              |
950  *                   |                  Yes  *              |
951  *                   |                       |              |
952  *                   |                 +===========+        |
953  *                   |                 v subtract  v        |
954  *                   |                 v unused bw v        |
955  *                   |                 +===========+        |
956  *                   |                       |              |
957  *                   |                       v              |
958  *                   |              +--------------------+  |
959  *                   +------------->| mac_tx_srs_enqueue |  |
960  *                                  +--------------------+  |
961  *                                           |              |
962  *                                           |              |
963  *                                     +------------+       |
964  *                                     |  return fc |       |
965  *                                     | cookie and |<------+
966  *                                     |    mblk_t  |
967  *                                     +------------+
968  */
969 
970 #include <sys/types.h>
971 #include <sys/callb.h>
972 #include <sys/sdt.h>
973 #include <sys/strsubr.h>
974 #include <sys/strsun.h>
975 #include <sys/vlan.h>
976 #include <sys/stack.h>
977 #include <sys/archsystm.h>
978 #include <inet/ipsec_impl.h>
979 #include <inet/ip_impl.h>
980 #include <inet/sadb.h>
981 #include <inet/ipsecesp.h>
982 #include <inet/ipsecah.h>
983 #include <inet/ip6.h>
984 
985 #include <sys/mac_impl.h>
986 #include <sys/mac_client_impl.h>
987 #include <sys/mac_client_priv.h>
988 #include <sys/mac_soft_ring.h>
989 #include <sys/mac_flow_impl.h>
990 
991 static mac_tx_cookie_t mac_tx_single_ring_mode(mac_soft_ring_set_t *, mblk_t *,
992     uintptr_t, uint16_t, mblk_t **);
993 static mac_tx_cookie_t mac_tx_serializer_mode(mac_soft_ring_set_t *, mblk_t *,
994     uintptr_t, uint16_t, mblk_t **);
995 static mac_tx_cookie_t mac_tx_fanout_mode(mac_soft_ring_set_t *, mblk_t *,
996     uintptr_t, uint16_t, mblk_t **);
997 static mac_tx_cookie_t mac_tx_bw_mode(mac_soft_ring_set_t *, mblk_t *,
998     uintptr_t, uint16_t, mblk_t **);
999 static mac_tx_cookie_t mac_tx_aggr_mode(mac_soft_ring_set_t *, mblk_t *,
1000     uintptr_t, uint16_t, mblk_t **);
1001 
1002 typedef struct mac_tx_mode_s {
1003 	mac_tx_srs_mode_t	mac_tx_mode;
1004 	mac_tx_func_t		mac_tx_func;
1005 } mac_tx_mode_t;
1006 
1007 /*
1008  * There are seven modes of operation on the Tx side. These modes get set
1009  * in mac_tx_srs_setup(). Except for the experimental TX_SERIALIZE mode,
1010  * none of the other modes are user configurable. They get selected by
1011  * the system depending upon whether the link (or flow) has multiple Tx
1012  * rings or a bandwidth configured, or if the link is an aggr, etc.
1013  *
1014  * When the Tx SRS is operating in aggr mode (st_mode) or if there are
1015  * multiple Tx rings owned by Tx SRS, then each Tx ring (pseudo or
1016  * otherwise) will have a soft ring associated with it. These soft rings
1017  * are stored in srs_tx_soft_rings[] array.
1018  *
1019  * Additionally in the case of aggr, there is the st_soft_rings[] array
1020  * in the mac_srs_tx_t structure. This array is used to store the same
1021  * set of soft rings that are present in srs_tx_soft_rings[] array but
1022  * in a different manner. The soft ring associated with the pseudo Tx
1023  * ring is saved at mr_index (of the pseudo ring) in st_soft_rings[]
1024  * array. This helps in quickly getting the soft ring associated with the
1025  * Tx ring when aggr_find_tx_ring() returns the pseudo Tx ring that is to
1026  * be used for transmit.
1027  */
1028 mac_tx_mode_t mac_tx_mode_list[] = {
1029 	{SRS_TX_DEFAULT,	mac_tx_single_ring_mode},
1030 	{SRS_TX_SERIALIZE,	mac_tx_serializer_mode},
1031 	{SRS_TX_FANOUT,		mac_tx_fanout_mode},
1032 	{SRS_TX_BW,		mac_tx_bw_mode},
1033 	{SRS_TX_BW_FANOUT,	mac_tx_bw_mode},
1034 	{SRS_TX_AGGR,		mac_tx_aggr_mode},
1035 	{SRS_TX_BW_AGGR,	mac_tx_bw_mode}
1036 };
1037 
1038 /*
1039  * Soft Ring Set (SRS) - The Run time code that deals with
1040  * dynamic polling from the hardware, bandwidth enforcement,
1041  * fanout etc.
1042  *
1043  * We try to use H/W classification on NIC and assign traffic for
1044  * a MAC address to a particular Rx ring or ring group. There is a
1045  * 1-1 mapping between a SRS and a Rx ring. The SRS dynamically
1046  * switches the underlying Rx ring between interrupt and
1047  * polling mode and enforces any specified B/W control.
1048  *
1049  * There is always a SRS created and tied to each H/W and S/W rule.
1050  * Whenever we create a H/W rule, we always add the the same rule to
1051  * S/W classifier and tie a SRS to it.
1052  *
1053  * In case a B/W control is specified, it is broken into bytes
1054  * per ticks and as soon as the quota for a tick is exhausted,
1055  * the underlying Rx ring is forced into poll mode for remainder of
1056  * the tick. The SRS poll thread only polls for bytes that are
1057  * allowed to come in the SRS. We typically let 4x the configured
1058  * B/W worth of packets to come in the SRS (to prevent unnecessary
1059  * drops due to bursts) but only process the specified amount.
1060  *
1061  * A MAC client (e.g. a VNIC or aggr) can have 1 or more
1062  * Rx rings (and corresponding SRSs) assigned to it. The SRS
1063  * in turn can have softrings to do protocol level fanout or
1064  * softrings to do S/W based fanout or both. In case the NIC
1065  * has no Rx rings, we do S/W classification to respective SRS.
1066  * The S/W classification rule is always setup and ready. This
1067  * allows the MAC layer to reassign Rx rings whenever needed
1068  * but packets still continue to flow via the default path and
1069  * getting S/W classified to correct SRS.
1070  *
1071  * The SRS's are used on both Tx and Rx side. They use the same
1072  * data structure but the processing routines have slightly different
1073  * semantics due to the fact that Rx side needs to do dynamic
1074  * polling etc.
1075  *
1076  * Dynamic Polling Notes
1077  * =====================
1078  *
1079  * Each Soft ring set is capable of switching its Rx ring between
1080  * interrupt and poll mode and actively 'polls' for packets in
1081  * poll mode. If the SRS is implementing a B/W limit, it makes
1082  * sure that only Max allowed packets are pulled in poll mode
1083  * and goes to poll mode as soon as B/W limit is exceeded. As
1084  * such, there are no overheads to implement B/W limits.
1085  *
1086  * In poll mode, its better to keep the pipeline going where the
1087  * SRS worker thread keeps processing packets and poll thread
1088  * keeps bringing more packets (specially if they get to run
1089  * on different CPUs). This also prevents the overheads associated
1090  * by excessive signalling (on NUMA machines, this can be
1091  * pretty devastating). The exception is latency optimized case
1092  * where worker thread does no work and interrupt and poll thread
1093  * are allowed to do their own drain.
1094  *
1095  * We use the following policy to control Dynamic Polling:
1096  * 1) We switch to poll mode anytime the processing
1097  *    thread causes a backlog to build up in SRS and
1098  *    its associated Soft Rings (sr_poll_pkt_cnt > 0).
1099  * 2) As long as the backlog stays under the low water
1100  *    mark (sr_lowat), we poll the H/W for more packets.
1101  * 3) If the backlog (sr_poll_pkt_cnt) exceeds low
1102  *    water mark, we stay in poll mode but don't poll
1103  *    the H/W for more packets.
1104  * 4) Anytime in polling mode, if we poll the H/W for
1105  *    packets and find nothing plus we have an existing
1106  *    backlog (sr_poll_pkt_cnt > 0), we stay in polling
1107  *    mode but don't poll the H/W for packets anymore
1108  *    (let the polling thread go to sleep).
1109  * 5) Once the backlog is relived (packets are processed)
1110  *    we reenable polling (by signalling the poll thread)
1111  *    only when the backlog dips below sr_poll_thres.
1112  * 6) sr_hiwat is used exclusively when we are not
1113  *    polling capable and is used to decide when to
1114  *    drop packets so the SRS queue length doesn't grow
1115  *    infinitely.
1116  *
1117  * NOTE: Also see the block level comment on top of mac_soft_ring.c
1118  */
1119 
1120 /*
1121  * mac_latency_optimize
1122  *
1123  * Controls whether the poll thread can process the packets inline
1124  * or let the SRS worker thread do the processing. This applies if
1125  * the SRS was not being processed. For latency sensitive traffic,
1126  * this needs to be true to allow inline processing. For throughput
1127  * under load, this should be false.
1128  *
1129  * This (and other similar) tunable should be rolled into a link
1130  * or flow specific workload hint that can be set using dladm
1131  * linkprop (instead of multiple such tunables).
1132  */
1133 boolean_t mac_latency_optimize = B_TRUE;
1134 
1135 /*
1136  * MAC_RX_SRS_ENQUEUE_CHAIN and MAC_TX_SRS_ENQUEUE_CHAIN
1137  *
1138  * queue a mp or chain in soft ring set and increment the
1139  * local count (srs_count) for the SRS and the shared counter
1140  * (srs_poll_pkt_cnt - shared between SRS and its soft rings
1141  * to track the total unprocessed packets for polling to work
1142  * correctly).
1143  *
1144  * The size (total bytes queued) counters are incremented only
1145  * if we are doing B/W control.
1146  */
1147 #define	MAC_SRS_ENQUEUE_CHAIN(mac_srs, head, tail, count, sz) {		\
1148 	ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock));			\
1149 	if ((mac_srs)->srs_last != NULL)				\
1150 		(mac_srs)->srs_last->b_next = (head);			\
1151 	else								\
1152 		(mac_srs)->srs_first = (head);				\
1153 	(mac_srs)->srs_last = (tail);					\
1154 	(mac_srs)->srs_count += count;					\
1155 }
1156 
1157 #define	MAC_RX_SRS_ENQUEUE_CHAIN(mac_srs, head, tail, count, sz) {	\
1158 	mac_srs_rx_t	*srs_rx = &(mac_srs)->srs_rx;			\
1159 									\
1160 	MAC_SRS_ENQUEUE_CHAIN(mac_srs, head, tail, count, sz);		\
1161 	srs_rx->sr_poll_pkt_cnt += count;				\
1162 	ASSERT(srs_rx->sr_poll_pkt_cnt > 0);				\
1163 	if ((mac_srs)->srs_type & SRST_BW_CONTROL) {			\
1164 		(mac_srs)->srs_size += (sz);				\
1165 		mutex_enter(&(mac_srs)->srs_bw->mac_bw_lock);		\
1166 		(mac_srs)->srs_bw->mac_bw_sz += (sz);			\
1167 		mutex_exit(&(mac_srs)->srs_bw->mac_bw_lock);		\
1168 	}								\
1169 }
1170 
1171 #define	MAC_TX_SRS_ENQUEUE_CHAIN(mac_srs, head, tail, count, sz) {	\
1172 	mac_srs->srs_state |= SRS_ENQUEUED;				\
1173 	MAC_SRS_ENQUEUE_CHAIN(mac_srs, head, tail, count, sz);		\
1174 	if ((mac_srs)->srs_type & SRST_BW_CONTROL) {			\
1175 		(mac_srs)->srs_size += (sz);				\
1176 		(mac_srs)->srs_bw->mac_bw_sz += (sz);			\
1177 	}								\
1178 }
1179 
1180 /*
1181  * Turn polling on routines
1182  */
1183 #define	MAC_SRS_POLLING_ON(mac_srs) {					\
1184 	ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock));			\
1185 	if (((mac_srs)->srs_state &					\
1186 	    (SRS_POLLING_CAPAB|SRS_POLLING)) == SRS_POLLING_CAPAB) {	\
1187 		(mac_srs)->srs_state |= SRS_POLLING;			\
1188 		(void) mac_hwring_disable_intr((mac_ring_handle_t)	\
1189 		    (mac_srs)->srs_ring);				\
1190 		(mac_srs)->srs_rx.sr_poll_on++;				\
1191 	}								\
1192 }
1193 
1194 #define	MAC_SRS_WORKER_POLLING_ON(mac_srs) {				\
1195 	ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock));			\
1196 	if (((mac_srs)->srs_state &					\
1197 	    (SRS_POLLING_CAPAB|SRS_WORKER|SRS_POLLING)) == 		\
1198 	    (SRS_POLLING_CAPAB|SRS_WORKER)) {				\
1199 		(mac_srs)->srs_state |= SRS_POLLING;			\
1200 		(void) mac_hwring_disable_intr((mac_ring_handle_t)	\
1201 		    (mac_srs)->srs_ring);				\
1202 		(mac_srs)->srs_rx.sr_worker_poll_on++;			\
1203 	}								\
1204 }
1205 
1206 /*
1207  * MAC_SRS_POLL_RING
1208  *
1209  * Signal the SRS poll thread to poll the underlying H/W ring
1210  * provided it wasn't already polling (SRS_GET_PKTS was set).
1211  *
1212  * Poll thread gets to run only from mac_rx_srs_drain() and only
1213  * if the drain was being done by the worker thread.
1214  */
1215 #define	MAC_SRS_POLL_RING(mac_srs) {					\
1216 	mac_srs_rx_t	*srs_rx = &(mac_srs)->srs_rx;			\
1217 									\
1218 	ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock));			\
1219 	srs_rx->sr_poll_thr_sig++;					\
1220 	if (((mac_srs)->srs_state & 					\
1221 	    (SRS_POLLING_CAPAB|SRS_WORKER|SRS_GET_PKTS)) ==		\
1222 		(SRS_WORKER|SRS_POLLING_CAPAB)) {			\
1223 		(mac_srs)->srs_state |= SRS_GET_PKTS;			\
1224 		cv_signal(&(mac_srs)->srs_cv);   			\
1225 	} else {							\
1226 		srs_rx->sr_poll_thr_busy++;				\
1227 	}								\
1228 }
1229 
1230 /*
1231  * MAC_SRS_CHECK_BW_CONTROL
1232  *
1233  * Check to see if next tick has started so we can reset the
1234  * SRS_BW_ENFORCED flag and allow more packets to come in the
1235  * system.
1236  */
1237 #define	MAC_SRS_CHECK_BW_CONTROL(mac_srs) {				\
1238 	ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock));			\
1239 	ASSERT(((mac_srs)->srs_type & SRST_TX) ||			\
1240 	    MUTEX_HELD(&(mac_srs)->srs_bw->mac_bw_lock));		\
1241 	clock_t now = ddi_get_lbolt();					\
1242 	if ((mac_srs)->srs_bw->mac_bw_curr_time != now) {		\
1243 		(mac_srs)->srs_bw->mac_bw_curr_time = now;		\
1244 		(mac_srs)->srs_bw->mac_bw_used = 0;	       		\
1245 		if ((mac_srs)->srs_bw->mac_bw_state & SRS_BW_ENFORCED)	\
1246 			(mac_srs)->srs_bw->mac_bw_state &= ~SRS_BW_ENFORCED; \
1247 	}								\
1248 }
1249 
1250 /*
1251  * MAC_SRS_WORKER_WAKEUP
1252  *
1253  * Wake up the SRS worker thread to process the queue as long as
1254  * no one else is processing the queue. If we are optimizing for
1255  * latency, we wake up the worker thread immediately or else we
1256  * wait mac_srs_worker_wakeup_ticks before worker thread gets
1257  * woken up.
1258  */
1259 int mac_srs_worker_wakeup_ticks = 0;
1260 #define	MAC_SRS_WORKER_WAKEUP(mac_srs) {				\
1261 	ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock));			\
1262 	if (!((mac_srs)->srs_state & SRS_PROC) &&			\
1263 		(mac_srs)->srs_tid == NULL) {				\
1264 		if (((mac_srs)->srs_state & SRS_LATENCY_OPT) ||		\
1265 			(mac_srs_worker_wakeup_ticks == 0))		\
1266 			cv_signal(&(mac_srs)->srs_async);		\
1267 		else							\
1268 			(mac_srs)->srs_tid =				\
1269 				timeout(mac_srs_fire, (mac_srs),	\
1270 					mac_srs_worker_wakeup_ticks);	\
1271 	}								\
1272 }
1273 
1274 #define	TX_BANDWIDTH_MODE(mac_srs)				\
1275 	((mac_srs)->srs_tx.st_mode == SRS_TX_BW ||		\
1276 	    (mac_srs)->srs_tx.st_mode == SRS_TX_BW_FANOUT ||	\
1277 	    (mac_srs)->srs_tx.st_mode == SRS_TX_BW_AGGR)
1278 
1279 #define	TX_SRS_TO_SOFT_RING(mac_srs, head, hint) {			\
1280 	if (tx_mode == SRS_TX_BW_FANOUT)				\
1281 		(void) mac_tx_fanout_mode(mac_srs, head, hint, 0, NULL);\
1282 	else								\
1283 		(void) mac_tx_aggr_mode(mac_srs, head, hint, 0, NULL);	\
1284 }
1285 
1286 /*
1287  * MAC_TX_SRS_BLOCK
1288  *
1289  * Always called from mac_tx_srs_drain() function. SRS_TX_BLOCKED
1290  * will be set only if srs_tx_woken_up is FALSE. If
1291  * srs_tx_woken_up is TRUE, it indicates that the wakeup arrived
1292  * before we grabbed srs_lock to set SRS_TX_BLOCKED. We need to
1293  * attempt to transmit again and not setting SRS_TX_BLOCKED does
1294  * that.
1295  */
1296 #define	MAC_TX_SRS_BLOCK(srs, mp)	{			\
1297 	ASSERT(MUTEX_HELD(&(srs)->srs_lock));			\
1298 	if ((srs)->srs_tx.st_woken_up) {			\
1299 		(srs)->srs_tx.st_woken_up = B_FALSE;		\
1300 	} else {						\
1301 		ASSERT(!((srs)->srs_state & SRS_TX_BLOCKED));	\
1302 		(srs)->srs_state |= SRS_TX_BLOCKED;		\
1303 		(srs)->srs_tx.st_stat.mts_blockcnt++;		\
1304 	}							\
1305 }
1306 
1307 /*
1308  * MAC_TX_SRS_TEST_HIWAT
1309  *
1310  * Called before queueing a packet onto Tx SRS to test and set
1311  * SRS_TX_HIWAT if srs_count exceeds srs_tx_hiwat.
1312  */
1313 #define	MAC_TX_SRS_TEST_HIWAT(srs, mp, tail, cnt, sz, cookie) {		\
1314 	boolean_t enqueue = 1;						\
1315 									\
1316 	if ((srs)->srs_count > (srs)->srs_tx.st_hiwat) {		\
1317 		/*							\
1318 		 * flow-controlled. Store srs in cookie so that it	\
1319 		 * can be returned as mac_tx_cookie_t to client		\
1320 		 */							\
1321 		(srs)->srs_state |= SRS_TX_HIWAT;			\
1322 		cookie = (mac_tx_cookie_t)srs;				\
1323 		(srs)->srs_tx.st_hiwat_cnt++;				\
1324 		if ((srs)->srs_count > (srs)->srs_tx.st_max_q_cnt) {	\
1325 			/* increment freed stats */			\
1326 			(srs)->srs_tx.st_stat.mts_sdrops += cnt;	\
1327 			/*						\
1328 			 * b_prev may be set to the fanout hint		\
1329 			 * hence can't use freemsg directly		\
1330 			 */						\
1331 			mac_pkt_drop(NULL, NULL, mp_chain, B_FALSE);	\
1332 			DTRACE_PROBE1(tx_queued_hiwat,			\
1333 			    mac_soft_ring_set_t *, srs);		\
1334 			enqueue = 0;					\
1335 		}							\
1336 	}								\
1337 	if (enqueue)							\
1338 		MAC_TX_SRS_ENQUEUE_CHAIN(srs, mp, tail, cnt, sz);	\
1339 }
1340 
1341 /* Some utility macros */
1342 #define	MAC_SRS_BW_LOCK(srs)						\
1343 	if (!(srs->srs_type & SRST_TX))					\
1344 		mutex_enter(&srs->srs_bw->mac_bw_lock);
1345 
1346 #define	MAC_SRS_BW_UNLOCK(srs)						\
1347 	if (!(srs->srs_type & SRST_TX))					\
1348 		mutex_exit(&srs->srs_bw->mac_bw_lock);
1349 
1350 #define	MAC_TX_SRS_DROP_MESSAGE(srs, mp, cookie) {		\
1351 	mac_pkt_drop(NULL, NULL, mp, B_FALSE);			\
1352 	/* increment freed stats */				\
1353 	mac_srs->srs_tx.st_stat.mts_sdrops++;			\
1354 	cookie = (mac_tx_cookie_t)srs;				\
1355 }
1356 
1357 #define	MAC_TX_SET_NO_ENQUEUE(srs, mp_chain, ret_mp, cookie) {		\
1358 	mac_srs->srs_state |= SRS_TX_WAKEUP_CLIENT;			\
1359 	cookie = (mac_tx_cookie_t)srs;					\
1360 	*ret_mp = mp_chain;						\
1361 }
1362 
1363 /*
1364  * MAC_RX_SRS_TOODEEP
1365  *
1366  * Macro called as part of receive-side processing to determine if handling
1367  * can occur in situ (in the interrupt thread) or if it should be left to a
1368  * worker thread.  Note that the constant used to make this determination is
1369  * not entirely made-up, and is a result of some emprical validation. That
1370  * said, the constant is left as a static variable to allow it to be
1371  * dynamically tuned in the field if and as needed.
1372  */
1373 static uintptr_t mac_rx_srs_stack_needed = 10240;
1374 static uint_t mac_rx_srs_stack_toodeep;
1375 
1376 #ifndef STACK_GROWTH_DOWN
1377 #error Downward stack growth assumed.
1378 #endif
1379 
1380 #define	MAC_RX_SRS_TOODEEP() (STACK_BIAS + (uintptr_t)getfp() - \
1381 	(uintptr_t)curthread->t_stkbase < mac_rx_srs_stack_needed && \
1382 	++mac_rx_srs_stack_toodeep)
1383 
1384 
1385 /*
1386  * Drop the rx packet and advance to the next one in the chain.
1387  */
1388 static void
mac_rx_drop_pkt(mac_soft_ring_set_t * srs,mblk_t * mp)1389 mac_rx_drop_pkt(mac_soft_ring_set_t *srs, mblk_t *mp)
1390 {
1391 	mac_srs_rx_t	*srs_rx = &srs->srs_rx;
1392 
1393 	ASSERT(mp->b_next == NULL);
1394 	mutex_enter(&srs->srs_lock);
1395 	MAC_UPDATE_SRS_COUNT_LOCKED(srs, 1);
1396 	MAC_UPDATE_SRS_SIZE_LOCKED(srs, msgdsize(mp));
1397 	mutex_exit(&srs->srs_lock);
1398 
1399 	srs_rx->sr_stat.mrs_sdrops++;
1400 	freemsg(mp);
1401 }
1402 
1403 /* DATAPATH RUNTIME ROUTINES */
1404 
1405 /*
1406  * mac_srs_fire
1407  *
1408  * Timer callback routine for waking up the SRS worker thread.
1409  */
1410 static void
mac_srs_fire(void * arg)1411 mac_srs_fire(void *arg)
1412 {
1413 	mac_soft_ring_set_t *mac_srs = (mac_soft_ring_set_t *)arg;
1414 
1415 	mutex_enter(&mac_srs->srs_lock);
1416 	if (mac_srs->srs_tid == 0) {
1417 		mutex_exit(&mac_srs->srs_lock);
1418 		return;
1419 	}
1420 
1421 	mac_srs->srs_tid = 0;
1422 	if (!(mac_srs->srs_state & SRS_PROC))
1423 		cv_signal(&mac_srs->srs_async);
1424 
1425 	mutex_exit(&mac_srs->srs_lock);
1426 }
1427 
1428 /*
1429  * 'hint' is fanout_hint (type of uint64_t) which is given by the TCP/IP stack,
1430  * and it is used on the TX path.
1431  */
1432 #define	HASH_HINT(hint)	\
1433 	((hint) ^ ((hint) >> 24) ^ ((hint) >> 16) ^ ((hint) >> 8))
1434 
1435 
1436 /*
1437  * hash based on the src address, dst address and the port information.
1438  */
1439 #define	HASH_ADDR(src, dst, ports)					\
1440 	(ntohl((src) + (dst)) ^ ((ports) >> 24) ^ ((ports) >> 16) ^	\
1441 	((ports) >> 8) ^ (ports))
1442 
1443 #define	COMPUTE_INDEX(key, sz)	(key % sz)
1444 
1445 #define	FANOUT_ENQUEUE_MP(head, tail, cnt, bw_ctl, sz, sz0, mp) {	\
1446 	if ((tail) != NULL) {						\
1447 		ASSERT((tail)->b_next == NULL);				\
1448 		(tail)->b_next = (mp);					\
1449 	} else {							\
1450 		ASSERT((head) == NULL);					\
1451 		(head) = (mp);						\
1452 	}								\
1453 	(tail) = (mp);							\
1454 	(cnt)++;							\
1455 	if ((bw_ctl))							\
1456 		(sz) += (sz0);						\
1457 }
1458 
1459 #define	MAC_FANOUT_DEFAULT	0
1460 #define	MAC_FANOUT_RND_ROBIN	1
1461 int mac_fanout_type = MAC_FANOUT_DEFAULT;
1462 
1463 #define	MAX_SR_TYPES	3
1464 /* fanout types for port based hashing */
1465 enum pkt_type {
1466 	V4_TCP = 0,
1467 	V4_UDP,
1468 	OTH,
1469 	UNDEF
1470 };
1471 
1472 /*
1473  * Pair of local and remote ports in the transport header
1474  */
1475 #define	PORTS_SIZE 4
1476 
1477 /*
1478  * mac_rx_srs_proto_fanout
1479  *
1480  * This routine delivers packets destined to an SRS into one of the
1481  * protocol soft rings.
1482  *
1483  * Given a chain of packets we need to split it up into multiple sub chains
1484  * destined into TCP, UDP or OTH soft ring. Instead of entering
1485  * the soft ring one packet at a time, we want to enter it in the form of a
1486  * chain otherwise we get this start/stop behaviour where the worker thread
1487  * goes to sleep and then next packets comes in forcing it to wake up etc.
1488  */
1489 static void
mac_rx_srs_proto_fanout(mac_soft_ring_set_t * mac_srs,mblk_t * head)1490 mac_rx_srs_proto_fanout(mac_soft_ring_set_t *mac_srs, mblk_t *head)
1491 {
1492 	struct ether_header		*ehp;
1493 	struct ether_vlan_header	*evhp;
1494 	uint32_t			sap;
1495 	ipha_t				*ipha;
1496 	uint8_t				*dstaddr;
1497 	size_t				hdrsize;
1498 	mblk_t				*mp;
1499 	mblk_t				*headmp[MAX_SR_TYPES];
1500 	mblk_t				*tailmp[MAX_SR_TYPES];
1501 	int				cnt[MAX_SR_TYPES];
1502 	size_t				sz[MAX_SR_TYPES];
1503 	size_t				sz1;
1504 	boolean_t			bw_ctl;
1505 	boolean_t			hw_classified;
1506 	boolean_t			dls_bypass;
1507 	boolean_t			is_ether;
1508 	boolean_t			is_unicast;
1509 	enum pkt_type			type;
1510 	mac_client_impl_t		*mcip = mac_srs->srs_mcip;
1511 
1512 	is_ether = (mcip->mci_mip->mi_info.mi_nativemedia == DL_ETHER);
1513 	bw_ctl = ((mac_srs->srs_type & SRST_BW_CONTROL) != 0);
1514 
1515 	/*
1516 	 * If we don't have a Rx ring, S/W classification would have done
1517 	 * its job and its a packet meant for us. If we were polling on
1518 	 * the default ring (i.e. there was a ring assigned to this SRS),
1519 	 * then we need to make sure that the mac address really belongs
1520 	 * to us.
1521 	 */
1522 	hw_classified = mac_srs->srs_ring != NULL &&
1523 	    mac_srs->srs_ring->mr_classify_type == MAC_HW_CLASSIFIER;
1524 
1525 	/*
1526 	 * Special clients (eg. VLAN, non ether, etc) need DLS
1527 	 * processing in the Rx path. SRST_DLS_BYPASS will be clear for
1528 	 * such SRSs. Another way of disabling bypass is to set the
1529 	 * MCIS_RX_BYPASS_DISABLE flag.
1530 	 */
1531 	dls_bypass = ((mac_srs->srs_type & SRST_DLS_BYPASS) != 0) &&
1532 	    ((mcip->mci_state_flags & MCIS_RX_BYPASS_DISABLE) == 0);
1533 
1534 	bzero(headmp, MAX_SR_TYPES * sizeof (mblk_t *));
1535 	bzero(tailmp, MAX_SR_TYPES * sizeof (mblk_t *));
1536 	bzero(cnt, MAX_SR_TYPES * sizeof (int));
1537 	bzero(sz, MAX_SR_TYPES * sizeof (size_t));
1538 
1539 	/*
1540 	 * We got a chain from SRS that we need to send to the soft rings.
1541 	 * Since squeues for TCP & IPv4 sap poll their soft rings (for
1542 	 * performance reasons), we need to separate out v4_tcp, v4_udp
1543 	 * and the rest goes in other.
1544 	 */
1545 	while (head != NULL) {
1546 		mp = head;
1547 		head = head->b_next;
1548 		mp->b_next = NULL;
1549 
1550 		type = OTH;
1551 		sz1 = (mp->b_cont == NULL) ? MBLKL(mp) : msgdsize(mp);
1552 
1553 		if (is_ether) {
1554 			/*
1555 			 * At this point we can be sure the packet at least
1556 			 * has an ether header.
1557 			 */
1558 			if (sz1 < sizeof (struct ether_header)) {
1559 				mac_rx_drop_pkt(mac_srs, mp);
1560 				continue;
1561 			}
1562 			ehp = (struct ether_header *)mp->b_rptr;
1563 
1564 			/*
1565 			 * Determine if this is a VLAN or non-VLAN packet.
1566 			 */
1567 			if ((sap = ntohs(ehp->ether_type)) == VLAN_TPID) {
1568 				evhp = (struct ether_vlan_header *)mp->b_rptr;
1569 				sap = ntohs(evhp->ether_type);
1570 				hdrsize = sizeof (struct ether_vlan_header);
1571 				/*
1572 				 * Check if the VID of the packet, if any,
1573 				 * belongs to this client.
1574 				 */
1575 				if (!mac_client_check_flow_vid(mcip,
1576 				    VLAN_ID(ntohs(evhp->ether_tci)))) {
1577 					mac_rx_drop_pkt(mac_srs, mp);
1578 					continue;
1579 				}
1580 			} else {
1581 				hdrsize = sizeof (struct ether_header);
1582 			}
1583 			is_unicast =
1584 			    ((((uint8_t *)&ehp->ether_dhost)[0] & 0x01) == 0);
1585 			dstaddr = (uint8_t *)&ehp->ether_dhost;
1586 		} else {
1587 			mac_header_info_t		mhi;
1588 
1589 			if (mac_header_info((mac_handle_t)mcip->mci_mip,
1590 			    mp, &mhi) != 0) {
1591 				mac_rx_drop_pkt(mac_srs, mp);
1592 				continue;
1593 			}
1594 			hdrsize = mhi.mhi_hdrsize;
1595 			sap = mhi.mhi_bindsap;
1596 			is_unicast = (mhi.mhi_dsttype == MAC_ADDRTYPE_UNICAST);
1597 			dstaddr = (uint8_t *)mhi.mhi_daddr;
1598 		}
1599 
1600 		if (!dls_bypass) {
1601 			FANOUT_ENQUEUE_MP(headmp[type], tailmp[type],
1602 			    cnt[type], bw_ctl, sz[type], sz1, mp);
1603 			continue;
1604 		}
1605 
1606 		if (sap == ETHERTYPE_IP) {
1607 			/*
1608 			 * If we are H/W classified, but we have promisc
1609 			 * on, then we need to check for the unicast address.
1610 			 */
1611 			if (hw_classified && mcip->mci_promisc_list != NULL) {
1612 				mac_address_t		*map;
1613 
1614 				rw_enter(&mcip->mci_rw_lock, RW_READER);
1615 				map = mcip->mci_unicast;
1616 				if (bcmp(dstaddr, map->ma_addr,
1617 				    map->ma_len) == 0)
1618 					type = UNDEF;
1619 				rw_exit(&mcip->mci_rw_lock);
1620 			} else if (is_unicast) {
1621 				type = UNDEF;
1622 			}
1623 		}
1624 
1625 		/*
1626 		 * This needs to become a contract with the driver for
1627 		 * the fast path.
1628 		 *
1629 		 * In the normal case the packet will have at least the L2
1630 		 * header and the IP + Transport header in the same mblk.
1631 		 * This is usually the case when the NIC driver sends up
1632 		 * the packet. This is also true when the stack generates
1633 		 * a packet that is looped back and when the stack uses the
1634 		 * fastpath mechanism. The normal case is optimized for
1635 		 * performance and may bypass DLS. All other cases go through
1636 		 * the 'OTH' type path without DLS bypass.
1637 		 */
1638 
1639 		ipha = (ipha_t *)(mp->b_rptr + hdrsize);
1640 		if ((type != OTH) && MBLK_RX_FANOUT_SLOWPATH(mp, ipha))
1641 			type = OTH;
1642 
1643 		if (type == OTH) {
1644 			FANOUT_ENQUEUE_MP(headmp[type], tailmp[type],
1645 			    cnt[type], bw_ctl, sz[type], sz1, mp);
1646 			continue;
1647 		}
1648 
1649 		ASSERT(type == UNDEF);
1650 		/*
1651 		 * We look for at least 4 bytes past the IP header to get
1652 		 * the port information. If we get an IP fragment, we don't
1653 		 * have the port information, and we use just the protocol
1654 		 * information.
1655 		 */
1656 		switch (ipha->ipha_protocol) {
1657 		case IPPROTO_TCP:
1658 			type = V4_TCP;
1659 			mp->b_rptr += hdrsize;
1660 			break;
1661 		case IPPROTO_UDP:
1662 			type = V4_UDP;
1663 			mp->b_rptr += hdrsize;
1664 			break;
1665 		default:
1666 			type = OTH;
1667 			break;
1668 		}
1669 
1670 		FANOUT_ENQUEUE_MP(headmp[type], tailmp[type], cnt[type],
1671 		    bw_ctl, sz[type], sz1, mp);
1672 	}
1673 
1674 	for (type = V4_TCP; type < UNDEF; type++) {
1675 		if (headmp[type] != NULL) {
1676 			mac_soft_ring_t			*softring;
1677 
1678 			ASSERT(tailmp[type]->b_next == NULL);
1679 			switch (type) {
1680 			case V4_TCP:
1681 				softring = mac_srs->srs_tcp_soft_rings[0];
1682 				break;
1683 			case V4_UDP:
1684 				softring = mac_srs->srs_udp_soft_rings[0];
1685 				break;
1686 			case OTH:
1687 				softring = mac_srs->srs_oth_soft_rings[0];
1688 			}
1689 			mac_rx_soft_ring_process(mcip, softring,
1690 			    headmp[type], tailmp[type], cnt[type], sz[type]);
1691 		}
1692 	}
1693 }
1694 
1695 int	fanout_unaligned = 0;
1696 
1697 /*
1698  * mac_rx_srs_long_fanout
1699  *
1700  * The fanout routine for VLANs, and for anything else that isn't performing
1701  * explicit dls bypass.  Returns -1 on an error (drop the packet due to a
1702  * malformed packet), 0 on success, with values written in *indx and *type.
1703  */
1704 static int
mac_rx_srs_long_fanout(mac_soft_ring_set_t * mac_srs,mblk_t * mp,uint32_t sap,size_t hdrsize,enum pkt_type * type,uint_t * indx)1705 mac_rx_srs_long_fanout(mac_soft_ring_set_t *mac_srs, mblk_t *mp,
1706     uint32_t sap, size_t hdrsize, enum pkt_type *type, uint_t *indx)
1707 {
1708 	ip6_t		*ip6h;
1709 	ipha_t		*ipha;
1710 	uint8_t		*whereptr;
1711 	uint_t		hash;
1712 	uint16_t	remlen;
1713 	uint8_t		nexthdr;
1714 	uint16_t	hdr_len;
1715 	uint32_t	src_val, dst_val;
1716 	boolean_t	modifiable = B_TRUE;
1717 	boolean_t	v6;
1718 
1719 	ASSERT(MBLKL(mp) >= hdrsize);
1720 
1721 	if (sap == ETHERTYPE_IPV6) {
1722 		v6 = B_TRUE;
1723 		hdr_len = IPV6_HDR_LEN;
1724 	} else if (sap == ETHERTYPE_IP) {
1725 		v6 = B_FALSE;
1726 		hdr_len = IP_SIMPLE_HDR_LENGTH;
1727 	} else {
1728 		*indx = 0;
1729 		*type = OTH;
1730 		return (0);
1731 	}
1732 
1733 	ip6h = (ip6_t *)(mp->b_rptr + hdrsize);
1734 	ipha = (ipha_t *)ip6h;
1735 
1736 	if ((uint8_t *)ip6h == mp->b_wptr) {
1737 		/*
1738 		 * The first mblk_t only includes the mac header.
1739 		 * Note that it is safe to change the mp pointer here,
1740 		 * as the subsequent operation does not assume mp
1741 		 * points to the start of the mac header.
1742 		 */
1743 		mp = mp->b_cont;
1744 
1745 		/*
1746 		 * Make sure the IP header points to an entire one.
1747 		 */
1748 		if (mp == NULL)
1749 			return (-1);
1750 
1751 		if (MBLKL(mp) < hdr_len) {
1752 			modifiable = (DB_REF(mp) == 1);
1753 
1754 			if (modifiable && !pullupmsg(mp, hdr_len))
1755 				return (-1);
1756 		}
1757 
1758 		ip6h = (ip6_t *)mp->b_rptr;
1759 		ipha = (ipha_t *)ip6h;
1760 	}
1761 
1762 	if (!modifiable || !(OK_32PTR((char *)ip6h)) ||
1763 	    ((uint8_t *)ip6h + hdr_len > mp->b_wptr)) {
1764 		/*
1765 		 * If either the IP header is not aligned, or it does not hold
1766 		 * the complete simple structure (a pullupmsg() is not an
1767 		 * option since it would result in an unaligned IP header),
1768 		 * fanout to the default ring.
1769 		 *
1770 		 * Note that this may cause packet reordering.
1771 		 */
1772 		*indx = 0;
1773 		*type = OTH;
1774 		fanout_unaligned++;
1775 		return (0);
1776 	}
1777 
1778 	/*
1779 	 * Extract next-header, full header length, and source-hash value
1780 	 * using v4/v6 specific fields.
1781 	 */
1782 	if (v6) {
1783 		remlen = ntohs(ip6h->ip6_plen);
1784 		nexthdr = ip6h->ip6_nxt;
1785 		src_val = V4_PART_OF_V6(ip6h->ip6_src);
1786 		dst_val = V4_PART_OF_V6(ip6h->ip6_dst);
1787 		/*
1788 		 * Do src based fanout if below tunable is set to B_TRUE or
1789 		 * when mac_ip_hdr_length_v6() fails because of malformed
1790 		 * packets or because mblks need to be concatenated using
1791 		 * pullupmsg().
1792 		 */
1793 		if (!mac_ip_hdr_length_v6(ip6h, mp->b_wptr, &hdr_len, &nexthdr,
1794 		    NULL)) {
1795 			goto src_dst_based_fanout;
1796 		}
1797 	} else {
1798 		hdr_len = IPH_HDR_LENGTH(ipha);
1799 		remlen = ntohs(ipha->ipha_length) - hdr_len;
1800 		nexthdr = ipha->ipha_protocol;
1801 		src_val = (uint32_t)ipha->ipha_src;
1802 		dst_val = (uint32_t)ipha->ipha_dst;
1803 		/*
1804 		 * Catch IPv4 fragment case here.  IPv6 has nexthdr == FRAG
1805 		 * for its equivalent case.
1806 		 */
1807 		if ((ntohs(ipha->ipha_fragment_offset_and_flags) &
1808 		    (IPH_MF | IPH_OFFSET)) != 0) {
1809 			goto src_dst_based_fanout;
1810 		}
1811 	}
1812 	if (remlen < MIN_EHDR_LEN)
1813 		return (-1);
1814 	whereptr = (uint8_t *)ip6h + hdr_len;
1815 
1816 	/* If the transport is one of below, we do port/SPI based fanout */
1817 	switch (nexthdr) {
1818 	case IPPROTO_TCP:
1819 	case IPPROTO_UDP:
1820 	case IPPROTO_SCTP:
1821 	case IPPROTO_ESP:
1822 		/*
1823 		 * If the ports or SPI in the transport header is not part of
1824 		 * the mblk, do src_based_fanout, instead of calling
1825 		 * pullupmsg().
1826 		 */
1827 		if (mp->b_cont == NULL || whereptr + PORTS_SIZE <= mp->b_wptr)
1828 			break;	/* out of switch... */
1829 		/* FALLTHRU */
1830 	default:
1831 		goto src_dst_based_fanout;
1832 	}
1833 
1834 	switch (nexthdr) {
1835 	case IPPROTO_TCP:
1836 		hash = HASH_ADDR(src_val, dst_val, *(uint32_t *)whereptr);
1837 		*indx = COMPUTE_INDEX(hash, mac_srs->srs_tcp_ring_count);
1838 		*type = OTH;
1839 		break;
1840 	case IPPROTO_UDP:
1841 	case IPPROTO_SCTP:
1842 	case IPPROTO_ESP:
1843 		if (mac_fanout_type == MAC_FANOUT_DEFAULT) {
1844 			hash = HASH_ADDR(src_val, dst_val,
1845 			    *(uint32_t *)whereptr);
1846 			*indx = COMPUTE_INDEX(hash,
1847 			    mac_srs->srs_udp_ring_count);
1848 		} else {
1849 			*indx = mac_srs->srs_ind % mac_srs->srs_udp_ring_count;
1850 			mac_srs->srs_ind++;
1851 		}
1852 		*type = OTH;
1853 		break;
1854 	}
1855 	return (0);
1856 
1857 src_dst_based_fanout:
1858 	hash = HASH_ADDR(src_val, dst_val, (uint32_t)0);
1859 	*indx = COMPUTE_INDEX(hash, mac_srs->srs_oth_ring_count);
1860 	*type = OTH;
1861 	return (0);
1862 }
1863 
1864 /*
1865  * mac_rx_srs_fanout
1866  *
1867  * This routine delivers packets destined to an SRS into a soft ring member
1868  * of the set.
1869  *
1870  * Given a chain of packets we need to split it up into multiple sub chains
1871  * destined for one of the TCP, UDP or OTH soft rings. Instead of entering
1872  * the soft ring one packet at a time, we want to enter it in the form of a
1873  * chain otherwise we get this start/stop behaviour where the worker thread
1874  * goes to sleep and then next packets comes in forcing it to wake up etc.
1875  *
1876  * Note:
1877  * Since we know what is the maximum fanout possible, we create a 2D array
1878  * of 'softring types * MAX_SR_FANOUT' for the head, tail, cnt and sz
1879  * variables so that we can enter the softrings with chain. We need the
1880  * MAX_SR_FANOUT so we can allocate the arrays on the stack (a kmem_alloc
1881  * for each packet would be expensive). If we ever want to have the
1882  * ability to have unlimited fanout, we should probably declare a head,
1883  * tail, cnt, sz with each soft ring (a data struct which contains a softring
1884  * along with these members) and create an array of this uber struct so we
1885  * don't have to do kmem_alloc.
1886  */
1887 int	fanout_oth1 = 0;
1888 int	fanout_oth2 = 0;
1889 int	fanout_oth3 = 0;
1890 int	fanout_oth4 = 0;
1891 int	fanout_oth5 = 0;
1892 
1893 static void
mac_rx_srs_fanout(mac_soft_ring_set_t * mac_srs,mblk_t * head)1894 mac_rx_srs_fanout(mac_soft_ring_set_t *mac_srs, mblk_t *head)
1895 {
1896 	struct ether_header		*ehp;
1897 	struct ether_vlan_header	*evhp;
1898 	uint32_t			sap;
1899 	ipha_t				*ipha;
1900 	uint8_t				*dstaddr;
1901 	uint_t				indx;
1902 	size_t				ports_offset;
1903 	size_t				ipha_len;
1904 	size_t				hdrsize;
1905 	uint_t				hash;
1906 	mblk_t				*mp;
1907 	mblk_t				*headmp[MAX_SR_TYPES][MAX_SR_FANOUT];
1908 	mblk_t				*tailmp[MAX_SR_TYPES][MAX_SR_FANOUT];
1909 	int				cnt[MAX_SR_TYPES][MAX_SR_FANOUT];
1910 	size_t				sz[MAX_SR_TYPES][MAX_SR_FANOUT];
1911 	size_t				sz1;
1912 	boolean_t			bw_ctl;
1913 	boolean_t			hw_classified;
1914 	boolean_t			dls_bypass;
1915 	boolean_t			is_ether;
1916 	boolean_t			is_unicast;
1917 	int				fanout_cnt;
1918 	enum pkt_type			type;
1919 	mac_client_impl_t		*mcip = mac_srs->srs_mcip;
1920 
1921 	is_ether = (mcip->mci_mip->mi_info.mi_nativemedia == DL_ETHER);
1922 	bw_ctl = ((mac_srs->srs_type & SRST_BW_CONTROL) != 0);
1923 
1924 	/*
1925 	 * If we don't have a Rx ring, S/W classification would have done
1926 	 * its job and its a packet meant for us. If we were polling on
1927 	 * the default ring (i.e. there was a ring assigned to this SRS),
1928 	 * then we need to make sure that the mac address really belongs
1929 	 * to us.
1930 	 */
1931 	hw_classified = mac_srs->srs_ring != NULL &&
1932 	    mac_srs->srs_ring->mr_classify_type == MAC_HW_CLASSIFIER;
1933 
1934 	/*
1935 	 * Special clients (eg. VLAN, non ether, etc) need DLS
1936 	 * processing in the Rx path. SRST_DLS_BYPASS will be clear for
1937 	 * such SRSs. Another way of disabling bypass is to set the
1938 	 * MCIS_RX_BYPASS_DISABLE flag.
1939 	 */
1940 	dls_bypass = ((mac_srs->srs_type & SRST_DLS_BYPASS) != 0) &&
1941 	    ((mcip->mci_state_flags & MCIS_RX_BYPASS_DISABLE) == 0);
1942 
1943 	/*
1944 	 * Since the softrings are never destroyed and we always
1945 	 * create equal number of softrings for TCP, UDP and rest,
1946 	 * its OK to check one of them for count and use it without
1947 	 * any lock. In future, if soft rings get destroyed because
1948 	 * of reduction in fanout, we will need to ensure that happens
1949 	 * behind the SRS_PROC.
1950 	 */
1951 	fanout_cnt = mac_srs->srs_tcp_ring_count;
1952 
1953 	bzero(headmp, MAX_SR_TYPES * MAX_SR_FANOUT * sizeof (mblk_t *));
1954 	bzero(tailmp, MAX_SR_TYPES * MAX_SR_FANOUT * sizeof (mblk_t *));
1955 	bzero(cnt, MAX_SR_TYPES * MAX_SR_FANOUT * sizeof (int));
1956 	bzero(sz, MAX_SR_TYPES * MAX_SR_FANOUT * sizeof (size_t));
1957 
1958 	/*
1959 	 * We got a chain from SRS that we need to send to the soft rings.
1960 	 * Since squeues for TCP & IPv4 sap poll their soft rings (for
1961 	 * performance reasons), we need to separate out v4_tcp, v4_udp
1962 	 * and the rest goes in other.
1963 	 */
1964 	while (head != NULL) {
1965 		mp = head;
1966 		head = head->b_next;
1967 		mp->b_next = NULL;
1968 
1969 		type = OTH;
1970 		sz1 = (mp->b_cont == NULL) ? MBLKL(mp) : msgdsize(mp);
1971 
1972 		if (is_ether) {
1973 			/*
1974 			 * At this point we can be sure the packet at least
1975 			 * has an ether header.
1976 			 */
1977 			if (sz1 < sizeof (struct ether_header)) {
1978 				mac_rx_drop_pkt(mac_srs, mp);
1979 				continue;
1980 			}
1981 			ehp = (struct ether_header *)mp->b_rptr;
1982 
1983 			/*
1984 			 * Determine if this is a VLAN or non-VLAN packet.
1985 			 */
1986 			if ((sap = ntohs(ehp->ether_type)) == VLAN_TPID) {
1987 				evhp = (struct ether_vlan_header *)mp->b_rptr;
1988 				sap = ntohs(evhp->ether_type);
1989 				hdrsize = sizeof (struct ether_vlan_header);
1990 				/*
1991 				 * Check if the VID of the packet, if any,
1992 				 * belongs to this client.
1993 				 */
1994 				if (!mac_client_check_flow_vid(mcip,
1995 				    VLAN_ID(ntohs(evhp->ether_tci)))) {
1996 					mac_rx_drop_pkt(mac_srs, mp);
1997 					continue;
1998 				}
1999 			} else {
2000 				hdrsize = sizeof (struct ether_header);
2001 			}
2002 			is_unicast =
2003 			    ((((uint8_t *)&ehp->ether_dhost)[0] & 0x01) == 0);
2004 			dstaddr = (uint8_t *)&ehp->ether_dhost;
2005 		} else {
2006 			mac_header_info_t		mhi;
2007 
2008 			if (mac_header_info((mac_handle_t)mcip->mci_mip,
2009 			    mp, &mhi) != 0) {
2010 				mac_rx_drop_pkt(mac_srs, mp);
2011 				continue;
2012 			}
2013 			hdrsize = mhi.mhi_hdrsize;
2014 			sap = mhi.mhi_bindsap;
2015 			is_unicast = (mhi.mhi_dsttype == MAC_ADDRTYPE_UNICAST);
2016 			dstaddr = (uint8_t *)mhi.mhi_daddr;
2017 		}
2018 
2019 		if (!dls_bypass) {
2020 			if (mac_rx_srs_long_fanout(mac_srs, mp, sap,
2021 			    hdrsize, &type, &indx) == -1) {
2022 				mac_rx_drop_pkt(mac_srs, mp);
2023 				continue;
2024 			}
2025 
2026 			FANOUT_ENQUEUE_MP(headmp[type][indx],
2027 			    tailmp[type][indx], cnt[type][indx], bw_ctl,
2028 			    sz[type][indx], sz1, mp);
2029 			continue;
2030 		}
2031 
2032 
2033 		/*
2034 		 * If we are using the default Rx ring where H/W or S/W
2035 		 * classification has not happened, we need to verify if
2036 		 * this unicast packet really belongs to us.
2037 		 */
2038 		if (sap == ETHERTYPE_IP) {
2039 			/*
2040 			 * If we are H/W classified, but we have promisc
2041 			 * on, then we need to check for the unicast address.
2042 			 */
2043 			if (hw_classified && mcip->mci_promisc_list != NULL) {
2044 				mac_address_t		*map;
2045 
2046 				rw_enter(&mcip->mci_rw_lock, RW_READER);
2047 				map = mcip->mci_unicast;
2048 				if (bcmp(dstaddr, map->ma_addr,
2049 				    map->ma_len) == 0)
2050 					type = UNDEF;
2051 				rw_exit(&mcip->mci_rw_lock);
2052 			} else if (is_unicast) {
2053 				type = UNDEF;
2054 			}
2055 		}
2056 
2057 		/*
2058 		 * This needs to become a contract with the driver for
2059 		 * the fast path.
2060 		 */
2061 
2062 		ipha = (ipha_t *)(mp->b_rptr + hdrsize);
2063 		if ((type != OTH) && MBLK_RX_FANOUT_SLOWPATH(mp, ipha)) {
2064 			type = OTH;
2065 			fanout_oth1++;
2066 		}
2067 
2068 		if (type != OTH) {
2069 			uint16_t	frag_offset_flags;
2070 
2071 			switch (ipha->ipha_protocol) {
2072 			case IPPROTO_TCP:
2073 			case IPPROTO_UDP:
2074 			case IPPROTO_SCTP:
2075 			case IPPROTO_ESP:
2076 				ipha_len = IPH_HDR_LENGTH(ipha);
2077 				if ((uchar_t *)ipha + ipha_len + PORTS_SIZE >
2078 				    mp->b_wptr) {
2079 					type = OTH;
2080 					break;
2081 				}
2082 				frag_offset_flags =
2083 				    ntohs(ipha->ipha_fragment_offset_and_flags);
2084 				if ((frag_offset_flags &
2085 				    (IPH_MF | IPH_OFFSET)) != 0) {
2086 					type = OTH;
2087 					fanout_oth3++;
2088 					break;
2089 				}
2090 				ports_offset = hdrsize + ipha_len;
2091 				break;
2092 			default:
2093 				type = OTH;
2094 				fanout_oth4++;
2095 				break;
2096 			}
2097 		}
2098 
2099 		if (type == OTH) {
2100 			if (mac_rx_srs_long_fanout(mac_srs, mp, sap,
2101 			    hdrsize, &type, &indx) == -1) {
2102 				mac_rx_drop_pkt(mac_srs, mp);
2103 				continue;
2104 			}
2105 
2106 			FANOUT_ENQUEUE_MP(headmp[type][indx],
2107 			    tailmp[type][indx], cnt[type][indx], bw_ctl,
2108 			    sz[type][indx], sz1, mp);
2109 			continue;
2110 		}
2111 
2112 		ASSERT(type == UNDEF);
2113 
2114 		/*
2115 		 * XXX-Sunay: We should hold srs_lock since ring_count
2116 		 * below can change. But if we are always called from
2117 		 * mac_rx_srs_drain and SRS_PROC is set, then we can
2118 		 * enforce that ring_count can't be changed i.e.
2119 		 * to change fanout type or ring count, the calling
2120 		 * thread needs to be behind SRS_PROC.
2121 		 */
2122 		switch (ipha->ipha_protocol) {
2123 		case IPPROTO_TCP:
2124 			/*
2125 			 * Note that for ESP, we fanout on SPI and it is at the
2126 			 * same offset as the 2x16-bit ports. So it is clumped
2127 			 * along with TCP, UDP and SCTP.
2128 			 */
2129 			hash = HASH_ADDR(ipha->ipha_src, ipha->ipha_dst,
2130 			    *(uint32_t *)(mp->b_rptr + ports_offset));
2131 			indx = COMPUTE_INDEX(hash, mac_srs->srs_tcp_ring_count);
2132 			type = V4_TCP;
2133 			mp->b_rptr += hdrsize;
2134 			break;
2135 		case IPPROTO_UDP:
2136 		case IPPROTO_SCTP:
2137 		case IPPROTO_ESP:
2138 			if (mac_fanout_type == MAC_FANOUT_DEFAULT) {
2139 				hash = HASH_ADDR(ipha->ipha_src, ipha->ipha_dst,
2140 				    *(uint32_t *)(mp->b_rptr + ports_offset));
2141 				indx = COMPUTE_INDEX(hash,
2142 				    mac_srs->srs_udp_ring_count);
2143 			} else {
2144 				indx = mac_srs->srs_ind %
2145 				    mac_srs->srs_udp_ring_count;
2146 				mac_srs->srs_ind++;
2147 			}
2148 			type = V4_UDP;
2149 			mp->b_rptr += hdrsize;
2150 			break;
2151 		default:
2152 			indx = 0;
2153 			type = OTH;
2154 		}
2155 
2156 		FANOUT_ENQUEUE_MP(headmp[type][indx], tailmp[type][indx],
2157 		    cnt[type][indx], bw_ctl, sz[type][indx], sz1, mp);
2158 	}
2159 
2160 	for (type = V4_TCP; type < UNDEF; type++) {
2161 		int	i;
2162 
2163 		for (i = 0; i < fanout_cnt; i++) {
2164 			if (headmp[type][i] != NULL) {
2165 				mac_soft_ring_t	*softring;
2166 
2167 				ASSERT(tailmp[type][i]->b_next == NULL);
2168 				switch (type) {
2169 				case V4_TCP:
2170 					softring =
2171 					    mac_srs->srs_tcp_soft_rings[i];
2172 					break;
2173 				case V4_UDP:
2174 					softring =
2175 					    mac_srs->srs_udp_soft_rings[i];
2176 					break;
2177 				case OTH:
2178 					softring =
2179 					    mac_srs->srs_oth_soft_rings[i];
2180 					break;
2181 				}
2182 				mac_rx_soft_ring_process(mcip,
2183 				    softring, headmp[type][i], tailmp[type][i],
2184 				    cnt[type][i], sz[type][i]);
2185 			}
2186 		}
2187 	}
2188 }
2189 
2190 #define	SRS_BYTES_TO_PICKUP	150000
2191 ssize_t	max_bytes_to_pickup = SRS_BYTES_TO_PICKUP;
2192 
2193 /*
2194  * mac_rx_srs_poll_ring
2195  *
2196  * This SRS Poll thread uses this routine to poll the underlying hardware
2197  * Rx ring to get a chain of packets. It can inline process that chain
2198  * if mac_latency_optimize is set (default) or signal the SRS worker thread
2199  * to do the remaining processing.
2200  *
2201  * Since packets come in the system via interrupt or poll path, we also
2202  * update the stats and deal with promiscous clients here.
2203  */
2204 void
mac_rx_srs_poll_ring(mac_soft_ring_set_t * mac_srs)2205 mac_rx_srs_poll_ring(mac_soft_ring_set_t *mac_srs)
2206 {
2207 	kmutex_t 		*lock = &mac_srs->srs_lock;
2208 	kcondvar_t 		*async = &mac_srs->srs_cv;
2209 	mac_srs_rx_t		*srs_rx = &mac_srs->srs_rx;
2210 	mblk_t 			*head, *tail, *mp;
2211 	callb_cpr_t 		cprinfo;
2212 	ssize_t 		bytes_to_pickup;
2213 	size_t 			sz;
2214 	int			count;
2215 	mac_client_impl_t	*smcip;
2216 
2217 	CALLB_CPR_INIT(&cprinfo, lock, callb_generic_cpr, "mac_srs_poll");
2218 	mutex_enter(lock);
2219 
2220 start:
2221 	for (;;) {
2222 		if (mac_srs->srs_state & SRS_PAUSE)
2223 			goto done;
2224 
2225 		CALLB_CPR_SAFE_BEGIN(&cprinfo);
2226 		cv_wait(async, lock);
2227 		CALLB_CPR_SAFE_END(&cprinfo, lock);
2228 
2229 		if (mac_srs->srs_state & SRS_PAUSE)
2230 			goto done;
2231 
2232 check_again:
2233 		if (mac_srs->srs_type & SRST_BW_CONTROL) {
2234 			/*
2235 			 * We pick as many bytes as we are allowed to queue.
2236 			 * Its possible that we will exceed the total
2237 			 * packets queued in case this SRS is part of the
2238 			 * Rx ring group since > 1 poll thread can be pulling
2239 			 * upto the max allowed packets at the same time
2240 			 * but that should be OK.
2241 			 */
2242 			mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
2243 			bytes_to_pickup =
2244 			    mac_srs->srs_bw->mac_bw_drop_threshold -
2245 			    mac_srs->srs_bw->mac_bw_sz;
2246 			/*
2247 			 * We shouldn't have been signalled if we
2248 			 * have 0 or less bytes to pick but since
2249 			 * some of the bytes accounting is driver
2250 			 * dependant, we do the safety check.
2251 			 */
2252 			if (bytes_to_pickup < 0)
2253 				bytes_to_pickup = 0;
2254 			mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2255 		} else {
2256 			/*
2257 			 * ToDO: Need to change the polling API
2258 			 * to add a packet count and a flag which
2259 			 * tells the driver whether we want packets
2260 			 * based on a count, or bytes, or all the
2261 			 * packets queued in the driver/HW. This
2262 			 * way, we never have to check the limits
2263 			 * on poll path. We truly let only as many
2264 			 * packets enter the system as we are willing
2265 			 * to process or queue.
2266 			 *
2267 			 * Something along the lines of
2268 			 * pkts_to_pickup = mac_soft_ring_max_q_cnt -
2269 			 *	mac_srs->srs_poll_pkt_cnt
2270 			 */
2271 
2272 			/*
2273 			 * Since we are not doing B/W control, pick
2274 			 * as many packets as allowed.
2275 			 */
2276 			bytes_to_pickup = max_bytes_to_pickup;
2277 		}
2278 
2279 		/* Poll the underlying Hardware */
2280 		mutex_exit(lock);
2281 		head = MAC_HWRING_POLL(mac_srs->srs_ring, (int)bytes_to_pickup);
2282 		mutex_enter(lock);
2283 
2284 		ASSERT((mac_srs->srs_state & SRS_POLL_THR_OWNER) ==
2285 		    SRS_POLL_THR_OWNER);
2286 
2287 		mp = tail = head;
2288 		count = 0;
2289 		sz = 0;
2290 		while (mp != NULL) {
2291 			tail = mp;
2292 			sz += msgdsize(mp);
2293 			mp = mp->b_next;
2294 			count++;
2295 		}
2296 
2297 		if (head != NULL) {
2298 			tail->b_next = NULL;
2299 			smcip = mac_srs->srs_mcip;
2300 
2301 			SRS_RX_STAT_UPDATE(mac_srs, pollbytes, sz);
2302 			SRS_RX_STAT_UPDATE(mac_srs, pollcnt, count);
2303 
2304 			/*
2305 			 * If there are any promiscuous mode callbacks
2306 			 * defined for this MAC client, pass them a copy
2307 			 * if appropriate and also update the counters.
2308 			 */
2309 			if (smcip != NULL) {
2310 				if (smcip->mci_mip->mi_promisc_list != NULL) {
2311 					mutex_exit(lock);
2312 					mac_promisc_dispatch(smcip->mci_mip,
2313 					    head, NULL);
2314 					mutex_enter(lock);
2315 				}
2316 			}
2317 			if (mac_srs->srs_type & SRST_BW_CONTROL) {
2318 				mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
2319 				mac_srs->srs_bw->mac_bw_polled += sz;
2320 				mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2321 			}
2322 			MAC_RX_SRS_ENQUEUE_CHAIN(mac_srs, head, tail,
2323 			    count, sz);
2324 			if (count <= 10)
2325 				srs_rx->sr_stat.mrs_chaincntundr10++;
2326 			else if (count > 10 && count <= 50)
2327 				srs_rx->sr_stat.mrs_chaincnt10to50++;
2328 			else
2329 				srs_rx->sr_stat.mrs_chaincntover50++;
2330 		}
2331 
2332 		/*
2333 		 * We are guaranteed that SRS_PROC will be set if we
2334 		 * are here. Also, poll thread gets to run only if
2335 		 * the drain was being done by a worker thread although
2336 		 * its possible that worker thread is still running
2337 		 * and poll thread was sent down to keep the pipeline
2338 		 * going instead of doing a complete drain and then
2339 		 * trying to poll the NIC.
2340 		 *
2341 		 * So we need to check SRS_WORKER flag to make sure
2342 		 * that the worker thread is not processing the queue
2343 		 * in parallel to us. The flags and conditions are
2344 		 * protected by the srs_lock to prevent any race. We
2345 		 * ensure that we don't drop the srs_lock from now
2346 		 * till the end and similarly we don't drop the srs_lock
2347 		 * in mac_rx_srs_drain() till similar condition check
2348 		 * are complete. The mac_rx_srs_drain() needs to ensure
2349 		 * that SRS_WORKER flag remains set as long as its
2350 		 * processing the queue.
2351 		 */
2352 		if (!(mac_srs->srs_state & SRS_WORKER) &&
2353 		    (mac_srs->srs_first != NULL)) {
2354 			/*
2355 			 * We have packets to process and worker thread
2356 			 * is not running. Check to see if poll thread is
2357 			 * allowed to process.
2358 			 */
2359 			if (mac_srs->srs_state & SRS_LATENCY_OPT) {
2360 				mac_srs->srs_drain_func(mac_srs, SRS_POLL_PROC);
2361 				if (!(mac_srs->srs_state & SRS_PAUSE) &&
2362 				    srs_rx->sr_poll_pkt_cnt <=
2363 				    srs_rx->sr_lowat) {
2364 					srs_rx->sr_poll_again++;
2365 					goto check_again;
2366 				}
2367 				/*
2368 				 * We are already above low water mark
2369 				 * so stay in the polling mode but no
2370 				 * need to poll. Once we dip below
2371 				 * the polling threshold, the processing
2372 				 * thread (soft ring) will signal us
2373 				 * to poll again (MAC_UPDATE_SRS_COUNT)
2374 				 */
2375 				srs_rx->sr_poll_drain_no_poll++;
2376 				mac_srs->srs_state &= ~(SRS_PROC|SRS_GET_PKTS);
2377 				/*
2378 				 * In B/W control case, its possible
2379 				 * that the backlog built up due to
2380 				 * B/W limit being reached and packets
2381 				 * are queued only in SRS. In this case,
2382 				 * we should schedule worker thread
2383 				 * since no one else will wake us up.
2384 				 */
2385 				if ((mac_srs->srs_type & SRST_BW_CONTROL) &&
2386 				    (mac_srs->srs_tid == NULL)) {
2387 					mac_srs->srs_tid =
2388 					    timeout(mac_srs_fire, mac_srs, 1);
2389 					srs_rx->sr_poll_worker_wakeup++;
2390 				}
2391 			} else {
2392 				/*
2393 				 * Wakeup the worker thread for more processing.
2394 				 * We optimize for throughput in this case.
2395 				 */
2396 				mac_srs->srs_state &= ~(SRS_PROC|SRS_GET_PKTS);
2397 				MAC_SRS_WORKER_WAKEUP(mac_srs);
2398 				srs_rx->sr_poll_sig_worker++;
2399 			}
2400 		} else if ((mac_srs->srs_first == NULL) &&
2401 		    !(mac_srs->srs_state & SRS_WORKER)) {
2402 			/*
2403 			 * There is nothing queued in SRS and
2404 			 * no worker thread running. Plus we
2405 			 * didn't get anything from the H/W
2406 			 * as well (head == NULL);
2407 			 */
2408 			ASSERT(head == NULL);
2409 			mac_srs->srs_state &=
2410 			    ~(SRS_PROC|SRS_GET_PKTS);
2411 
2412 			/*
2413 			 * If we have a packets in soft ring, don't allow
2414 			 * more packets to come into this SRS by keeping the
2415 			 * interrupts off but not polling the H/W. The
2416 			 * poll thread will get signaled as soon as
2417 			 * srs_poll_pkt_cnt dips below poll threshold.
2418 			 */
2419 			if (srs_rx->sr_poll_pkt_cnt == 0) {
2420 				srs_rx->sr_poll_intr_enable++;
2421 				MAC_SRS_POLLING_OFF(mac_srs);
2422 			} else {
2423 				/*
2424 				 * We know nothing is queued in SRS
2425 				 * since we are here after checking
2426 				 * srs_first is NULL. The backlog
2427 				 * is entirely due to packets queued
2428 				 * in Soft ring which will wake us up
2429 				 * and get the interface out of polling
2430 				 * mode once the backlog dips below
2431 				 * sr_poll_thres.
2432 				 */
2433 				srs_rx->sr_poll_no_poll++;
2434 			}
2435 		} else {
2436 			/*
2437 			 * Worker thread is already running.
2438 			 * Nothing much to do. If the polling
2439 			 * was enabled, worker thread will deal
2440 			 * with that.
2441 			 */
2442 			mac_srs->srs_state &= ~SRS_GET_PKTS;
2443 			srs_rx->sr_poll_goto_sleep++;
2444 		}
2445 	}
2446 done:
2447 	mac_srs->srs_state |= SRS_POLL_THR_QUIESCED;
2448 	cv_signal(&mac_srs->srs_async);
2449 	/*
2450 	 * If this is a temporary quiesce then wait for the restart signal
2451 	 * from the srs worker. Then clear the flags and signal the srs worker
2452 	 * to ensure a positive handshake and go back to start.
2453 	 */
2454 	while (!(mac_srs->srs_state & (SRS_CONDEMNED | SRS_POLL_THR_RESTART)))
2455 		cv_wait(async, lock);
2456 	if (mac_srs->srs_state & SRS_POLL_THR_RESTART) {
2457 		ASSERT(!(mac_srs->srs_state & SRS_CONDEMNED));
2458 		mac_srs->srs_state &=
2459 		    ~(SRS_POLL_THR_QUIESCED | SRS_POLL_THR_RESTART);
2460 		cv_signal(&mac_srs->srs_async);
2461 		goto start;
2462 	} else {
2463 		mac_srs->srs_state |= SRS_POLL_THR_EXITED;
2464 		cv_signal(&mac_srs->srs_async);
2465 		CALLB_CPR_EXIT(&cprinfo);
2466 		thread_exit();
2467 	}
2468 }
2469 
2470 /*
2471  * mac_srs_pick_chain
2472  *
2473  * In Bandwidth control case, checks how many packets can be processed
2474  * and return them in a sub chain.
2475  */
2476 static mblk_t *
mac_srs_pick_chain(mac_soft_ring_set_t * mac_srs,mblk_t ** chain_tail,size_t * chain_sz,int * chain_cnt)2477 mac_srs_pick_chain(mac_soft_ring_set_t *mac_srs, mblk_t **chain_tail,
2478     size_t *chain_sz, int *chain_cnt)
2479 {
2480 	mblk_t 			*head = NULL;
2481 	mblk_t 			*tail = NULL;
2482 	size_t			sz;
2483 	size_t 			tsz = 0;
2484 	int			cnt = 0;
2485 	mblk_t 			*mp;
2486 
2487 	ASSERT(MUTEX_HELD(&mac_srs->srs_lock));
2488 	mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
2489 	if (((mac_srs->srs_bw->mac_bw_used + mac_srs->srs_size) <=
2490 	    mac_srs->srs_bw->mac_bw_limit) ||
2491 	    (mac_srs->srs_bw->mac_bw_limit == 0)) {
2492 		mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2493 		head = mac_srs->srs_first;
2494 		mac_srs->srs_first = NULL;
2495 		*chain_tail = mac_srs->srs_last;
2496 		mac_srs->srs_last = NULL;
2497 		*chain_sz = mac_srs->srs_size;
2498 		*chain_cnt = mac_srs->srs_count;
2499 		mac_srs->srs_count = 0;
2500 		mac_srs->srs_size = 0;
2501 		return (head);
2502 	}
2503 
2504 	/*
2505 	 * Can't clear the entire backlog.
2506 	 * Need to find how many packets to pick
2507 	 */
2508 	ASSERT(MUTEX_HELD(&mac_srs->srs_bw->mac_bw_lock));
2509 	while ((mp = mac_srs->srs_first) != NULL) {
2510 		sz = msgdsize(mp);
2511 		if ((tsz + sz + mac_srs->srs_bw->mac_bw_used) >
2512 		    mac_srs->srs_bw->mac_bw_limit) {
2513 			if (!(mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED))
2514 				mac_srs->srs_bw->mac_bw_state |=
2515 				    SRS_BW_ENFORCED;
2516 			break;
2517 		}
2518 
2519 		/*
2520 		 * The _size & cnt is  decremented from the softrings
2521 		 * when they send up the packet for polling to work
2522 		 * properly.
2523 		 */
2524 		tsz += sz;
2525 		cnt++;
2526 		mac_srs->srs_count--;
2527 		mac_srs->srs_size -= sz;
2528 		if (tail != NULL)
2529 			tail->b_next = mp;
2530 		else
2531 			head = mp;
2532 		tail = mp;
2533 		mac_srs->srs_first = mac_srs->srs_first->b_next;
2534 	}
2535 	mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2536 	if (mac_srs->srs_first == NULL)
2537 		mac_srs->srs_last = NULL;
2538 
2539 	if (tail != NULL)
2540 		tail->b_next = NULL;
2541 	*chain_tail = tail;
2542 	*chain_cnt = cnt;
2543 	*chain_sz = tsz;
2544 
2545 	return (head);
2546 }
2547 
2548 /*
2549  * mac_rx_srs_drain
2550  *
2551  * The SRS drain routine. Gets to run to clear the queue. Any thread
2552  * (worker, interrupt, poll) can call this based on processing model.
2553  * The first thing we do is disable interrupts if possible and then
2554  * drain the queue. we also try to poll the underlying hardware if
2555  * there is a dedicated hardware Rx ring assigned to this SRS.
2556  *
2557  * There is a equivalent drain routine in bandwidth control mode
2558  * mac_rx_srs_drain_bw. There is some code duplication between the two
2559  * routines but they are highly performance sensitive and are easier
2560  * to read/debug if they stay separate. Any code changes here might
2561  * also apply to mac_rx_srs_drain_bw as well.
2562  */
2563 void
mac_rx_srs_drain(mac_soft_ring_set_t * mac_srs,uint_t proc_type)2564 mac_rx_srs_drain(mac_soft_ring_set_t *mac_srs, uint_t proc_type)
2565 {
2566 	mblk_t 			*head;
2567 	mblk_t			*tail;
2568 	timeout_id_t 		tid;
2569 	int			cnt = 0;
2570 	mac_client_impl_t	*mcip = mac_srs->srs_mcip;
2571 	mac_srs_rx_t		*srs_rx = &mac_srs->srs_rx;
2572 
2573 	ASSERT(MUTEX_HELD(&mac_srs->srs_lock));
2574 	ASSERT(!(mac_srs->srs_type & SRST_BW_CONTROL));
2575 
2576 	/* If we are blanked i.e. can't do upcalls, then we are done */
2577 	if (mac_srs->srs_state & (SRS_BLANK | SRS_PAUSE)) {
2578 		ASSERT((mac_srs->srs_type & SRST_NO_SOFT_RINGS) ||
2579 		    (mac_srs->srs_state & SRS_PAUSE));
2580 		goto out;
2581 	}
2582 
2583 	if (mac_srs->srs_first == NULL)
2584 		goto out;
2585 
2586 	if (!(mac_srs->srs_state & SRS_LATENCY_OPT) &&
2587 	    (srs_rx->sr_poll_pkt_cnt <= srs_rx->sr_lowat)) {
2588 		/*
2589 		 * In the normal case, the SRS worker thread does no
2590 		 * work and we wait for a backlog to build up before
2591 		 * we switch into polling mode. In case we are
2592 		 * optimizing for throughput, we use the worker thread
2593 		 * as well. The goal is to let worker thread process
2594 		 * the queue and poll thread to feed packets into
2595 		 * the queue. As such, we should signal the poll
2596 		 * thread to try and get more packets.
2597 		 *
2598 		 * We could have pulled this check in the POLL_RING
2599 		 * macro itself but keeping it explicit here makes
2600 		 * the architecture more human understandable.
2601 		 */
2602 		MAC_SRS_POLL_RING(mac_srs);
2603 	}
2604 
2605 again:
2606 	head = mac_srs->srs_first;
2607 	mac_srs->srs_first = NULL;
2608 	tail = mac_srs->srs_last;
2609 	mac_srs->srs_last = NULL;
2610 	cnt = mac_srs->srs_count;
2611 	mac_srs->srs_count = 0;
2612 
2613 	ASSERT(head != NULL);
2614 	ASSERT(tail != NULL);
2615 
2616 	if ((tid = mac_srs->srs_tid) != 0)
2617 		mac_srs->srs_tid = 0;
2618 
2619 	mac_srs->srs_state |= (SRS_PROC|proc_type);
2620 
2621 
2622 	/*
2623 	 * mcip is NULL for broadcast and multicast flows. The promisc
2624 	 * callbacks for broadcast and multicast packets are delivered from
2625 	 * mac_rx() and we don't need to worry about that case in this path
2626 	 */
2627 	if (mcip != NULL) {
2628 		if (mcip->mci_promisc_list != NULL) {
2629 			mutex_exit(&mac_srs->srs_lock);
2630 			mac_promisc_client_dispatch(mcip, head);
2631 			mutex_enter(&mac_srs->srs_lock);
2632 		}
2633 		if (MAC_PROTECT_ENABLED(mcip, MPT_IPNOSPOOF)) {
2634 			mutex_exit(&mac_srs->srs_lock);
2635 			mac_protect_intercept_dhcp(mcip, head);
2636 			mutex_enter(&mac_srs->srs_lock);
2637 		}
2638 	}
2639 
2640 	/*
2641 	 * Check if SRS itself is doing the processing
2642 	 * This direct path does not apply when subflows are present. In this
2643 	 * case, packets need to be dispatched to a soft ring according to the
2644 	 * flow's bandwidth and other resources contraints.
2645 	 */
2646 	if (mac_srs->srs_type & SRST_NO_SOFT_RINGS) {
2647 		mac_direct_rx_t		proc;
2648 		void			*arg1;
2649 		mac_resource_handle_t	arg2;
2650 
2651 		/*
2652 		 * This is the case when a Rx is directly
2653 		 * assigned and we have a fully classified
2654 		 * protocol chain. We can deal with it in
2655 		 * one shot.
2656 		 */
2657 		proc = srs_rx->sr_func;
2658 		arg1 = srs_rx->sr_arg1;
2659 		arg2 = srs_rx->sr_arg2;
2660 
2661 		mac_srs->srs_state |= SRS_CLIENT_PROC;
2662 		mutex_exit(&mac_srs->srs_lock);
2663 		if (tid != 0) {
2664 			(void) untimeout(tid);
2665 			tid = 0;
2666 		}
2667 
2668 		proc(arg1, arg2, head, NULL);
2669 		/*
2670 		 * Decrement the size and count here itelf
2671 		 * since the packet has been processed.
2672 		 */
2673 		mutex_enter(&mac_srs->srs_lock);
2674 		MAC_UPDATE_SRS_COUNT_LOCKED(mac_srs, cnt);
2675 		if (mac_srs->srs_state & SRS_CLIENT_WAIT)
2676 			cv_signal(&mac_srs->srs_client_cv);
2677 		mac_srs->srs_state &= ~SRS_CLIENT_PROC;
2678 	} else {
2679 		/* Some kind of softrings based fanout is required */
2680 		mutex_exit(&mac_srs->srs_lock);
2681 		if (tid != 0) {
2682 			(void) untimeout(tid);
2683 			tid = 0;
2684 		}
2685 
2686 		/*
2687 		 * Since the fanout routines can deal with chains,
2688 		 * shoot the entire chain up.
2689 		 */
2690 		if (mac_srs->srs_type & SRST_FANOUT_SRC_IP)
2691 			mac_rx_srs_fanout(mac_srs, head);
2692 		else
2693 			mac_rx_srs_proto_fanout(mac_srs, head);
2694 		mutex_enter(&mac_srs->srs_lock);
2695 	}
2696 
2697 	if (!(mac_srs->srs_state & (SRS_BLANK|SRS_PAUSE)) &&
2698 	    (mac_srs->srs_first != NULL)) {
2699 		/*
2700 		 * More packets arrived while we were clearing the
2701 		 * SRS. This can be possible because of one of
2702 		 * three conditions below:
2703 		 * 1) The driver is using multiple worker threads
2704 		 *    to send the packets to us.
2705 		 * 2) The driver has a race in switching
2706 		 *    between interrupt and polling mode or
2707 		 * 3) Packets are arriving in this SRS via the
2708 		 *    S/W classification as well.
2709 		 *
2710 		 * We should switch to polling mode and see if we
2711 		 * need to send the poll thread down. Also, signal
2712 		 * the worker thread to process whats just arrived.
2713 		 */
2714 		MAC_SRS_POLLING_ON(mac_srs);
2715 		if (srs_rx->sr_poll_pkt_cnt <= srs_rx->sr_lowat) {
2716 			srs_rx->sr_drain_poll_sig++;
2717 			MAC_SRS_POLL_RING(mac_srs);
2718 		}
2719 
2720 		/*
2721 		 * If we didn't signal the poll thread, we need
2722 		 * to deal with the pending packets ourselves.
2723 		 */
2724 		if (proc_type == SRS_WORKER) {
2725 			srs_rx->sr_drain_again++;
2726 			goto again;
2727 		} else {
2728 			srs_rx->sr_drain_worker_sig++;
2729 			cv_signal(&mac_srs->srs_async);
2730 		}
2731 	}
2732 
2733 out:
2734 	if (mac_srs->srs_state & SRS_GET_PKTS) {
2735 		/*
2736 		 * Poll thread is already running. Leave the
2737 		 * SRS_RPOC set and hand over the control to
2738 		 * poll thread.
2739 		 */
2740 		mac_srs->srs_state &= ~proc_type;
2741 		srs_rx->sr_drain_poll_running++;
2742 		return;
2743 	}
2744 
2745 	/*
2746 	 * Even if there are no packets queued in SRS, we
2747 	 * need to make sure that the shared counter is
2748 	 * clear and any associated softrings have cleared
2749 	 * all the backlog. Otherwise, leave the interface
2750 	 * in polling mode and the poll thread will get
2751 	 * signalled once the count goes down to zero.
2752 	 *
2753 	 * If someone is already draining the queue (SRS_PROC is
2754 	 * set) when the srs_poll_pkt_cnt goes down to zero,
2755 	 * then it means that drain is already running and we
2756 	 * will turn off polling at that time if there is
2757 	 * no backlog.
2758 	 *
2759 	 * As long as there are packets queued either
2760 	 * in soft ring set or its soft rings, we will leave
2761 	 * the interface in polling mode (even if the drain
2762 	 * was done being the interrupt thread). We signal
2763 	 * the poll thread as well if we have dipped below
2764 	 * low water mark.
2765 	 *
2766 	 * NOTE: We can't use the MAC_SRS_POLLING_ON macro
2767 	 * since that turn polling on only for worker thread.
2768 	 * Its not worth turning polling on for interrupt
2769 	 * thread (since NIC will not issue another interrupt)
2770 	 * unless a backlog builds up.
2771 	 */
2772 	if ((srs_rx->sr_poll_pkt_cnt > 0) &&
2773 	    (mac_srs->srs_state & SRS_POLLING_CAPAB)) {
2774 		mac_srs->srs_state &= ~(SRS_PROC|proc_type);
2775 		srs_rx->sr_drain_keep_polling++;
2776 		MAC_SRS_POLLING_ON(mac_srs);
2777 		if (srs_rx->sr_poll_pkt_cnt <= srs_rx->sr_lowat)
2778 			MAC_SRS_POLL_RING(mac_srs);
2779 		return;
2780 	}
2781 
2782 	/* Nothing else to do. Get out of poll mode */
2783 	MAC_SRS_POLLING_OFF(mac_srs);
2784 	mac_srs->srs_state &= ~(SRS_PROC|proc_type);
2785 	srs_rx->sr_drain_finish_intr++;
2786 }
2787 
2788 /*
2789  * mac_rx_srs_drain_bw
2790  *
2791  * The SRS BW drain routine. Gets to run to clear the queue. Any thread
2792  * (worker, interrupt, poll) can call this based on processing model.
2793  * The first thing we do is disable interrupts if possible and then
2794  * drain the queue. we also try to poll the underlying hardware if
2795  * there is a dedicated hardware Rx ring assigned to this SRS.
2796  *
2797  * There is a equivalent drain routine in non bandwidth control mode
2798  * mac_rx_srs_drain. There is some code duplication between the two
2799  * routines but they are highly performance sensitive and are easier
2800  * to read/debug if they stay separate. Any code changes here might
2801  * also apply to mac_rx_srs_drain as well.
2802  */
2803 void
mac_rx_srs_drain_bw(mac_soft_ring_set_t * mac_srs,uint_t proc_type)2804 mac_rx_srs_drain_bw(mac_soft_ring_set_t *mac_srs, uint_t proc_type)
2805 {
2806 	mblk_t 			*head;
2807 	mblk_t			*tail;
2808 	timeout_id_t 		tid;
2809 	size_t			sz = 0;
2810 	int			cnt = 0;
2811 	mac_client_impl_t	*mcip = mac_srs->srs_mcip;
2812 	mac_srs_rx_t		*srs_rx = &mac_srs->srs_rx;
2813 	clock_t			now;
2814 
2815 	ASSERT(MUTEX_HELD(&mac_srs->srs_lock));
2816 	ASSERT(mac_srs->srs_type & SRST_BW_CONTROL);
2817 again:
2818 	/* Check if we are doing B/W control */
2819 	mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
2820 	now = ddi_get_lbolt();
2821 	if (mac_srs->srs_bw->mac_bw_curr_time != now) {
2822 		mac_srs->srs_bw->mac_bw_curr_time = now;
2823 		mac_srs->srs_bw->mac_bw_used = 0;
2824 		if (mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED)
2825 			mac_srs->srs_bw->mac_bw_state &= ~SRS_BW_ENFORCED;
2826 	} else if (mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED) {
2827 		mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2828 		goto done;
2829 	} else if (mac_srs->srs_bw->mac_bw_used >
2830 	    mac_srs->srs_bw->mac_bw_limit) {
2831 		mac_srs->srs_bw->mac_bw_state |= SRS_BW_ENFORCED;
2832 		mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2833 		goto done;
2834 	}
2835 	mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2836 
2837 	/* If we are blanked i.e. can't do upcalls, then we are done */
2838 	if (mac_srs->srs_state & (SRS_BLANK | SRS_PAUSE)) {
2839 		ASSERT((mac_srs->srs_type & SRST_NO_SOFT_RINGS) ||
2840 		    (mac_srs->srs_state & SRS_PAUSE));
2841 		goto done;
2842 	}
2843 
2844 	sz = 0;
2845 	cnt = 0;
2846 	if ((head = mac_srs_pick_chain(mac_srs, &tail, &sz, &cnt)) == NULL) {
2847 		/*
2848 		 * We couldn't pick up a single packet.
2849 		 */
2850 		mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
2851 		if ((mac_srs->srs_bw->mac_bw_used == 0) &&
2852 		    (mac_srs->srs_size != 0) &&
2853 		    !(mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED)) {
2854 			/*
2855 			 * Seems like configured B/W doesn't
2856 			 * even allow processing of 1 packet
2857 			 * per tick.
2858 			 *
2859 			 * XXX: raise the limit to processing
2860 			 * at least 1 packet per tick.
2861 			 */
2862 			mac_srs->srs_bw->mac_bw_limit +=
2863 			    mac_srs->srs_bw->mac_bw_limit;
2864 			mac_srs->srs_bw->mac_bw_drop_threshold +=
2865 			    mac_srs->srs_bw->mac_bw_drop_threshold;
2866 			cmn_err(CE_NOTE, "mac_rx_srs_drain: srs(%p) "
2867 			    "raised B/W limit to %d since not even a "
2868 			    "single packet can be processed per "
2869 			    "tick %d\n", (void *)mac_srs,
2870 			    (int)mac_srs->srs_bw->mac_bw_limit,
2871 			    (int)msgdsize(mac_srs->srs_first));
2872 		}
2873 		mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2874 		goto done;
2875 	}
2876 
2877 	ASSERT(head != NULL);
2878 	ASSERT(tail != NULL);
2879 
2880 	/* zero bandwidth: drop all and return to interrupt mode */
2881 	mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
2882 	if (mac_srs->srs_bw->mac_bw_limit == 0) {
2883 		srs_rx->sr_stat.mrs_sdrops += cnt;
2884 		ASSERT(mac_srs->srs_bw->mac_bw_sz >= sz);
2885 		mac_srs->srs_bw->mac_bw_sz -= sz;
2886 		mac_srs->srs_bw->mac_bw_drop_bytes += sz;
2887 		mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2888 		mac_pkt_drop(NULL, NULL, head, B_FALSE);
2889 		goto leave_poll;
2890 	} else {
2891 		mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2892 	}
2893 
2894 	if ((tid = mac_srs->srs_tid) != 0)
2895 		mac_srs->srs_tid = 0;
2896 
2897 	mac_srs->srs_state |= (SRS_PROC|proc_type);
2898 	MAC_SRS_WORKER_POLLING_ON(mac_srs);
2899 
2900 	/*
2901 	 * mcip is NULL for broadcast and multicast flows. The promisc
2902 	 * callbacks for broadcast and multicast packets are delivered from
2903 	 * mac_rx() and we don't need to worry about that case in this path
2904 	 */
2905 	if (mcip != NULL) {
2906 		if (mcip->mci_promisc_list != NULL) {
2907 			mutex_exit(&mac_srs->srs_lock);
2908 			mac_promisc_client_dispatch(mcip, head);
2909 			mutex_enter(&mac_srs->srs_lock);
2910 		}
2911 		if (MAC_PROTECT_ENABLED(mcip, MPT_IPNOSPOOF)) {
2912 			mutex_exit(&mac_srs->srs_lock);
2913 			mac_protect_intercept_dhcp(mcip, head);
2914 			mutex_enter(&mac_srs->srs_lock);
2915 		}
2916 	}
2917 
2918 	/*
2919 	 * Check if SRS itself is doing the processing
2920 	 * This direct path does not apply when subflows are present. In this
2921 	 * case, packets need to be dispatched to a soft ring according to the
2922 	 * flow's bandwidth and other resources contraints.
2923 	 */
2924 	if (mac_srs->srs_type & SRST_NO_SOFT_RINGS) {
2925 		mac_direct_rx_t		proc;
2926 		void			*arg1;
2927 		mac_resource_handle_t	arg2;
2928 
2929 		/*
2930 		 * This is the case when a Rx is directly
2931 		 * assigned and we have a fully classified
2932 		 * protocol chain. We can deal with it in
2933 		 * one shot.
2934 		 */
2935 		proc = srs_rx->sr_func;
2936 		arg1 = srs_rx->sr_arg1;
2937 		arg2 = srs_rx->sr_arg2;
2938 
2939 		mac_srs->srs_state |= SRS_CLIENT_PROC;
2940 		mutex_exit(&mac_srs->srs_lock);
2941 		if (tid != 0) {
2942 			(void) untimeout(tid);
2943 			tid = 0;
2944 		}
2945 
2946 		proc(arg1, arg2, head, NULL);
2947 		/*
2948 		 * Decrement the size and count here itelf
2949 		 * since the packet has been processed.
2950 		 */
2951 		mutex_enter(&mac_srs->srs_lock);
2952 		MAC_UPDATE_SRS_COUNT_LOCKED(mac_srs, cnt);
2953 		MAC_UPDATE_SRS_SIZE_LOCKED(mac_srs, sz);
2954 
2955 		if (mac_srs->srs_state & SRS_CLIENT_WAIT)
2956 			cv_signal(&mac_srs->srs_client_cv);
2957 		mac_srs->srs_state &= ~SRS_CLIENT_PROC;
2958 	} else {
2959 		/* Some kind of softrings based fanout is required */
2960 		mutex_exit(&mac_srs->srs_lock);
2961 		if (tid != 0) {
2962 			(void) untimeout(tid);
2963 			tid = 0;
2964 		}
2965 
2966 		/*
2967 		 * Since the fanout routines can deal with chains,
2968 		 * shoot the entire chain up.
2969 		 */
2970 		if (mac_srs->srs_type & SRST_FANOUT_SRC_IP)
2971 			mac_rx_srs_fanout(mac_srs, head);
2972 		else
2973 			mac_rx_srs_proto_fanout(mac_srs, head);
2974 		mutex_enter(&mac_srs->srs_lock);
2975 	}
2976 
2977 	/*
2978 	 * Send the poll thread to pick up any packets arrived
2979 	 * so far. This also serves as the last check in case
2980 	 * nothing else is queued in the SRS. The poll thread
2981 	 * is signalled only in the case the drain was done
2982 	 * by the worker thread and SRS_WORKER is set. The
2983 	 * worker thread can run in parallel as long as the
2984 	 * SRS_WORKER flag is set. We we have nothing else to
2985 	 * process, we can exit while leaving SRS_PROC set
2986 	 * which gives the poll thread control to process and
2987 	 * cleanup once it returns from the NIC.
2988 	 *
2989 	 * If we have nothing else to process, we need to
2990 	 * ensure that we keep holding the srs_lock till
2991 	 * all the checks below are done and control is
2992 	 * handed to the poll thread if it was running.
2993 	 */
2994 	mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
2995 	if (!(mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED)) {
2996 		if (mac_srs->srs_first != NULL) {
2997 			if (proc_type == SRS_WORKER) {
2998 				mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
2999 				if (srs_rx->sr_poll_pkt_cnt <=
3000 				    srs_rx->sr_lowat)
3001 					MAC_SRS_POLL_RING(mac_srs);
3002 				goto again;
3003 			} else {
3004 				cv_signal(&mac_srs->srs_async);
3005 			}
3006 		}
3007 	}
3008 	mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
3009 
3010 done:
3011 
3012 	if (mac_srs->srs_state & SRS_GET_PKTS) {
3013 		/*
3014 		 * Poll thread is already running. Leave the
3015 		 * SRS_RPOC set and hand over the control to
3016 		 * poll thread.
3017 		 */
3018 		mac_srs->srs_state &= ~proc_type;
3019 		return;
3020 	}
3021 
3022 	/*
3023 	 * If we can't process packets because we have exceeded
3024 	 * B/W limit for this tick, just set the timeout
3025 	 * and leave.
3026 	 *
3027 	 * Even if there are no packets queued in SRS, we
3028 	 * need to make sure that the shared counter is
3029 	 * clear and any associated softrings have cleared
3030 	 * all the backlog. Otherwise, leave the interface
3031 	 * in polling mode and the poll thread will get
3032 	 * signalled once the count goes down to zero.
3033 	 *
3034 	 * If someone is already draining the queue (SRS_PROC is
3035 	 * set) when the srs_poll_pkt_cnt goes down to zero,
3036 	 * then it means that drain is already running and we
3037 	 * will turn off polling at that time if there is
3038 	 * no backlog. As long as there are packets queued either
3039 	 * is soft ring set or its soft rings, we will leave
3040 	 * the interface in polling mode.
3041 	 */
3042 	mutex_enter(&mac_srs->srs_bw->mac_bw_lock);
3043 	if ((mac_srs->srs_state & SRS_POLLING_CAPAB) &&
3044 	    ((mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED) ||
3045 	    (srs_rx->sr_poll_pkt_cnt > 0))) {
3046 		MAC_SRS_POLLING_ON(mac_srs);
3047 		mac_srs->srs_state &= ~(SRS_PROC|proc_type);
3048 		if ((mac_srs->srs_first != NULL) &&
3049 		    (mac_srs->srs_tid == NULL))
3050 			mac_srs->srs_tid = timeout(mac_srs_fire,
3051 			    mac_srs, 1);
3052 		mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
3053 		return;
3054 	}
3055 	mutex_exit(&mac_srs->srs_bw->mac_bw_lock);
3056 
3057 leave_poll:
3058 
3059 	/* Nothing else to do. Get out of poll mode */
3060 	MAC_SRS_POLLING_OFF(mac_srs);
3061 	mac_srs->srs_state &= ~(SRS_PROC|proc_type);
3062 }
3063 
3064 /*
3065  * mac_srs_worker
3066  *
3067  * The SRS worker routine. Drains the queue when no one else is
3068  * processing it.
3069  */
3070 void
mac_srs_worker(mac_soft_ring_set_t * mac_srs)3071 mac_srs_worker(mac_soft_ring_set_t *mac_srs)
3072 {
3073 	kmutex_t 		*lock = &mac_srs->srs_lock;
3074 	kcondvar_t 		*async = &mac_srs->srs_async;
3075 	callb_cpr_t		cprinfo;
3076 	boolean_t		bw_ctl_flag;
3077 
3078 	CALLB_CPR_INIT(&cprinfo, lock, callb_generic_cpr, "srs_worker");
3079 	mutex_enter(lock);
3080 
3081 start:
3082 	for (;;) {
3083 		bw_ctl_flag = B_FALSE;
3084 		if (mac_srs->srs_type & SRST_BW_CONTROL) {
3085 			MAC_SRS_BW_LOCK(mac_srs);
3086 			MAC_SRS_CHECK_BW_CONTROL(mac_srs);
3087 			if (mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED)
3088 				bw_ctl_flag = B_TRUE;
3089 			MAC_SRS_BW_UNLOCK(mac_srs);
3090 		}
3091 		/*
3092 		 * The SRS_BW_ENFORCED flag may change since we have dropped
3093 		 * the mac_bw_lock. However the drain function can handle both
3094 		 * a drainable SRS or a bandwidth controlled SRS, and the
3095 		 * effect of scheduling a timeout is to wakeup the worker
3096 		 * thread which in turn will call the drain function. Since
3097 		 * we release the srs_lock atomically only in the cv_wait there
3098 		 * isn't a fear of waiting for ever.
3099 		 */
3100 		while (((mac_srs->srs_state & SRS_PROC) ||
3101 		    (mac_srs->srs_first == NULL) || bw_ctl_flag ||
3102 		    (mac_srs->srs_state & SRS_TX_BLOCKED)) &&
3103 		    !(mac_srs->srs_state & SRS_PAUSE)) {
3104 			/*
3105 			 * If we have packets queued and we are here
3106 			 * because B/W control is in place, we better
3107 			 * schedule the worker wakeup after 1 tick
3108 			 * to see if bandwidth control can be relaxed.
3109 			 */
3110 			if (bw_ctl_flag && mac_srs->srs_tid == NULL) {
3111 				/*
3112 				 * We need to ensure that a timer  is already
3113 				 * scheduled or we force  schedule one for
3114 				 * later so that we can continue processing
3115 				 * after this  quanta is over.
3116 				 */
3117 				mac_srs->srs_tid = timeout(mac_srs_fire,
3118 				    mac_srs, 1);
3119 			}
3120 wait:
3121 			CALLB_CPR_SAFE_BEGIN(&cprinfo);
3122 			cv_wait(async, lock);
3123 			CALLB_CPR_SAFE_END(&cprinfo, lock);
3124 
3125 			if (mac_srs->srs_state & SRS_PAUSE)
3126 				goto done;
3127 			if (mac_srs->srs_state & SRS_PROC)
3128 				goto wait;
3129 
3130 			if (mac_srs->srs_first != NULL &&
3131 			    mac_srs->srs_type & SRST_BW_CONTROL) {
3132 				MAC_SRS_BW_LOCK(mac_srs);
3133 				if (mac_srs->srs_bw->mac_bw_state &
3134 				    SRS_BW_ENFORCED) {
3135 					MAC_SRS_CHECK_BW_CONTROL(mac_srs);
3136 				}
3137 				bw_ctl_flag = mac_srs->srs_bw->mac_bw_state &
3138 				    SRS_BW_ENFORCED;
3139 				MAC_SRS_BW_UNLOCK(mac_srs);
3140 			}
3141 		}
3142 
3143 		if (mac_srs->srs_state & SRS_PAUSE)
3144 			goto done;
3145 		mac_srs->srs_drain_func(mac_srs, SRS_WORKER);
3146 	}
3147 done:
3148 	/*
3149 	 * The Rx SRS quiesce logic first cuts off packet supply to the SRS
3150 	 * from both hard and soft classifications and waits for such threads
3151 	 * to finish before signaling the worker. So at this point the only
3152 	 * thread left that could be competing with the worker is the poll
3153 	 * thread. In the case of Tx, there shouldn't be any thread holding
3154 	 * SRS_PROC at this point.
3155 	 */
3156 	if (!(mac_srs->srs_state & SRS_PROC)) {
3157 		mac_srs->srs_state |= SRS_PROC;
3158 	} else {
3159 		ASSERT((mac_srs->srs_type & SRST_TX) == 0);
3160 		/*
3161 		 * Poll thread still owns the SRS and is still running
3162 		 */
3163 		ASSERT((mac_srs->srs_poll_thr == NULL) ||
3164 		    ((mac_srs->srs_state & SRS_POLL_THR_OWNER) ==
3165 		    SRS_POLL_THR_OWNER));
3166 	}
3167 	mac_srs_worker_quiesce(mac_srs);
3168 	/*
3169 	 * Wait for the SRS_RESTART or SRS_CONDEMNED signal from the initiator
3170 	 * of the quiesce operation
3171 	 */
3172 	while (!(mac_srs->srs_state & (SRS_CONDEMNED | SRS_RESTART)))
3173 		cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock);
3174 
3175 	if (mac_srs->srs_state & SRS_RESTART) {
3176 		ASSERT(!(mac_srs->srs_state & SRS_CONDEMNED));
3177 		mac_srs_worker_restart(mac_srs);
3178 		mac_srs->srs_state &= ~SRS_PROC;
3179 		goto start;
3180 	}
3181 
3182 	if (!(mac_srs->srs_state & SRS_CONDEMNED_DONE))
3183 		mac_srs_worker_quiesce(mac_srs);
3184 
3185 	mac_srs->srs_state &= ~SRS_PROC;
3186 	/* The macro drops the srs_lock */
3187 	CALLB_CPR_EXIT(&cprinfo);
3188 	thread_exit();
3189 }
3190 
3191 /*
3192  * mac_rx_srs_subflow_process
3193  *
3194  * Receive side routine called from interrupt path when there are
3195  * sub flows present on this SRS.
3196  */
3197 /* ARGSUSED */
3198 void
mac_rx_srs_subflow_process(void * arg,mac_resource_handle_t srs,mblk_t * mp_chain,boolean_t loopback)3199 mac_rx_srs_subflow_process(void *arg, mac_resource_handle_t srs,
3200     mblk_t *mp_chain, boolean_t loopback)
3201 {
3202 	flow_entry_t		*flent = NULL;
3203 	flow_entry_t		*prev_flent = NULL;
3204 	mblk_t			*mp = NULL;
3205 	mblk_t			*tail = NULL;
3206 	mac_soft_ring_set_t	*mac_srs = (mac_soft_ring_set_t *)srs;
3207 	mac_client_impl_t	*mcip;
3208 
3209 	mcip = mac_srs->srs_mcip;
3210 	ASSERT(mcip != NULL);
3211 
3212 	/*
3213 	 * We need to determine the SRS for every packet
3214 	 * by walking the flow table, if we don't get any,
3215 	 * then we proceed using the SRS we came with.
3216 	 */
3217 	mp = tail = mp_chain;
3218 	while (mp != NULL) {
3219 
3220 		/*
3221 		 * We will increment the stats for the mactching subflow.
3222 		 * when we get the bytes/pkt count for the classified packets
3223 		 * later in mac_rx_srs_process.
3224 		 */
3225 		(void) mac_flow_lookup(mcip->mci_subflow_tab, mp,
3226 		    FLOW_INBOUND, &flent);
3227 
3228 		if (mp == mp_chain || flent == prev_flent) {
3229 			if (prev_flent != NULL)
3230 				FLOW_REFRELE(prev_flent);
3231 			prev_flent = flent;
3232 			flent = NULL;
3233 			tail = mp;
3234 			mp = mp->b_next;
3235 			continue;
3236 		}
3237 		tail->b_next = NULL;
3238 		/*
3239 		 * A null indicates, this is for the mac_srs itself.
3240 		 * XXX-venu : probably assert for fe_rx_srs_cnt == 0.
3241 		 */
3242 		if (prev_flent == NULL || prev_flent->fe_rx_srs_cnt == 0) {
3243 			mac_rx_srs_process(arg,
3244 			    (mac_resource_handle_t)mac_srs, mp_chain,
3245 			    loopback);
3246 		} else {
3247 			(prev_flent->fe_cb_fn)(prev_flent->fe_cb_arg1,
3248 			    prev_flent->fe_cb_arg2, mp_chain, loopback);
3249 			FLOW_REFRELE(prev_flent);
3250 		}
3251 		prev_flent = flent;
3252 		flent = NULL;
3253 		mp_chain = mp;
3254 		tail = mp;
3255 		mp = mp->b_next;
3256 	}
3257 	/* Last chain */
3258 	ASSERT(mp_chain != NULL);
3259 	if (prev_flent == NULL || prev_flent->fe_rx_srs_cnt == 0) {
3260 		mac_rx_srs_process(arg,
3261 		    (mac_resource_handle_t)mac_srs, mp_chain, loopback);
3262 	} else {
3263 		(prev_flent->fe_cb_fn)(prev_flent->fe_cb_arg1,
3264 		    prev_flent->fe_cb_arg2, mp_chain, loopback);
3265 		FLOW_REFRELE(prev_flent);
3266 	}
3267 }
3268 
3269 /*
3270  * mac_rx_srs_process
3271  *
3272  * Receive side routine called from the interrupt path.
3273  *
3274  * loopback is set to force a context switch on the loopback
3275  * path between MAC clients.
3276  */
3277 /* ARGSUSED */
3278 void
mac_rx_srs_process(void * arg,mac_resource_handle_t srs,mblk_t * mp_chain,boolean_t loopback)3279 mac_rx_srs_process(void *arg, mac_resource_handle_t srs, mblk_t *mp_chain,
3280     boolean_t loopback)
3281 {
3282 	mac_soft_ring_set_t	*mac_srs = (mac_soft_ring_set_t *)srs;
3283 	mblk_t			*mp, *tail, *head;
3284 	int			count = 0;
3285 	int			count1;
3286 	size_t			sz = 0;
3287 	size_t			chain_sz, sz1;
3288 	mac_bw_ctl_t		*mac_bw;
3289 	mac_srs_rx_t		*srs_rx = &mac_srs->srs_rx;
3290 
3291 	/*
3292 	 * Set the tail, count and sz. We set the sz irrespective
3293 	 * of whether we are doing B/W control or not for the
3294 	 * purpose of updating the stats.
3295 	 */
3296 	mp = tail = mp_chain;
3297 	while (mp != NULL) {
3298 		tail = mp;
3299 		count++;
3300 		sz += msgdsize(mp);
3301 		mp = mp->b_next;
3302 	}
3303 
3304 	mutex_enter(&mac_srs->srs_lock);
3305 
3306 	if (loopback) {
3307 		SRS_RX_STAT_UPDATE(mac_srs, lclbytes, sz);
3308 		SRS_RX_STAT_UPDATE(mac_srs, lclcnt, count);
3309 
3310 	} else {
3311 		SRS_RX_STAT_UPDATE(mac_srs, intrbytes, sz);
3312 		SRS_RX_STAT_UPDATE(mac_srs, intrcnt, count);
3313 	}
3314 
3315 	/*
3316 	 * If the SRS in already being processed; has been blanked;
3317 	 * can be processed by worker thread only; or the B/W limit
3318 	 * has been reached, then queue the chain and check if
3319 	 * worker thread needs to be awakend.
3320 	 */
3321 	if (mac_srs->srs_type & SRST_BW_CONTROL) {
3322 		mac_bw = mac_srs->srs_bw;
3323 		ASSERT(mac_bw != NULL);
3324 		mutex_enter(&mac_bw->mac_bw_lock);
3325 		mac_bw->mac_bw_intr += sz;
3326 		if (mac_bw->mac_bw_limit == 0) {
3327 			/* zero bandwidth: drop all */
3328 			srs_rx->sr_stat.mrs_sdrops += count;
3329 			mac_bw->mac_bw_drop_bytes += sz;
3330 			mutex_exit(&mac_bw->mac_bw_lock);
3331 			mutex_exit(&mac_srs->srs_lock);
3332 			mac_pkt_drop(NULL, NULL, mp_chain, B_FALSE);
3333 			return;
3334 		} else {
3335 			if ((mac_bw->mac_bw_sz + sz) <=
3336 			    mac_bw->mac_bw_drop_threshold) {
3337 				mutex_exit(&mac_bw->mac_bw_lock);
3338 				MAC_RX_SRS_ENQUEUE_CHAIN(mac_srs, mp_chain,
3339 				    tail, count, sz);
3340 			} else {
3341 				mp = mp_chain;
3342 				chain_sz = 0;
3343 				count1 = 0;
3344 				tail = NULL;
3345 				head = NULL;
3346 				while (mp != NULL) {
3347 					sz1 = msgdsize(mp);
3348 					if (mac_bw->mac_bw_sz + chain_sz + sz1 >
3349 					    mac_bw->mac_bw_drop_threshold)
3350 						break;
3351 					chain_sz += sz1;
3352 					count1++;
3353 					tail = mp;
3354 					mp = mp->b_next;
3355 				}
3356 				mutex_exit(&mac_bw->mac_bw_lock);
3357 				if (tail != NULL) {
3358 					head = tail->b_next;
3359 					tail->b_next = NULL;
3360 					MAC_RX_SRS_ENQUEUE_CHAIN(mac_srs,
3361 					    mp_chain, tail, count1, chain_sz);
3362 					sz -= chain_sz;
3363 					count -= count1;
3364 				} else {
3365 					/* Can't pick up any */
3366 					head = mp_chain;
3367 				}
3368 				if (head != NULL) {
3369 					/* Drop any packet over the threshold */
3370 					srs_rx->sr_stat.mrs_sdrops += count;
3371 					mutex_enter(&mac_bw->mac_bw_lock);
3372 					mac_bw->mac_bw_drop_bytes += sz;
3373 					mutex_exit(&mac_bw->mac_bw_lock);
3374 					freemsgchain(head);
3375 				}
3376 			}
3377 			MAC_SRS_WORKER_WAKEUP(mac_srs);
3378 			mutex_exit(&mac_srs->srs_lock);
3379 			return;
3380 		}
3381 	}
3382 
3383 	/*
3384 	 * If the total number of packets queued in the SRS and
3385 	 * its associated soft rings exceeds the max allowed,
3386 	 * then drop the chain. If we are polling capable, this
3387 	 * shouldn't be happening.
3388 	 */
3389 	if (!(mac_srs->srs_type & SRST_BW_CONTROL) &&
3390 	    (srs_rx->sr_poll_pkt_cnt > srs_rx->sr_hiwat)) {
3391 		mac_bw = mac_srs->srs_bw;
3392 		srs_rx->sr_stat.mrs_sdrops += count;
3393 		mutex_enter(&mac_bw->mac_bw_lock);
3394 		mac_bw->mac_bw_drop_bytes += sz;
3395 		mutex_exit(&mac_bw->mac_bw_lock);
3396 		freemsgchain(mp_chain);
3397 		mutex_exit(&mac_srs->srs_lock);
3398 		return;
3399 	}
3400 
3401 	MAC_RX_SRS_ENQUEUE_CHAIN(mac_srs, mp_chain, tail, count, sz);
3402 
3403 	if (!(mac_srs->srs_state & SRS_PROC)) {
3404 		/*
3405 		 * If we are coming via loopback, if we are not optimizing for
3406 		 * latency, or if our stack is running deep, we should signal
3407 		 * the worker thread.
3408 		 */
3409 		if (loopback || !(mac_srs->srs_state & SRS_LATENCY_OPT) ||
3410 		    MAC_RX_SRS_TOODEEP()) {
3411 			/*
3412 			 * For loopback, We need to let the worker take
3413 			 * over as we don't want to continue in the same
3414 			 * thread even if we can. This could lead to stack
3415 			 * overflows and may also end up using
3416 			 * resources (cpu) incorrectly.
3417 			 */
3418 			cv_signal(&mac_srs->srs_async);
3419 		} else {
3420 			/*
3421 			 * Seems like no one is processing the SRS and
3422 			 * there is no backlog. We also inline process
3423 			 * our packet if its a single packet in non
3424 			 * latency optimized case (in latency optimized
3425 			 * case, we inline process chains of any size).
3426 			 */
3427 			mac_srs->srs_drain_func(mac_srs, SRS_PROC_FAST);
3428 		}
3429 	}
3430 	mutex_exit(&mac_srs->srs_lock);
3431 }
3432 
3433 /* TX SIDE ROUTINES (RUNTIME) */
3434 
3435 /*
3436  * mac_tx_srs_no_desc
3437  *
3438  * This routine is called by Tx single ring default mode
3439  * when Tx ring runs out of descs.
3440  */
3441 mac_tx_cookie_t
mac_tx_srs_no_desc(mac_soft_ring_set_t * mac_srs,mblk_t * mp_chain,uint16_t flag,mblk_t ** ret_mp)3442 mac_tx_srs_no_desc(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
3443     uint16_t flag, mblk_t **ret_mp)
3444 {
3445 	mac_tx_cookie_t cookie = NULL;
3446 	mac_srs_tx_t *srs_tx = &mac_srs->srs_tx;
3447 	boolean_t wakeup_worker = B_TRUE;
3448 	uint32_t tx_mode = srs_tx->st_mode;
3449 	int cnt, sz;
3450 	mblk_t *tail;
3451 
3452 	ASSERT(tx_mode == SRS_TX_DEFAULT || tx_mode == SRS_TX_BW);
3453 	if (flag & MAC_DROP_ON_NO_DESC) {
3454 		MAC_TX_SRS_DROP_MESSAGE(mac_srs, mp_chain, cookie);
3455 	} else {
3456 		if (mac_srs->srs_first != NULL)
3457 			wakeup_worker = B_FALSE;
3458 		MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz);
3459 		if (flag & MAC_TX_NO_ENQUEUE) {
3460 			/*
3461 			 * If TX_QUEUED is not set, queue the
3462 			 * packet and let mac_tx_srs_drain()
3463 			 * set the TX_BLOCKED bit for the
3464 			 * reasons explained above. Otherwise,
3465 			 * return the mblks.
3466 			 */
3467 			if (wakeup_worker) {
3468 				MAC_TX_SRS_ENQUEUE_CHAIN(mac_srs,
3469 				    mp_chain, tail, cnt, sz);
3470 			} else {
3471 				MAC_TX_SET_NO_ENQUEUE(mac_srs,
3472 				    mp_chain, ret_mp, cookie);
3473 			}
3474 		} else {
3475 			MAC_TX_SRS_TEST_HIWAT(mac_srs, mp_chain,
3476 			    tail, cnt, sz, cookie);
3477 		}
3478 		if (wakeup_worker)
3479 			cv_signal(&mac_srs->srs_async);
3480 	}
3481 	return (cookie);
3482 }
3483 
3484 /*
3485  * mac_tx_srs_enqueue
3486  *
3487  * This routine is called when Tx SRS is operating in either serializer
3488  * or bandwidth mode. In serializer mode, a packet will get enqueued
3489  * when a thread cannot enter SRS exclusively. In bandwidth mode,
3490  * packets gets queued if allowed byte-count limit for a tick is
3491  * exceeded. The action that gets taken when MAC_DROP_ON_NO_DESC and
3492  * MAC_TX_NO_ENQUEUE is set is different than when operaing in either
3493  * the default mode or fanout mode. Here packets get dropped or
3494  * returned back to the caller only after hi-watermark worth of data
3495  * is queued.
3496  */
3497 static mac_tx_cookie_t
mac_tx_srs_enqueue(mac_soft_ring_set_t * mac_srs,mblk_t * mp_chain,uint16_t flag,uintptr_t fanout_hint,mblk_t ** ret_mp)3498 mac_tx_srs_enqueue(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
3499     uint16_t flag, uintptr_t fanout_hint, mblk_t **ret_mp)
3500 {
3501 	mac_tx_cookie_t cookie = NULL;
3502 	int cnt, sz;
3503 	mblk_t *tail;
3504 	boolean_t wakeup_worker = B_TRUE;
3505 
3506 	/*
3507 	 * Ignore fanout hint if we don't have multiple tx rings.
3508 	 */
3509 	if (!MAC_TX_SOFT_RINGS(mac_srs))
3510 		fanout_hint = 0;
3511 
3512 	if (mac_srs->srs_first != NULL)
3513 		wakeup_worker = B_FALSE;
3514 	MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz);
3515 	if (flag & MAC_DROP_ON_NO_DESC) {
3516 		if (mac_srs->srs_count > mac_srs->srs_tx.st_hiwat) {
3517 			MAC_TX_SRS_DROP_MESSAGE(mac_srs, mp_chain, cookie);
3518 		} else {
3519 			MAC_TX_SRS_ENQUEUE_CHAIN(mac_srs,
3520 			    mp_chain, tail, cnt, sz);
3521 		}
3522 	} else if (flag & MAC_TX_NO_ENQUEUE) {
3523 		if ((mac_srs->srs_count > mac_srs->srs_tx.st_hiwat) ||
3524 		    (mac_srs->srs_state & SRS_TX_WAKEUP_CLIENT)) {
3525 			MAC_TX_SET_NO_ENQUEUE(mac_srs, mp_chain,
3526 			    ret_mp, cookie);
3527 		} else {
3528 			mp_chain->b_prev = (mblk_t *)fanout_hint;
3529 			MAC_TX_SRS_ENQUEUE_CHAIN(mac_srs,
3530 			    mp_chain, tail, cnt, sz);
3531 		}
3532 	} else {
3533 		/*
3534 		 * If you are BW_ENFORCED, just enqueue the
3535 		 * packet. srs_worker will drain it at the
3536 		 * prescribed rate. Before enqueueing, save
3537 		 * the fanout hint.
3538 		 */
3539 		mp_chain->b_prev = (mblk_t *)fanout_hint;
3540 		MAC_TX_SRS_TEST_HIWAT(mac_srs, mp_chain,
3541 		    tail, cnt, sz, cookie);
3542 	}
3543 	if (wakeup_worker)
3544 		cv_signal(&mac_srs->srs_async);
3545 	return (cookie);
3546 }
3547 
3548 /*
3549  * There are seven tx modes:
3550  *
3551  * 1) Default mode (SRS_TX_DEFAULT)
3552  * 2) Serialization mode (SRS_TX_SERIALIZE)
3553  * 3) Fanout mode (SRS_TX_FANOUT)
3554  * 4) Bandwdith mode (SRS_TX_BW)
3555  * 5) Fanout and Bandwidth mode (SRS_TX_BW_FANOUT)
3556  * 6) aggr Tx mode (SRS_TX_AGGR)
3557  * 7) aggr Tx bw mode (SRS_TX_BW_AGGR)
3558  *
3559  * The tx mode in which an SRS operates is decided in mac_tx_srs_setup()
3560  * based on the number of Tx rings requested for an SRS and whether
3561  * bandwidth control is requested or not.
3562  *
3563  * The default mode (i.e., no fanout/no bandwidth) is used when the
3564  * underlying NIC does not have Tx rings or just one Tx ring. In this mode,
3565  * the SRS acts as a pass-thru. Packets will go directly to mac_tx_send().
3566  * When the underlying Tx ring runs out of Tx descs, it starts queueing up
3567  * packets in SRS. When flow-control is relieved, the srs_worker drains
3568  * the queued packets and informs blocked clients to restart sending
3569  * packets.
3570  *
3571  * In the SRS_TX_SERIALIZE mode, all calls to mac_tx() are serialized. This
3572  * mode is used when the link has no Tx rings or only one Tx ring.
3573  *
3574  * In the SRS_TX_FANOUT mode, packets will be fanned out to multiple
3575  * Tx rings. Each Tx ring will have a soft ring associated with it.
3576  * These soft rings will be hung off the Tx SRS. Queueing if it happens
3577  * due to lack of Tx desc will be in individual soft ring (and not srs)
3578  * associated with Tx ring.
3579  *
3580  * In the TX_BW mode, tx srs will allow packets to go down to Tx ring
3581  * only if bw is available. Otherwise the packets will be queued in
3582  * SRS. If fanout to multiple Tx rings is configured, the packets will
3583  * be fanned out among the soft rings associated with the Tx rings.
3584  *
3585  * In SRS_TX_AGGR mode, mac_tx_aggr_mode() routine is called. This routine
3586  * invokes an aggr function, aggr_find_tx_ring(), to find a pseudo Tx ring
3587  * belonging to a port on which the packet has to be sent. Aggr will
3588  * always have a pseudo Tx ring associated with it even when it is an
3589  * aggregation over a single NIC that has no Tx rings. Even in such a
3590  * case, the single pseudo Tx ring will have a soft ring associated with
3591  * it and the soft ring will hang off the SRS.
3592  *
3593  * If a bandwidth is specified for an aggr, SRS_TX_BW_AGGR mode is used.
3594  * In this mode, the bandwidth is first applied on the outgoing packets
3595  * and later mac_tx_addr_mode() function is called to send the packet out
3596  * of one of the pseudo Tx rings.
3597  *
3598  * Four flags are used in srs_state for indicating flow control
3599  * conditions : SRS_TX_BLOCKED, SRS_TX_HIWAT, SRS_TX_WAKEUP_CLIENT.
3600  * SRS_TX_BLOCKED indicates out of Tx descs. SRS expects a wakeup from the
3601  * driver below.
3602  * SRS_TX_HIWAT indicates packet count enqueued in Tx SRS exceeded Tx hiwat
3603  * and flow-control pressure is applied back to clients. The clients expect
3604  * wakeup when flow-control is relieved.
3605  * SRS_TX_WAKEUP_CLIENT get set when (flag == MAC_TX_NO_ENQUEUE) and mblk
3606  * got returned back to client either due to lack of Tx descs or due to bw
3607  * control reasons. The clients expect a wakeup when condition is relieved.
3608  *
3609  * The fourth argument to mac_tx() is the flag. Normally it will be 0 but
3610  * some clients set the following values too: MAC_DROP_ON_NO_DESC,
3611  * MAC_TX_NO_ENQUEUE
3612  * Mac clients that do not want packets to be enqueued in the mac layer set
3613  * MAC_DROP_ON_NO_DESC value. The packets won't be queued in the Tx SRS or
3614  * Tx soft rings but instead get dropped when the NIC runs out of desc. The
3615  * behaviour of this flag is different when the Tx is running in serializer
3616  * or bandwidth mode. Under these (Serializer, bandwidth) modes, the packet
3617  * get dropped when Tx high watermark is reached.
3618  * There are some mac clients like vsw, aggr that want the mblks to be
3619  * returned back to clients instead of being queued in Tx SRS (or Tx soft
3620  * rings) under flow-control (i.e., out of desc or exceeding bw limits)
3621  * conditions. These clients call mac_tx() with MAC_TX_NO_ENQUEUE flag set.
3622  * In the default and Tx fanout mode, the un-transmitted mblks will be
3623  * returned back to the clients when the driver runs out of Tx descs.
3624  * SRS_TX_WAKEUP_CLIENT (or S_RING_WAKEUP_CLIENT) will be set in SRS (or
3625  * soft ring) so that the clients can be woken up when Tx desc become
3626  * available. When running in serializer or bandwidth mode mode,
3627  * SRS_TX_WAKEUP_CLIENT will be set when tx hi-watermark is reached.
3628  */
3629 
3630 mac_tx_func_t
mac_tx_get_func(uint32_t mode)3631 mac_tx_get_func(uint32_t mode)
3632 {
3633 	return (mac_tx_mode_list[mode].mac_tx_func);
3634 }
3635 
3636 /* ARGSUSED */
3637 static mac_tx_cookie_t
mac_tx_single_ring_mode(mac_soft_ring_set_t * mac_srs,mblk_t * mp_chain,uintptr_t fanout_hint,uint16_t flag,mblk_t ** ret_mp)3638 mac_tx_single_ring_mode(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
3639     uintptr_t fanout_hint, uint16_t flag, mblk_t **ret_mp)
3640 {
3641 	mac_srs_tx_t		*srs_tx = &mac_srs->srs_tx;
3642 	mac_tx_stats_t		stats;
3643 	mac_tx_cookie_t		cookie = NULL;
3644 
3645 	ASSERT(srs_tx->st_mode == SRS_TX_DEFAULT);
3646 
3647 	/* Regular case with a single Tx ring */
3648 	/*
3649 	 * SRS_TX_BLOCKED is set when underlying NIC runs
3650 	 * out of Tx descs and messages start getting
3651 	 * queued. It won't get reset until
3652 	 * tx_srs_drain() completely drains out the
3653 	 * messages.
3654 	 */
3655 	if ((mac_srs->srs_state & SRS_ENQUEUED) != 0) {
3656 		/* Tx descs/resources not available */
3657 		mutex_enter(&mac_srs->srs_lock);
3658 		if ((mac_srs->srs_state & SRS_ENQUEUED) != 0) {
3659 			cookie = mac_tx_srs_no_desc(mac_srs, mp_chain,
3660 			    flag, ret_mp);
3661 			mutex_exit(&mac_srs->srs_lock);
3662 			return (cookie);
3663 		}
3664 		/*
3665 		 * While we were computing mblk count, the
3666 		 * flow control condition got relieved.
3667 		 * Continue with the transmission.
3668 		 */
3669 		mutex_exit(&mac_srs->srs_lock);
3670 	}
3671 
3672 	mp_chain = mac_tx_send(srs_tx->st_arg1, srs_tx->st_arg2,
3673 	    mp_chain, &stats);
3674 
3675 	/*
3676 	 * Multiple threads could be here sending packets.
3677 	 * Under such conditions, it is not possible to
3678 	 * automically set SRS_TX_BLOCKED bit to indicate
3679 	 * out of tx desc condition. To atomically set
3680 	 * this, we queue the returned packet and do
3681 	 * the setting of SRS_TX_BLOCKED in
3682 	 * mac_tx_srs_drain().
3683 	 */
3684 	if (mp_chain != NULL) {
3685 		mutex_enter(&mac_srs->srs_lock);
3686 		cookie = mac_tx_srs_no_desc(mac_srs, mp_chain, flag, ret_mp);
3687 		mutex_exit(&mac_srs->srs_lock);
3688 		return (cookie);
3689 	}
3690 	SRS_TX_STATS_UPDATE(mac_srs, &stats);
3691 
3692 	return (NULL);
3693 }
3694 
3695 /*
3696  * mac_tx_serialize_mode
3697  *
3698  * This is an experimental mode implemented as per the request of PAE.
3699  * In this mode, all callers attempting to send a packet to the NIC
3700  * will get serialized. Only one thread at any time will access the
3701  * NIC to send the packet out.
3702  */
3703 /* ARGSUSED */
3704 static mac_tx_cookie_t
mac_tx_serializer_mode(mac_soft_ring_set_t * mac_srs,mblk_t * mp_chain,uintptr_t fanout_hint,uint16_t flag,mblk_t ** ret_mp)3705 mac_tx_serializer_mode(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
3706     uintptr_t fanout_hint, uint16_t flag, mblk_t **ret_mp)
3707 {
3708 	mac_tx_stats_t		stats;
3709 	mac_tx_cookie_t		cookie = NULL;
3710 	mac_srs_tx_t		*srs_tx = &mac_srs->srs_tx;
3711 
3712 	/* Single ring, serialize below */
3713 	ASSERT(srs_tx->st_mode == SRS_TX_SERIALIZE);
3714 	mutex_enter(&mac_srs->srs_lock);
3715 	if ((mac_srs->srs_first != NULL) ||
3716 	    (mac_srs->srs_state & SRS_PROC)) {
3717 		/*
3718 		 * In serialization mode, queue all packets until
3719 		 * TX_HIWAT is set.
3720 		 * If drop bit is set, drop if TX_HIWAT is set.
3721 		 * If no_enqueue is set, still enqueue until hiwat
3722 		 * is set and return mblks after TX_HIWAT is set.
3723 		 */
3724 		cookie = mac_tx_srs_enqueue(mac_srs, mp_chain,
3725 		    flag, NULL, ret_mp);
3726 		mutex_exit(&mac_srs->srs_lock);
3727 		return (cookie);
3728 	}
3729 	/*
3730 	 * No packets queued, nothing on proc and no flow
3731 	 * control condition. Fast-path, ok. Do inline
3732 	 * processing.
3733 	 */
3734 	mac_srs->srs_state |= SRS_PROC;
3735 	mutex_exit(&mac_srs->srs_lock);
3736 
3737 	mp_chain = mac_tx_send(srs_tx->st_arg1, srs_tx->st_arg2,
3738 	    mp_chain, &stats);
3739 
3740 	mutex_enter(&mac_srs->srs_lock);
3741 	mac_srs->srs_state &= ~SRS_PROC;
3742 	if (mp_chain != NULL) {
3743 		cookie = mac_tx_srs_enqueue(mac_srs,
3744 		    mp_chain, flag, NULL, ret_mp);
3745 	}
3746 	if (mac_srs->srs_first != NULL) {
3747 		/*
3748 		 * We processed inline our packet and a new
3749 		 * packet/s got queued while we were
3750 		 * processing. Wakeup srs worker
3751 		 */
3752 		cv_signal(&mac_srs->srs_async);
3753 	}
3754 	mutex_exit(&mac_srs->srs_lock);
3755 
3756 	if (cookie == NULL)
3757 		SRS_TX_STATS_UPDATE(mac_srs, &stats);
3758 
3759 	return (cookie);
3760 }
3761 
3762 /*
3763  * mac_tx_fanout_mode
3764  *
3765  * In this mode, the SRS will have access to multiple Tx rings to send
3766  * the packet out. The fanout hint that is passed as an argument is
3767  * used to find an appropriate ring to fanout the traffic. Each Tx
3768  * ring, in turn,  will have a soft ring associated with it. If a Tx
3769  * ring runs out of Tx desc's the returned packet will be queued in
3770  * the soft ring associated with that Tx ring. The srs itself will not
3771  * queue any packets.
3772  */
3773 
3774 #define	MAC_TX_SOFT_RING_PROCESS(chain) {		       		\
3775 	index = COMPUTE_INDEX(hash, mac_srs->srs_tx_ring_count),	\
3776 	softring = mac_srs->srs_tx_soft_rings[index];			\
3777 	cookie = mac_tx_soft_ring_process(softring, chain, flag, ret_mp); \
3778 	DTRACE_PROBE2(tx__fanout, uint64_t, hash, uint_t, index);	\
3779 }
3780 
3781 static mac_tx_cookie_t
mac_tx_fanout_mode(mac_soft_ring_set_t * mac_srs,mblk_t * mp_chain,uintptr_t fanout_hint,uint16_t flag,mblk_t ** ret_mp)3782 mac_tx_fanout_mode(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
3783     uintptr_t fanout_hint, uint16_t flag, mblk_t **ret_mp)
3784 {
3785 	mac_soft_ring_t		*softring;
3786 	uint64_t		hash;
3787 	uint_t			index;
3788 	mac_tx_cookie_t		cookie = NULL;
3789 
3790 	ASSERT(mac_srs->srs_tx.st_mode == SRS_TX_FANOUT ||
3791 	    mac_srs->srs_tx.st_mode == SRS_TX_BW_FANOUT);
3792 	if (fanout_hint != 0) {
3793 		/*
3794 		 * The hint is specified by the caller, simply pass the
3795 		 * whole chain to the soft ring.
3796 		 */
3797 		hash = HASH_HINT(fanout_hint);
3798 		MAC_TX_SOFT_RING_PROCESS(mp_chain);
3799 	} else {
3800 		mblk_t *last_mp, *cur_mp, *sub_chain;
3801 		uint64_t last_hash = 0;
3802 		uint_t media = mac_srs->srs_mcip->mci_mip->mi_info.mi_media;
3803 
3804 		/*
3805 		 * Compute the hash from the contents (headers) of the
3806 		 * packets of the mblk chain. Split the chains into
3807 		 * subchains of the same conversation.
3808 		 *
3809 		 * Since there may be more than one ring used for
3810 		 * sub-chains of the same call, and since the caller
3811 		 * does not maintain per conversation state since it
3812 		 * passed a zero hint, unsent subchains will be
3813 		 * dropped.
3814 		 */
3815 
3816 		flag |= MAC_DROP_ON_NO_DESC;
3817 		ret_mp = NULL;
3818 
3819 		ASSERT(ret_mp == NULL);
3820 
3821 		sub_chain = NULL;
3822 		last_mp = NULL;
3823 
3824 		for (cur_mp = mp_chain; cur_mp != NULL;
3825 		    cur_mp = cur_mp->b_next) {
3826 			hash = mac_pkt_hash(media, cur_mp, MAC_PKT_HASH_L4,
3827 			    B_TRUE);
3828 			if (last_hash != 0 && hash != last_hash) {
3829 				/*
3830 				 * Starting a different subchain, send current
3831 				 * chain out.
3832 				 */
3833 				ASSERT(last_mp != NULL);
3834 				last_mp->b_next = NULL;
3835 				MAC_TX_SOFT_RING_PROCESS(sub_chain);
3836 				sub_chain = NULL;
3837 			}
3838 
3839 			/* add packet to subchain */
3840 			if (sub_chain == NULL)
3841 				sub_chain = cur_mp;
3842 			last_mp = cur_mp;
3843 			last_hash = hash;
3844 		}
3845 
3846 		if (sub_chain != NULL) {
3847 			/* send last subchain */
3848 			ASSERT(last_mp != NULL);
3849 			last_mp->b_next = NULL;
3850 			MAC_TX_SOFT_RING_PROCESS(sub_chain);
3851 		}
3852 
3853 		cookie = NULL;
3854 	}
3855 
3856 	return (cookie);
3857 }
3858 
3859 /*
3860  * mac_tx_bw_mode
3861  *
3862  * In the bandwidth mode, Tx srs will allow packets to go down to Tx ring
3863  * only if bw is available. Otherwise the packets will be queued in
3864  * SRS. If the SRS has multiple Tx rings, then packets will get fanned
3865  * out to a Tx rings.
3866  */
3867 static mac_tx_cookie_t
mac_tx_bw_mode(mac_soft_ring_set_t * mac_srs,mblk_t * mp_chain,uintptr_t fanout_hint,uint16_t flag,mblk_t ** ret_mp)3868 mac_tx_bw_mode(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
3869     uintptr_t fanout_hint, uint16_t flag, mblk_t **ret_mp)
3870 {
3871 	int			cnt, sz;
3872 	mblk_t			*tail;
3873 	mac_tx_cookie_t		cookie = NULL;
3874 	mac_srs_tx_t		*srs_tx = &mac_srs->srs_tx;
3875 	clock_t			now;
3876 
3877 	ASSERT(TX_BANDWIDTH_MODE(mac_srs));
3878 	ASSERT(mac_srs->srs_type & SRST_BW_CONTROL);
3879 	mutex_enter(&mac_srs->srs_lock);
3880 	if (mac_srs->srs_bw->mac_bw_limit == 0) {
3881 		/*
3882 		 * zero bandwidth, no traffic is sent: drop the packets,
3883 		 * or return the whole chain if the caller requests all
3884 		 * unsent packets back.
3885 		 */
3886 		if (flag & MAC_TX_NO_ENQUEUE) {
3887 			cookie = (mac_tx_cookie_t)mac_srs;
3888 			*ret_mp = mp_chain;
3889 		} else {
3890 			MAC_TX_SRS_DROP_MESSAGE(mac_srs, mp_chain, cookie);
3891 		}
3892 		mutex_exit(&mac_srs->srs_lock);
3893 		return (cookie);
3894 	} else if ((mac_srs->srs_first != NULL) ||
3895 	    (mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED)) {
3896 		cookie = mac_tx_srs_enqueue(mac_srs, mp_chain, flag,
3897 		    fanout_hint, ret_mp);
3898 		mutex_exit(&mac_srs->srs_lock);
3899 		return (cookie);
3900 	}
3901 	MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz);
3902 	now = ddi_get_lbolt();
3903 	if (mac_srs->srs_bw->mac_bw_curr_time != now) {
3904 		mac_srs->srs_bw->mac_bw_curr_time = now;
3905 		mac_srs->srs_bw->mac_bw_used = 0;
3906 	} else if (mac_srs->srs_bw->mac_bw_used >
3907 	    mac_srs->srs_bw->mac_bw_limit) {
3908 		mac_srs->srs_bw->mac_bw_state |= SRS_BW_ENFORCED;
3909 		MAC_TX_SRS_ENQUEUE_CHAIN(mac_srs,
3910 		    mp_chain, tail, cnt, sz);
3911 		/*
3912 		 * Wakeup worker thread. Note that worker
3913 		 * thread has to be woken up so that it
3914 		 * can fire up the timer to be woken up
3915 		 * on the next tick. Also once
3916 		 * BW_ENFORCED is set, it can only be
3917 		 * reset by srs_worker thread. Until then
3918 		 * all packets will get queued up in SRS
3919 		 * and hence this this code path won't be
3920 		 * entered until BW_ENFORCED is reset.
3921 		 */
3922 		cv_signal(&mac_srs->srs_async);
3923 		mutex_exit(&mac_srs->srs_lock);
3924 		return (cookie);
3925 	}
3926 
3927 	mac_srs->srs_bw->mac_bw_used += sz;
3928 	mutex_exit(&mac_srs->srs_lock);
3929 
3930 	if (srs_tx->st_mode == SRS_TX_BW_FANOUT) {
3931 		mac_soft_ring_t *softring;
3932 		uint_t indx, hash;
3933 
3934 		hash = HASH_HINT(fanout_hint);
3935 		indx = COMPUTE_INDEX(hash,
3936 		    mac_srs->srs_tx_ring_count);
3937 		softring = mac_srs->srs_tx_soft_rings[indx];
3938 		return (mac_tx_soft_ring_process(softring, mp_chain, flag,
3939 		    ret_mp));
3940 	} else if (srs_tx->st_mode == SRS_TX_BW_AGGR) {
3941 		return (mac_tx_aggr_mode(mac_srs, mp_chain,
3942 		    fanout_hint, flag, ret_mp));
3943 	} else {
3944 		mac_tx_stats_t		stats;
3945 
3946 		mp_chain = mac_tx_send(srs_tx->st_arg1, srs_tx->st_arg2,
3947 		    mp_chain, &stats);
3948 
3949 		if (mp_chain != NULL) {
3950 			mutex_enter(&mac_srs->srs_lock);
3951 			MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz);
3952 			if (mac_srs->srs_bw->mac_bw_used > sz)
3953 				mac_srs->srs_bw->mac_bw_used -= sz;
3954 			else
3955 				mac_srs->srs_bw->mac_bw_used = 0;
3956 			cookie = mac_tx_srs_enqueue(mac_srs, mp_chain, flag,
3957 			    fanout_hint, ret_mp);
3958 			mutex_exit(&mac_srs->srs_lock);
3959 			return (cookie);
3960 		}
3961 		SRS_TX_STATS_UPDATE(mac_srs, &stats);
3962 
3963 		return (NULL);
3964 	}
3965 }
3966 
3967 /*
3968  * mac_tx_aggr_mode
3969  *
3970  * This routine invokes an aggr function, aggr_find_tx_ring(), to find
3971  * a (pseudo) Tx ring belonging to a port on which the packet has to
3972  * be sent. aggr_find_tx_ring() first finds the outgoing port based on
3973  * L2/L3/L4 policy and then uses the fanout_hint passed to it to pick
3974  * a Tx ring from the selected port.
3975  *
3976  * Note that a port can be deleted from the aggregation. In such a case,
3977  * the aggregation layer first separates the port from the rest of the
3978  * ports making sure that port (and thus any Tx rings associated with
3979  * it) won't get selected in the call to aggr_find_tx_ring() function.
3980  * Later calls are made to mac_group_rem_ring() passing pseudo Tx ring
3981  * handles one by one which in turn will quiesce the Tx SRS and remove
3982  * the soft ring associated with the pseudo Tx ring. Unlike Rx side
3983  * where a cookie is used to protect against mac_rx_ring() calls on
3984  * rings that have been removed, no such cookie is needed on the Tx
3985  * side as the pseudo Tx ring won't be available anymore to
3986  * aggr_find_tx_ring() once the port has been removed.
3987  */
3988 static mac_tx_cookie_t
mac_tx_aggr_mode(mac_soft_ring_set_t * mac_srs,mblk_t * mp_chain,uintptr_t fanout_hint,uint16_t flag,mblk_t ** ret_mp)3989 mac_tx_aggr_mode(mac_soft_ring_set_t *mac_srs, mblk_t *mp_chain,
3990     uintptr_t fanout_hint, uint16_t flag, mblk_t **ret_mp)
3991 {
3992 	mac_srs_tx_t		*srs_tx = &mac_srs->srs_tx;
3993 	mac_tx_ring_fn_t	find_tx_ring_fn;
3994 	mac_ring_handle_t	ring = NULL;
3995 	void			*arg;
3996 	mac_soft_ring_t		*sringp;
3997 
3998 	find_tx_ring_fn = srs_tx->st_capab_aggr.mca_find_tx_ring_fn;
3999 	arg = srs_tx->st_capab_aggr.mca_arg;
4000 	if (find_tx_ring_fn(arg, mp_chain, fanout_hint, &ring) == NULL)
4001 		return (NULL);
4002 	sringp = srs_tx->st_soft_rings[((mac_ring_t *)ring)->mr_index];
4003 	return (mac_tx_soft_ring_process(sringp, mp_chain, flag, ret_mp));
4004 }
4005 
4006 void
mac_tx_invoke_callbacks(mac_client_impl_t * mcip,mac_tx_cookie_t cookie)4007 mac_tx_invoke_callbacks(mac_client_impl_t *mcip, mac_tx_cookie_t cookie)
4008 {
4009 	mac_cb_t *mcb;
4010 	mac_tx_notify_cb_t *mtnfp;
4011 
4012 	/* Wakeup callback registered clients */
4013 	MAC_CALLBACK_WALKER_INC(&mcip->mci_tx_notify_cb_info);
4014 	for (mcb = mcip->mci_tx_notify_cb_list; mcb != NULL;
4015 	    mcb = mcb->mcb_nextp) {
4016 		mtnfp = (mac_tx_notify_cb_t *)mcb->mcb_objp;
4017 		mtnfp->mtnf_fn(mtnfp->mtnf_arg, cookie);
4018 	}
4019 	MAC_CALLBACK_WALKER_DCR(&mcip->mci_tx_notify_cb_info,
4020 	    &mcip->mci_tx_notify_cb_list);
4021 }
4022 
4023 /* ARGSUSED */
4024 void
mac_tx_srs_drain(mac_soft_ring_set_t * mac_srs,uint_t proc_type)4025 mac_tx_srs_drain(mac_soft_ring_set_t *mac_srs, uint_t proc_type)
4026 {
4027 	mblk_t			*head, *tail;
4028 	size_t			sz;
4029 	uint32_t		tx_mode;
4030 	uint_t			saved_pkt_count;
4031 	mac_tx_stats_t		stats;
4032 	mac_srs_tx_t		*srs_tx = &mac_srs->srs_tx;
4033 	clock_t			now;
4034 
4035 	saved_pkt_count = 0;
4036 	ASSERT(mutex_owned(&mac_srs->srs_lock));
4037 	ASSERT(!(mac_srs->srs_state & SRS_PROC));
4038 
4039 	mac_srs->srs_state |= SRS_PROC;
4040 
4041 	tx_mode = srs_tx->st_mode;
4042 	if (tx_mode == SRS_TX_DEFAULT || tx_mode == SRS_TX_SERIALIZE) {
4043 		if (mac_srs->srs_first != NULL) {
4044 			head = mac_srs->srs_first;
4045 			tail = mac_srs->srs_last;
4046 			saved_pkt_count = mac_srs->srs_count;
4047 			mac_srs->srs_first = NULL;
4048 			mac_srs->srs_last = NULL;
4049 			mac_srs->srs_count = 0;
4050 			mutex_exit(&mac_srs->srs_lock);
4051 
4052 			head = mac_tx_send(srs_tx->st_arg1, srs_tx->st_arg2,
4053 			    head, &stats);
4054 
4055 			mutex_enter(&mac_srs->srs_lock);
4056 			if (head != NULL) {
4057 				/* Device out of tx desc, set block */
4058 				if (head->b_next == NULL)
4059 					VERIFY(head == tail);
4060 				tail->b_next = mac_srs->srs_first;
4061 				mac_srs->srs_first = head;
4062 				mac_srs->srs_count +=
4063 				    (saved_pkt_count - stats.mts_opackets);
4064 				if (mac_srs->srs_last == NULL)
4065 					mac_srs->srs_last = tail;
4066 				MAC_TX_SRS_BLOCK(mac_srs, head);
4067 			} else {
4068 				srs_tx->st_woken_up = B_FALSE;
4069 				SRS_TX_STATS_UPDATE(mac_srs, &stats);
4070 			}
4071 		}
4072 	} else if (tx_mode == SRS_TX_BW) {
4073 		/*
4074 		 * We are here because the timer fired and we have some data
4075 		 * to tranmit. Also mac_tx_srs_worker should have reset
4076 		 * SRS_BW_ENFORCED flag
4077 		 */
4078 		ASSERT(!(mac_srs->srs_bw->mac_bw_state & SRS_BW_ENFORCED));
4079 		head = tail = mac_srs->srs_first;
4080 		while (mac_srs->srs_first != NULL) {
4081 			tail = mac_srs->srs_first;
4082 			tail->b_prev = NULL;
4083 			mac_srs->srs_first = tail->b_next;
4084 			if (mac_srs->srs_first == NULL)
4085 				mac_srs->srs_last = NULL;
4086 			mac_srs->srs_count--;
4087 			sz = msgdsize(tail);
4088 			mac_srs->srs_size -= sz;
4089 			saved_pkt_count++;
4090 			MAC_TX_UPDATE_BW_INFO(mac_srs, sz);
4091 
4092 			if (mac_srs->srs_bw->mac_bw_used <
4093 			    mac_srs->srs_bw->mac_bw_limit)
4094 				continue;
4095 
4096 			now = ddi_get_lbolt();
4097 			if (mac_srs->srs_bw->mac_bw_curr_time != now) {
4098 				mac_srs->srs_bw->mac_bw_curr_time = now;
4099 				mac_srs->srs_bw->mac_bw_used = sz;
4100 				continue;
4101 			}
4102 			mac_srs->srs_bw->mac_bw_state |= SRS_BW_ENFORCED;
4103 			break;
4104 		}
4105 
4106 		ASSERT((head == NULL && tail == NULL) ||
4107 		    (head != NULL && tail != NULL));
4108 		if (tail != NULL) {
4109 			tail->b_next = NULL;
4110 			mutex_exit(&mac_srs->srs_lock);
4111 
4112 			head = mac_tx_send(srs_tx->st_arg1, srs_tx->st_arg2,
4113 			    head, &stats);
4114 
4115 			mutex_enter(&mac_srs->srs_lock);
4116 			if (head != NULL) {
4117 				uint_t size_sent;
4118 
4119 				/* Device out of tx desc, set block */
4120 				if (head->b_next == NULL)
4121 					VERIFY(head == tail);
4122 				tail->b_next = mac_srs->srs_first;
4123 				mac_srs->srs_first = head;
4124 				mac_srs->srs_count +=
4125 				    (saved_pkt_count - stats.mts_opackets);
4126 				if (mac_srs->srs_last == NULL)
4127 					mac_srs->srs_last = tail;
4128 				size_sent = sz - stats.mts_obytes;
4129 				mac_srs->srs_size += size_sent;
4130 				mac_srs->srs_bw->mac_bw_sz += size_sent;
4131 				if (mac_srs->srs_bw->mac_bw_used > size_sent) {
4132 					mac_srs->srs_bw->mac_bw_used -=
4133 					    size_sent;
4134 				} else {
4135 					mac_srs->srs_bw->mac_bw_used = 0;
4136 				}
4137 				MAC_TX_SRS_BLOCK(mac_srs, head);
4138 			} else {
4139 				srs_tx->st_woken_up = B_FALSE;
4140 				SRS_TX_STATS_UPDATE(mac_srs, &stats);
4141 			}
4142 		}
4143 	} else if (tx_mode == SRS_TX_BW_FANOUT || tx_mode == SRS_TX_BW_AGGR) {
4144 		mblk_t *prev;
4145 		uint64_t hint;
4146 
4147 		/*
4148 		 * We are here because the timer fired and we
4149 		 * have some quota to tranmit.
4150 		 */
4151 		prev = NULL;
4152 		head = tail = mac_srs->srs_first;
4153 		while (mac_srs->srs_first != NULL) {
4154 			tail = mac_srs->srs_first;
4155 			mac_srs->srs_first = tail->b_next;
4156 			if (mac_srs->srs_first == NULL)
4157 				mac_srs->srs_last = NULL;
4158 			mac_srs->srs_count--;
4159 			sz = msgdsize(tail);
4160 			mac_srs->srs_size -= sz;
4161 			mac_srs->srs_bw->mac_bw_used += sz;
4162 			if (prev == NULL)
4163 				hint = (ulong_t)tail->b_prev;
4164 			if (hint != (ulong_t)tail->b_prev) {
4165 				prev->b_next = NULL;
4166 				mutex_exit(&mac_srs->srs_lock);
4167 				TX_SRS_TO_SOFT_RING(mac_srs, head, hint);
4168 				head = tail;
4169 				hint = (ulong_t)tail->b_prev;
4170 				mutex_enter(&mac_srs->srs_lock);
4171 			}
4172 
4173 			prev = tail;
4174 			tail->b_prev = NULL;
4175 			if (mac_srs->srs_bw->mac_bw_used <
4176 			    mac_srs->srs_bw->mac_bw_limit)
4177 				continue;
4178 
4179 			now = ddi_get_lbolt();
4180 			if (mac_srs->srs_bw->mac_bw_curr_time != now) {
4181 				mac_srs->srs_bw->mac_bw_curr_time = now;
4182 				mac_srs->srs_bw->mac_bw_used = 0;
4183 				continue;
4184 			}
4185 			mac_srs->srs_bw->mac_bw_state |= SRS_BW_ENFORCED;
4186 			break;
4187 		}
4188 		ASSERT((head == NULL && tail == NULL) ||
4189 		    (head != NULL && tail != NULL));
4190 		if (tail != NULL) {
4191 			tail->b_next = NULL;
4192 			mutex_exit(&mac_srs->srs_lock);
4193 			TX_SRS_TO_SOFT_RING(mac_srs, head, hint);
4194 			mutex_enter(&mac_srs->srs_lock);
4195 		}
4196 	}
4197 	/*
4198 	 * SRS_TX_FANOUT case not considered here because packets
4199 	 * won't be queued in the SRS for this case. Packets will
4200 	 * be sent directly to soft rings underneath and if there
4201 	 * is any queueing at all, it would be in Tx side soft
4202 	 * rings.
4203 	 */
4204 
4205 	/*
4206 	 * When srs_count becomes 0, reset SRS_TX_HIWAT and
4207 	 * SRS_TX_WAKEUP_CLIENT and wakeup registered clients.
4208 	 */
4209 	if (mac_srs->srs_count == 0 && (mac_srs->srs_state &
4210 	    (SRS_TX_HIWAT | SRS_TX_WAKEUP_CLIENT | SRS_ENQUEUED))) {
4211 		mac_client_impl_t *mcip = mac_srs->srs_mcip;
4212 		boolean_t wakeup_required = B_FALSE;
4213 
4214 		if (mac_srs->srs_state &
4215 		    (SRS_TX_HIWAT|SRS_TX_WAKEUP_CLIENT)) {
4216 			wakeup_required = B_TRUE;
4217 		}
4218 		mac_srs->srs_state &= ~(SRS_TX_HIWAT |
4219 		    SRS_TX_WAKEUP_CLIENT | SRS_ENQUEUED);
4220 		mutex_exit(&mac_srs->srs_lock);
4221 		if (wakeup_required) {
4222 			mac_tx_invoke_callbacks(mcip, (mac_tx_cookie_t)mac_srs);
4223 			/*
4224 			 * If the client is not the primary MAC client, then we
4225 			 * need to send the notification to the clients upper
4226 			 * MAC, i.e. mci_upper_mip.
4227 			 */
4228 			mac_tx_notify(mcip->mci_upper_mip != NULL ?
4229 			    mcip->mci_upper_mip : mcip->mci_mip);
4230 		}
4231 		mutex_enter(&mac_srs->srs_lock);
4232 	}
4233 	mac_srs->srs_state &= ~SRS_PROC;
4234 }
4235 
4236 /*
4237  * Given a packet, get the flow_entry that identifies the flow
4238  * to which that packet belongs. The flow_entry will contain
4239  * the transmit function to be used to send the packet. If the
4240  * function returns NULL, the packet should be sent using the
4241  * underlying NIC.
4242  */
4243 static flow_entry_t *
mac_tx_classify(mac_impl_t * mip,mblk_t * mp)4244 mac_tx_classify(mac_impl_t *mip, mblk_t *mp)
4245 {
4246 	flow_entry_t		*flent = NULL;
4247 	mac_client_impl_t	*mcip;
4248 	int	err;
4249 
4250 	/*
4251 	 * Do classification on the packet.
4252 	 */
4253 	err = mac_flow_lookup(mip->mi_flow_tab, mp, FLOW_OUTBOUND, &flent);
4254 	if (err != 0)
4255 		return (NULL);
4256 
4257 	/*
4258 	 * This flent might just be an additional one on the MAC client,
4259 	 * i.e. for classification purposes (different fdesc), however
4260 	 * the resources, SRS et. al., are in the mci_flent, so if
4261 	 * this isn't the mci_flent, we need to get it.
4262 	 */
4263 	if ((mcip = flent->fe_mcip) != NULL && mcip->mci_flent != flent) {
4264 		FLOW_REFRELE(flent);
4265 		flent = mcip->mci_flent;
4266 		FLOW_TRY_REFHOLD(flent, err);
4267 		if (err != 0)
4268 			return (NULL);
4269 	}
4270 
4271 	return (flent);
4272 }
4273 
4274 /*
4275  * This macro is only meant to be used by mac_tx_send().
4276  */
4277 #define	CHECK_VID_AND_ADD_TAG(mp) {			\
4278 	if (vid_check) {				\
4279 		int err = 0;				\
4280 							\
4281 		MAC_VID_CHECK(src_mcip, (mp), err);	\
4282 		if (err != 0) {				\
4283 			freemsg((mp));			\
4284 			(mp) = next;			\
4285 			oerrors++;			\
4286 			continue;			\
4287 		}					\
4288 	}						\
4289 	if (add_tag) {					\
4290 		(mp) = mac_add_vlan_tag((mp), 0, vid);	\
4291 		if ((mp) == NULL) {			\
4292 			(mp) = next;			\
4293 			oerrors++;			\
4294 			continue;			\
4295 		}					\
4296 	}						\
4297 }
4298 
4299 mblk_t *
mac_tx_send(mac_client_handle_t mch,mac_ring_handle_t ring,mblk_t * mp_chain,mac_tx_stats_t * stats)4300 mac_tx_send(mac_client_handle_t mch, mac_ring_handle_t ring, mblk_t *mp_chain,
4301     mac_tx_stats_t *stats)
4302 {
4303 	mac_client_impl_t *src_mcip = (mac_client_impl_t *)mch;
4304 	mac_impl_t *mip = src_mcip->mci_mip;
4305 	uint_t obytes = 0, opackets = 0, oerrors = 0;
4306 	mblk_t *mp = NULL, *next;
4307 	boolean_t vid_check, add_tag;
4308 	uint16_t vid = 0;
4309 
4310 	if (mip->mi_nclients > 1) {
4311 		vid_check = MAC_VID_CHECK_NEEDED(src_mcip);
4312 		add_tag = MAC_TAG_NEEDED(src_mcip);
4313 		if (add_tag)
4314 			vid = mac_client_vid(mch);
4315 	} else {
4316 		ASSERT(mip->mi_nclients == 1);
4317 		vid_check = add_tag = B_FALSE;
4318 	}
4319 
4320 	/*
4321 	 * Fastpath: if there's only one client, we simply send
4322 	 * the packet down to the underlying NIC.
4323 	 */
4324 	if (mip->mi_nactiveclients == 1) {
4325 		DTRACE_PROBE2(fastpath,
4326 		    mac_client_impl_t *, src_mcip, mblk_t *, mp_chain);
4327 
4328 		mp = mp_chain;
4329 		while (mp != NULL) {
4330 			next = mp->b_next;
4331 			mp->b_next = NULL;
4332 			opackets++;
4333 			obytes += (mp->b_cont == NULL ? MBLKL(mp) :
4334 			    msgdsize(mp));
4335 
4336 			CHECK_VID_AND_ADD_TAG(mp);
4337 			MAC_TX(mip, ring, mp, src_mcip);
4338 
4339 			/*
4340 			 * If the driver is out of descriptors and does a
4341 			 * partial send it will return a chain of unsent
4342 			 * mblks. Adjust the accounting stats.
4343 			 */
4344 			if (mp != NULL) {
4345 				opackets--;
4346 				obytes -= msgdsize(mp);
4347 				mp->b_next = next;
4348 				break;
4349 			}
4350 			mp = next;
4351 		}
4352 		goto done;
4353 	}
4354 
4355 	/*
4356 	 * No fastpath, we either have more than one MAC client
4357 	 * defined on top of the same MAC, or one or more MAC
4358 	 * client promiscuous callbacks.
4359 	 */
4360 	DTRACE_PROBE3(slowpath, mac_client_impl_t *,
4361 	    src_mcip, int, mip->mi_nclients, mblk_t *, mp_chain);
4362 
4363 	mp = mp_chain;
4364 	while (mp != NULL) {
4365 		flow_entry_t *dst_flow_ent;
4366 		void *flow_cookie;
4367 		size_t	pkt_size;
4368 		mblk_t *mp1;
4369 
4370 		next = mp->b_next;
4371 		mp->b_next = NULL;
4372 		opackets++;
4373 		pkt_size = (mp->b_cont == NULL ? MBLKL(mp) : msgdsize(mp));
4374 		obytes += pkt_size;
4375 		CHECK_VID_AND_ADD_TAG(mp);
4376 
4377 		/*
4378 		 * Find the destination.
4379 		 */
4380 		dst_flow_ent = mac_tx_classify(mip, mp);
4381 
4382 		if (dst_flow_ent != NULL) {
4383 			size_t	hdrsize;
4384 			int	err = 0;
4385 
4386 			if (mip->mi_info.mi_nativemedia == DL_ETHER) {
4387 				struct ether_vlan_header *evhp =
4388 				    (struct ether_vlan_header *)mp->b_rptr;
4389 
4390 				if (ntohs(evhp->ether_tpid) == ETHERTYPE_VLAN)
4391 					hdrsize = sizeof (*evhp);
4392 				else
4393 					hdrsize = sizeof (struct ether_header);
4394 			} else {
4395 				mac_header_info_t	mhi;
4396 
4397 				err = mac_header_info((mac_handle_t)mip,
4398 				    mp, &mhi);
4399 				if (err == 0)
4400 					hdrsize = mhi.mhi_hdrsize;
4401 			}
4402 
4403 			/*
4404 			 * Got a matching flow. It's either another
4405 			 * MAC client, or a broadcast/multicast flow.
4406 			 * Make sure the packet size is within the
4407 			 * allowed size. If not drop the packet and
4408 			 * move to next packet.
4409 			 */
4410 			if (err != 0 ||
4411 			    (pkt_size - hdrsize) > mip->mi_sdu_max) {
4412 				oerrors++;
4413 				DTRACE_PROBE2(loopback__drop, size_t, pkt_size,
4414 				    mblk_t *, mp);
4415 				freemsg(mp);
4416 				mp = next;
4417 				FLOW_REFRELE(dst_flow_ent);
4418 				continue;
4419 			}
4420 			flow_cookie = mac_flow_get_client_cookie(dst_flow_ent);
4421 			if (flow_cookie != NULL) {
4422 				/*
4423 				 * The vnic_bcast_send function expects
4424 				 * to receive the sender MAC client
4425 				 * as value for arg2.
4426 				 */
4427 				mac_bcast_send(flow_cookie, src_mcip, mp,
4428 				    B_TRUE);
4429 			} else {
4430 				/*
4431 				 * loopback the packet to a local MAC
4432 				 * client. We force a context switch
4433 				 * if both source and destination MAC
4434 				 * clients are used by IP, i.e.
4435 				 * bypass is set.
4436 				 */
4437 				boolean_t do_switch;
4438 				mac_client_impl_t *dst_mcip =
4439 				    dst_flow_ent->fe_mcip;
4440 
4441 				/*
4442 				 * Check if there are promiscuous mode
4443 				 * callbacks defined. This check is
4444 				 * done here in the 'else' case and
4445 				 * not in other cases because this
4446 				 * path is for local loopback
4447 				 * communication which does not go
4448 				 * through MAC_TX(). For paths that go
4449 				 * through MAC_TX(), the promisc_list
4450 				 * check is done inside the MAC_TX()
4451 				 * macro.
4452 				 */
4453 				if (mip->mi_promisc_list != NULL)
4454 					mac_promisc_dispatch(mip, mp, src_mcip);
4455 
4456 				do_switch = ((src_mcip->mci_state_flags &
4457 				    dst_mcip->mci_state_flags &
4458 				    MCIS_CLIENT_POLL_CAPABLE) != 0);
4459 
4460 				if ((mp1 = mac_fix_cksum(mp)) != NULL) {
4461 					(dst_flow_ent->fe_cb_fn)(
4462 					    dst_flow_ent->fe_cb_arg1,
4463 					    dst_flow_ent->fe_cb_arg2,
4464 					    mp1, do_switch);
4465 				}
4466 			}
4467 			FLOW_REFRELE(dst_flow_ent);
4468 		} else {
4469 			/*
4470 			 * Unknown destination, send via the underlying
4471 			 * NIC.
4472 			 */
4473 			MAC_TX(mip, ring, mp, src_mcip);
4474 			if (mp != NULL) {
4475 				/*
4476 				 * Adjust for the last packet that
4477 				 * could not be transmitted
4478 				 */
4479 				opackets--;
4480 				obytes -= pkt_size;
4481 				mp->b_next = next;
4482 				break;
4483 			}
4484 		}
4485 		mp = next;
4486 	}
4487 
4488 done:
4489 	stats->mts_obytes = obytes;
4490 	stats->mts_opackets = opackets;
4491 	stats->mts_oerrors = oerrors;
4492 	return (mp);
4493 }
4494 
4495 /*
4496  * mac_tx_srs_ring_present
4497  *
4498  * Returns whether the specified ring is part of the specified SRS.
4499  */
4500 boolean_t
mac_tx_srs_ring_present(mac_soft_ring_set_t * srs,mac_ring_t * tx_ring)4501 mac_tx_srs_ring_present(mac_soft_ring_set_t *srs, mac_ring_t *tx_ring)
4502 {
4503 	int i;
4504 	mac_soft_ring_t *soft_ring;
4505 
4506 	if (srs->srs_tx.st_arg2 == tx_ring)
4507 		return (B_TRUE);
4508 
4509 	for (i = 0; i < srs->srs_tx_ring_count; i++) {
4510 		soft_ring =  srs->srs_tx_soft_rings[i];
4511 		if (soft_ring->s_ring_tx_arg2 == tx_ring)
4512 			return (B_TRUE);
4513 	}
4514 
4515 	return (B_FALSE);
4516 }
4517 
4518 /*
4519  * mac_tx_srs_get_soft_ring
4520  *
4521  * Returns the TX soft ring associated with the given ring, if present.
4522  */
4523 mac_soft_ring_t *
mac_tx_srs_get_soft_ring(mac_soft_ring_set_t * srs,mac_ring_t * tx_ring)4524 mac_tx_srs_get_soft_ring(mac_soft_ring_set_t *srs, mac_ring_t *tx_ring)
4525 {
4526 	int		i;
4527 	mac_soft_ring_t	*soft_ring;
4528 
4529 	if (srs->srs_tx.st_arg2 == tx_ring)
4530 		return (NULL);
4531 
4532 	for (i = 0; i < srs->srs_tx_ring_count; i++) {
4533 		soft_ring =  srs->srs_tx_soft_rings[i];
4534 		if (soft_ring->s_ring_tx_arg2 == tx_ring)
4535 			return (soft_ring);
4536 	}
4537 
4538 	return (NULL);
4539 }
4540 
4541 /*
4542  * mac_tx_srs_wakeup
4543  *
4544  * Called when Tx desc become available. Wakeup the appropriate worker
4545  * thread after resetting the SRS_TX_BLOCKED/S_RING_BLOCK bit in the
4546  * state field.
4547  */
4548 void
mac_tx_srs_wakeup(mac_soft_ring_set_t * mac_srs,mac_ring_handle_t ring)4549 mac_tx_srs_wakeup(mac_soft_ring_set_t *mac_srs, mac_ring_handle_t ring)
4550 {
4551 	int i;
4552 	mac_soft_ring_t *sringp;
4553 	mac_srs_tx_t *srs_tx = &mac_srs->srs_tx;
4554 
4555 	mutex_enter(&mac_srs->srs_lock);
4556 	/*
4557 	 * srs_tx_ring_count == 0 is the single ring mode case. In
4558 	 * this mode, there will not be Tx soft rings associated
4559 	 * with the SRS.
4560 	 */
4561 	if (!MAC_TX_SOFT_RINGS(mac_srs)) {
4562 		if (srs_tx->st_arg2 == ring &&
4563 		    mac_srs->srs_state & SRS_TX_BLOCKED) {
4564 			mac_srs->srs_state &= ~SRS_TX_BLOCKED;
4565 			srs_tx->st_stat.mts_unblockcnt++;
4566 			cv_signal(&mac_srs->srs_async);
4567 		}
4568 		/*
4569 		 * A wakeup can come before tx_srs_drain() could
4570 		 * grab srs lock and set SRS_TX_BLOCKED. So
4571 		 * always set woken_up flag when we come here.
4572 		 */
4573 		srs_tx->st_woken_up = B_TRUE;
4574 		mutex_exit(&mac_srs->srs_lock);
4575 		return;
4576 	}
4577 
4578 	/*
4579 	 * If you are here, it is for FANOUT, BW_FANOUT,
4580 	 * AGGR_MODE or AGGR_BW_MODE case
4581 	 */
4582 	for (i = 0; i < mac_srs->srs_tx_ring_count; i++) {
4583 		sringp = mac_srs->srs_tx_soft_rings[i];
4584 		mutex_enter(&sringp->s_ring_lock);
4585 		if (sringp->s_ring_tx_arg2 == ring) {
4586 			if (sringp->s_ring_state & S_RING_BLOCK) {
4587 				sringp->s_ring_state &= ~S_RING_BLOCK;
4588 				sringp->s_st_stat.mts_unblockcnt++;
4589 				cv_signal(&sringp->s_ring_async);
4590 			}
4591 			sringp->s_ring_tx_woken_up = B_TRUE;
4592 		}
4593 		mutex_exit(&sringp->s_ring_lock);
4594 	}
4595 	mutex_exit(&mac_srs->srs_lock);
4596 }
4597 
4598 /*
4599  * Once the driver is done draining, send a MAC_NOTE_TX notification to unleash
4600  * the blocked clients again.
4601  */
4602 void
mac_tx_notify(mac_impl_t * mip)4603 mac_tx_notify(mac_impl_t *mip)
4604 {
4605 	i_mac_notify(mip, MAC_NOTE_TX);
4606 }
4607 
4608 /*
4609  * RX SOFTRING RELATED FUNCTIONS
4610  *
4611  * These functions really belong in mac_soft_ring.c and here for
4612  * a short period.
4613  */
4614 
4615 #define	SOFT_RING_ENQUEUE_CHAIN(ringp, mp, tail, cnt, sz) {	       	\
4616 	/*								\
4617 	 * Enqueue our mblk chain.					\
4618 	 */								\
4619 	ASSERT(MUTEX_HELD(&(ringp)->s_ring_lock));			\
4620 									\
4621 	if ((ringp)->s_ring_last != NULL)				\
4622 		(ringp)->s_ring_last->b_next = (mp);			\
4623 	else								\
4624 		(ringp)->s_ring_first = (mp);				\
4625 	(ringp)->s_ring_last = (tail);					\
4626 	(ringp)->s_ring_count += (cnt);					\
4627 	ASSERT((ringp)->s_ring_count > 0);				\
4628 	if ((ringp)->s_ring_type & ST_RING_BW_CTL) {			\
4629 		(ringp)->s_ring_size += sz;				\
4630 	}								\
4631 }
4632 
4633 /*
4634  * Default entry point to deliver a packet chain to a MAC client.
4635  * If the MAC client has flows, do the classification with these
4636  * flows as well.
4637  */
4638 /* ARGSUSED */
4639 void
mac_rx_deliver(void * arg1,mac_resource_handle_t mrh,mblk_t * mp_chain,mac_header_info_t * arg3)4640 mac_rx_deliver(void *arg1, mac_resource_handle_t mrh, mblk_t *mp_chain,
4641     mac_header_info_t *arg3)
4642 {
4643 	mac_client_impl_t *mcip = arg1;
4644 
4645 	if (mcip->mci_nvids == 1 &&
4646 	    !(mcip->mci_state_flags & MCIS_STRIP_DISABLE)) {
4647 		/*
4648 		 * If the client has exactly one VID associated with it
4649 		 * and striping of VLAN header is not disabled,
4650 		 * remove the VLAN tag from the packet before
4651 		 * passing it on to the client's receive callback.
4652 		 * Note that this needs to be done after we dispatch
4653 		 * the packet to the promiscuous listeners of the
4654 		 * client, since they expect to see the whole
4655 		 * frame including the VLAN headers.
4656 		 */
4657 		mp_chain = mac_strip_vlan_tag_chain(mp_chain);
4658 	}
4659 
4660 	mcip->mci_rx_fn(mcip->mci_rx_arg, mrh, mp_chain, B_FALSE);
4661 }
4662 
4663 /*
4664  * mac_rx_soft_ring_process
4665  *
4666  * process a chain for a given soft ring. The number of packets queued
4667  * in the SRS and its associated soft rings (including this one) is
4668  * very small (tracked by srs_poll_pkt_cnt), then allow the entering
4669  * thread (interrupt or poll thread) to do inline processing. This
4670  * helps keep the latency down under low load.
4671  *
4672  * The proc and arg for each mblk is already stored in the mblk in
4673  * appropriate places.
4674  */
4675 /* ARGSUSED */
4676 void
mac_rx_soft_ring_process(mac_client_impl_t * mcip,mac_soft_ring_t * ringp,mblk_t * mp_chain,mblk_t * tail,int cnt,size_t sz)4677 mac_rx_soft_ring_process(mac_client_impl_t *mcip, mac_soft_ring_t *ringp,
4678     mblk_t *mp_chain, mblk_t *tail, int cnt, size_t sz)
4679 {
4680 	mac_direct_rx_t		proc;
4681 	void			*arg1;
4682 	mac_resource_handle_t	arg2;
4683 	mac_soft_ring_set_t	*mac_srs = ringp->s_ring_set;
4684 
4685 	ASSERT(ringp != NULL);
4686 	ASSERT(mp_chain != NULL);
4687 	ASSERT(tail != NULL);
4688 	ASSERT(MUTEX_NOT_HELD(&ringp->s_ring_lock));
4689 
4690 	mutex_enter(&ringp->s_ring_lock);
4691 	ringp->s_ring_total_inpkt += cnt;
4692 	ringp->s_ring_total_rbytes += sz;
4693 	if ((mac_srs->srs_rx.sr_poll_pkt_cnt <= 1) &&
4694 	    !(ringp->s_ring_type & ST_RING_WORKER_ONLY)) {
4695 		/* If on processor or blanking on, then enqueue and return */
4696 		if (ringp->s_ring_state & S_RING_BLANK ||
4697 		    ringp->s_ring_state & S_RING_PROC) {
4698 			SOFT_RING_ENQUEUE_CHAIN(ringp, mp_chain, tail, cnt, sz);
4699 			mutex_exit(&ringp->s_ring_lock);
4700 			return;
4701 		}
4702 		proc = ringp->s_ring_rx_func;
4703 		arg1 = ringp->s_ring_rx_arg1;
4704 		arg2 = ringp->s_ring_rx_arg2;
4705 		/*
4706 		 * See if anything is already queued. If we are the
4707 		 * first packet, do inline processing else queue the
4708 		 * packet and do the drain.
4709 		 */
4710 		if (ringp->s_ring_first == NULL) {
4711 			/*
4712 			 * Fast-path, ok to process and nothing queued.
4713 			 */
4714 			ringp->s_ring_run = curthread;
4715 			ringp->s_ring_state |= (S_RING_PROC);
4716 
4717 			mutex_exit(&ringp->s_ring_lock);
4718 
4719 			/*
4720 			 * We are the chain of 1 packet so
4721 			 * go through this fast path.
4722 			 */
4723 			ASSERT(mp_chain->b_next == NULL);
4724 
4725 			(*proc)(arg1, arg2, mp_chain, NULL);
4726 
4727 			ASSERT(MUTEX_NOT_HELD(&ringp->s_ring_lock));
4728 			/*
4729 			 * If we have a soft ring set which is doing
4730 			 * bandwidth control, we need to decrement
4731 			 * srs_size and count so it the SRS can have a
4732 			 * accurate idea of what is the real data
4733 			 * queued between SRS and its soft rings. We
4734 			 * decrement the counters only when the packet
4735 			 * gets processed by both SRS and the soft ring.
4736 			 */
4737 			mutex_enter(&mac_srs->srs_lock);
4738 			MAC_UPDATE_SRS_COUNT_LOCKED(mac_srs, cnt);
4739 			MAC_UPDATE_SRS_SIZE_LOCKED(mac_srs, sz);
4740 			mutex_exit(&mac_srs->srs_lock);
4741 
4742 			mutex_enter(&ringp->s_ring_lock);
4743 			ringp->s_ring_run = NULL;
4744 			ringp->s_ring_state &= ~S_RING_PROC;
4745 			if (ringp->s_ring_state & S_RING_CLIENT_WAIT)
4746 				cv_signal(&ringp->s_ring_client_cv);
4747 
4748 			if ((ringp->s_ring_first == NULL) ||
4749 			    (ringp->s_ring_state & S_RING_BLANK)) {
4750 				/*
4751 				 * We processed inline our packet and
4752 				 * nothing new has arrived or our
4753 				 * receiver doesn't want to receive
4754 				 * any packets. We are done.
4755 				 */
4756 				mutex_exit(&ringp->s_ring_lock);
4757 				return;
4758 			}
4759 		} else {
4760 			SOFT_RING_ENQUEUE_CHAIN(ringp,
4761 			    mp_chain, tail, cnt, sz);
4762 		}
4763 
4764 		/*
4765 		 * We are here because either we couldn't do inline
4766 		 * processing (because something was already
4767 		 * queued), or we had a chain of more than one
4768 		 * packet, or something else arrived after we were
4769 		 * done with inline processing.
4770 		 */
4771 		ASSERT(MUTEX_HELD(&ringp->s_ring_lock));
4772 		ASSERT(ringp->s_ring_first != NULL);
4773 
4774 		ringp->s_ring_drain_func(ringp);
4775 		mutex_exit(&ringp->s_ring_lock);
4776 		return;
4777 	} else {
4778 		/* ST_RING_WORKER_ONLY case */
4779 		SOFT_RING_ENQUEUE_CHAIN(ringp, mp_chain, tail, cnt, sz);
4780 		mac_soft_ring_worker_wakeup(ringp);
4781 		mutex_exit(&ringp->s_ring_lock);
4782 	}
4783 }
4784 
4785 /*
4786  * TX SOFTRING RELATED FUNCTIONS
4787  *
4788  * These functions really belong in mac_soft_ring.c and here for
4789  * a short period.
4790  */
4791 
4792 #define	TX_SOFT_RING_ENQUEUE_CHAIN(ringp, mp, tail, cnt, sz) {	       	\
4793 	ASSERT(MUTEX_HELD(&ringp->s_ring_lock));			\
4794 	ringp->s_ring_state |= S_RING_ENQUEUED;				\
4795 	SOFT_RING_ENQUEUE_CHAIN(ringp, mp_chain, tail, cnt, sz);	\
4796 }
4797 
4798 /*
4799  * mac_tx_sring_queued
4800  *
4801  * When we are out of transmit descriptors and we already have a
4802  * queue that exceeds hiwat (or the client called us with
4803  * MAC_TX_NO_ENQUEUE or MAC_DROP_ON_NO_DESC flag), return the
4804  * soft ring pointer as the opaque cookie for the client enable
4805  * flow control.
4806  */
4807 static mac_tx_cookie_t
mac_tx_sring_enqueue(mac_soft_ring_t * ringp,mblk_t * mp_chain,uint16_t flag,mblk_t ** ret_mp)4808 mac_tx_sring_enqueue(mac_soft_ring_t *ringp, mblk_t *mp_chain, uint16_t flag,
4809     mblk_t **ret_mp)
4810 {
4811 	int cnt;
4812 	size_t sz;
4813 	mblk_t *tail;
4814 	mac_soft_ring_set_t *mac_srs = ringp->s_ring_set;
4815 	mac_tx_cookie_t cookie = NULL;
4816 	boolean_t wakeup_worker = B_TRUE;
4817 
4818 	ASSERT(MUTEX_HELD(&ringp->s_ring_lock));
4819 	MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz);
4820 	if (flag & MAC_DROP_ON_NO_DESC) {
4821 		mac_pkt_drop(NULL, NULL, mp_chain, B_FALSE);
4822 		/* increment freed stats */
4823 		ringp->s_ring_drops += cnt;
4824 		cookie = (mac_tx_cookie_t)ringp;
4825 	} else {
4826 		if (ringp->s_ring_first != NULL)
4827 			wakeup_worker = B_FALSE;
4828 
4829 		if (flag & MAC_TX_NO_ENQUEUE) {
4830 			/*
4831 			 * If QUEUED is not set, queue the packet
4832 			 * and let mac_tx_soft_ring_drain() set
4833 			 * the TX_BLOCKED bit for the reasons
4834 			 * explained above. Otherwise, return the
4835 			 * mblks.
4836 			 */
4837 			if (wakeup_worker) {
4838 				TX_SOFT_RING_ENQUEUE_CHAIN(ringp,
4839 				    mp_chain, tail, cnt, sz);
4840 			} else {
4841 				ringp->s_ring_state |= S_RING_WAKEUP_CLIENT;
4842 				cookie = (mac_tx_cookie_t)ringp;
4843 				*ret_mp = mp_chain;
4844 			}
4845 		} else {
4846 			boolean_t enqueue = B_TRUE;
4847 
4848 			if (ringp->s_ring_count > ringp->s_ring_tx_hiwat) {
4849 				/*
4850 				 * flow-controlled. Store ringp in cookie
4851 				 * so that it can be returned as
4852 				 * mac_tx_cookie_t to client
4853 				 */
4854 				ringp->s_ring_state |= S_RING_TX_HIWAT;
4855 				cookie = (mac_tx_cookie_t)ringp;
4856 				ringp->s_ring_hiwat_cnt++;
4857 				if (ringp->s_ring_count >
4858 				    ringp->s_ring_tx_max_q_cnt) {
4859 					/* increment freed stats */
4860 					ringp->s_ring_drops += cnt;
4861 					/*
4862 					 * b_prev may be set to the fanout hint
4863 					 * hence can't use freemsg directly
4864 					 */
4865 					mac_pkt_drop(NULL, NULL,
4866 					    mp_chain, B_FALSE);
4867 					DTRACE_PROBE1(tx_queued_hiwat,
4868 					    mac_soft_ring_t *, ringp);
4869 					enqueue = B_FALSE;
4870 				}
4871 			}
4872 			if (enqueue) {
4873 				TX_SOFT_RING_ENQUEUE_CHAIN(ringp, mp_chain,
4874 				    tail, cnt, sz);
4875 			}
4876 		}
4877 		if (wakeup_worker)
4878 			cv_signal(&ringp->s_ring_async);
4879 	}
4880 	return (cookie);
4881 }
4882 
4883 
4884 /*
4885  * mac_tx_soft_ring_process
4886  *
4887  * This routine is called when fanning out outgoing traffic among
4888  * multipe Tx rings.
4889  * Note that a soft ring is associated with a h/w Tx ring.
4890  */
4891 mac_tx_cookie_t
mac_tx_soft_ring_process(mac_soft_ring_t * ringp,mblk_t * mp_chain,uint16_t flag,mblk_t ** ret_mp)4892 mac_tx_soft_ring_process(mac_soft_ring_t *ringp, mblk_t *mp_chain,
4893     uint16_t flag, mblk_t **ret_mp)
4894 {
4895 	mac_soft_ring_set_t *mac_srs = ringp->s_ring_set;
4896 	int	cnt;
4897 	size_t	sz;
4898 	mblk_t	*tail;
4899 	mac_tx_cookie_t cookie = NULL;
4900 
4901 	ASSERT(ringp != NULL);
4902 	ASSERT(mp_chain != NULL);
4903 	ASSERT(MUTEX_NOT_HELD(&ringp->s_ring_lock));
4904 	/*
4905 	 * The following modes can come here: SRS_TX_BW_FANOUT,
4906 	 * SRS_TX_FANOUT, SRS_TX_AGGR, SRS_TX_BW_AGGR.
4907 	 */
4908 	ASSERT(MAC_TX_SOFT_RINGS(mac_srs));
4909 	ASSERT(mac_srs->srs_tx.st_mode == SRS_TX_FANOUT ||
4910 	    mac_srs->srs_tx.st_mode == SRS_TX_BW_FANOUT ||
4911 	    mac_srs->srs_tx.st_mode == SRS_TX_AGGR ||
4912 	    mac_srs->srs_tx.st_mode == SRS_TX_BW_AGGR);
4913 
4914 	if (ringp->s_ring_type & ST_RING_WORKER_ONLY) {
4915 		/* Serialization mode */
4916 
4917 		mutex_enter(&ringp->s_ring_lock);
4918 		if (ringp->s_ring_count > ringp->s_ring_tx_hiwat) {
4919 			cookie = mac_tx_sring_enqueue(ringp, mp_chain,
4920 			    flag, ret_mp);
4921 			mutex_exit(&ringp->s_ring_lock);
4922 			return (cookie);
4923 		}
4924 		MAC_COUNT_CHAIN(mac_srs, mp_chain, tail, cnt, sz);
4925 		TX_SOFT_RING_ENQUEUE_CHAIN(ringp, mp_chain, tail, cnt, sz);
4926 		if (ringp->s_ring_state & (S_RING_BLOCK | S_RING_PROC)) {
4927 			/*
4928 			 * If ring is blocked due to lack of Tx
4929 			 * descs, just return. Worker thread
4930 			 * will get scheduled when Tx desc's
4931 			 * become available.
4932 			 */
4933 			mutex_exit(&ringp->s_ring_lock);
4934 			return (cookie);
4935 		}
4936 		mac_soft_ring_worker_wakeup(ringp);
4937 		mutex_exit(&ringp->s_ring_lock);
4938 		return (cookie);
4939 	} else {
4940 		/* Default fanout mode */
4941 		/*
4942 		 * S_RING_BLOCKED is set when underlying NIC runs
4943 		 * out of Tx descs and messages start getting
4944 		 * queued. It won't get reset until
4945 		 * tx_srs_drain() completely drains out the
4946 		 * messages.
4947 		 */
4948 		mac_tx_stats_t		stats;
4949 
4950 		if (ringp->s_ring_state & S_RING_ENQUEUED) {
4951 			/* Tx descs/resources not available */
4952 			mutex_enter(&ringp->s_ring_lock);
4953 			if (ringp->s_ring_state & S_RING_ENQUEUED) {
4954 				cookie = mac_tx_sring_enqueue(ringp, mp_chain,
4955 				    flag, ret_mp);
4956 				mutex_exit(&ringp->s_ring_lock);
4957 				return (cookie);
4958 			}
4959 			/*
4960 			 * While we were computing mblk count, the
4961 			 * flow control condition got relieved.
4962 			 * Continue with the transmission.
4963 			 */
4964 			mutex_exit(&ringp->s_ring_lock);
4965 		}
4966 
4967 		mp_chain = mac_tx_send(ringp->s_ring_tx_arg1,
4968 		    ringp->s_ring_tx_arg2, mp_chain, &stats);
4969 
4970 		/*
4971 		 * Multiple threads could be here sending packets.
4972 		 * Under such conditions, it is not possible to
4973 		 * automically set S_RING_BLOCKED bit to indicate
4974 		 * out of tx desc condition. To atomically set
4975 		 * this, we queue the returned packet and do
4976 		 * the setting of S_RING_BLOCKED in
4977 		 * mac_tx_soft_ring_drain().
4978 		 */
4979 		if (mp_chain != NULL) {
4980 			mutex_enter(&ringp->s_ring_lock);
4981 			cookie =
4982 			    mac_tx_sring_enqueue(ringp, mp_chain, flag, ret_mp);
4983 			mutex_exit(&ringp->s_ring_lock);
4984 			return (cookie);
4985 		}
4986 		SRS_TX_STATS_UPDATE(mac_srs, &stats);
4987 		SOFTRING_TX_STATS_UPDATE(ringp, &stats);
4988 
4989 		return (NULL);
4990 	}
4991 }
4992