xref: /titanic_41/usr/src/uts/common/io/mac/mac_datapath_setup.c (revision 657f87de670449e1422db4f51fb2880a7cb69d5a)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
23  */
24 
25 #include <sys/types.h>
26 #include <sys/callb.h>
27 #include <sys/cpupart.h>
28 #include <sys/pool.h>
29 #include <sys/pool_pset.h>
30 #include <sys/sdt.h>
31 #include <sys/strsubr.h>
32 #include <sys/strsun.h>
33 #include <sys/vlan.h>
34 #include <inet/ipsec_impl.h>
35 #include <inet/ip_impl.h>
36 #include <inet/sadb.h>
37 #include <inet/ipsecesp.h>
38 #include <inet/ipsecah.h>
39 
40 #include <sys/mac_impl.h>
41 #include <sys/mac_client_impl.h>
42 #include <sys/mac_client_priv.h>
43 #include <sys/mac_soft_ring.h>
44 #include <sys/mac_flow_impl.h>
45 #include <sys/mac_stat.h>
46 
47 static void mac_srs_soft_rings_signal(mac_soft_ring_set_t *, uint_t);
48 static void mac_srs_update_fanout_list(mac_soft_ring_set_t *);
49 static void mac_srs_poll_unbind(mac_soft_ring_set_t *);
50 static void mac_srs_worker_unbind(mac_soft_ring_set_t *);
51 static void mac_srs_soft_rings_quiesce(mac_soft_ring_set_t *, uint_t);
52 
53 static int mac_srs_cpu_setup(cpu_setup_t, int, void *);
54 static void mac_srs_worker_bind(mac_soft_ring_set_t *, processorid_t);
55 static void mac_srs_poll_bind(mac_soft_ring_set_t *, processorid_t);
56 static void mac_srs_threads_unbind(mac_soft_ring_set_t *);
57 static void mac_srs_add_glist(mac_soft_ring_set_t *);
58 static void mac_srs_remove_glist(mac_soft_ring_set_t *);
59 static void mac_srs_fanout_list_free(mac_soft_ring_set_t *);
60 static void mac_soft_ring_remove(mac_soft_ring_set_t *, mac_soft_ring_t *);
61 
62 static int mac_compute_soft_ring_count(flow_entry_t *, int, int);
63 static void mac_walk_srs_and_bind(int);
64 static void mac_walk_srs_and_unbind(int);
65 
66 extern boolean_t mac_latency_optimize;
67 
68 static kmem_cache_t *mac_srs_cache;
69 kmem_cache_t *mac_soft_ring_cache;
70 
71 /*
72  * The duration in msec we wait before signalling the soft ring
73  * worker thread in case packets get queued.
74  */
75 uint32_t mac_soft_ring_worker_wait = 0;
76 
77 /*
78  * A global tunable for turning polling on/off. By default, dynamic
79  * polling is always on and is always very beneficial. It should be
80  * turned off with absolute care and for the rare workload (very
81  * low latency sensitive traffic).
82  */
83 int mac_poll_enable = B_TRUE;
84 
85 /*
86  * Need to set mac_soft_ring_max_q_cnt based on bandwidth and perhaps latency.
87  * Large values could end up in consuming lot of system memory and cause
88  * system hang.
89  */
90 int mac_soft_ring_max_q_cnt = 1024;
91 int mac_soft_ring_min_q_cnt = 256;
92 int mac_soft_ring_poll_thres = 16;
93 
94 boolean_t mac_tx_serialize = B_FALSE;
95 
96 /*
97  * mac_tx_srs_hiwat is the queue depth threshold at which callers of
98  * mac_tx() will be notified of flow control condition.
99  *
100  * TCP does not honour flow control condition sent up by mac_tx().
101  * Thus provision is made for TCP to allow more packets to be queued
102  * in SRS upto a maximum of mac_tx_srs_max_q_cnt.
103  *
104  * Note that mac_tx_srs_hiwat is always be lesser than
105  * mac_tx_srs_max_q_cnt.
106  */
107 uint32_t mac_tx_srs_max_q_cnt = 100000;
108 uint32_t mac_tx_srs_hiwat = 1000;
109 
110 /*
111  * mac_rx_soft_ring_count, mac_soft_ring_10gig_count:
112  *
113  * Global tunables that determines the number of soft rings to be used for
114  * fanning out incoming traffic on a link. These count will be used only
115  * when no explicit set of CPUs was assigned to the data-links.
116  *
117  * mac_rx_soft_ring_count tunable will come into effect only if
118  * mac_soft_ring_enable is set. mac_soft_ring_enable is turned on by
119  * default only for sun4v platforms.
120  *
121  * mac_rx_soft_ring_10gig_count will come into effect if you are running on a
122  * 10Gbps link and is not dependent upon mac_soft_ring_enable.
123  *
124  * The number of soft rings for fanout for a link or a flow is determined
125  * by mac_compute_soft_ring_count() routine. This routine will take into
126  * account mac_soft_ring_enable, mac_rx_soft_ring_count and
127  * mac_rx_soft_ring_10gig_count to determine the soft ring count for a link.
128  *
129  * If a bandwidth is specified, the determination of the number of soft
130  * rings is based on specified bandwidth, CPU speed and number of CPUs in
131  * the system.
132  */
133 uint_t mac_rx_soft_ring_count = 8;
134 uint_t mac_rx_soft_ring_10gig_count = 8;
135 
136 /*
137  * Every Tx and Rx mac_soft_ring_set_t (mac_srs) created gets added
138  * to mac_srs_g_list and mac_srs_g_lock protects mac_srs_g_list. The
139  * list is used to walk the list of all MAC threads when a CPU is
140  * coming online or going offline.
141  */
142 static mac_soft_ring_set_t *mac_srs_g_list = NULL;
143 static krwlock_t mac_srs_g_lock;
144 
145 /*
146  * Whether the SRS threads should be bound, or not.
147  */
148 boolean_t mac_srs_thread_bind = B_TRUE;
149 
150 /*
151  * Whether Rx/Tx interrupts should be re-targeted. Disabled by default.
152  * dladm command would override this.
153  */
154 boolean_t mac_tx_intr_retarget = B_FALSE;
155 boolean_t mac_rx_intr_retarget = B_FALSE;
156 
157 /*
158  * If cpu bindings are specified by user, then Tx SRS and its soft
159  * rings should also be bound to the CPUs specified by user. The
160  * CPUs for Tx bindings are at the end of the cpu list provided by
161  * the user. If enough CPUs are not available (for Tx and Rx
162  * SRSes), then the CPUs are shared by both Tx and Rx SRSes.
163  */
164 #define	BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp) {			\
165 	processorid_t cpuid;						\
166 	int i;								\
167 	mac_soft_ring_t *softring;					\
168 	mac_cpus_t *srs_cpu;						\
169 									\
170 	srs_cpu = &mac_tx_srs->srs_cpu;					\
171 	cpuid = srs_cpu->mc_tx_fanout_cpus[0];				\
172 	mac_srs_worker_bind(mac_tx_srs, cpuid);				\
173 	if (MAC_TX_SOFT_RINGS(mac_tx_srs)) {				\
174 		for (i = 0; i < mac_tx_srs->srs_tx_ring_count; i++) {	\
175 			cpuid = srs_cpu->mc_tx_fanout_cpus[i];		\
176 			softring = mac_tx_srs->srs_tx_soft_rings[i];	\
177 			if (cpuid != -1) {				\
178 				(void) mac_soft_ring_bind(softring,	\
179 				    cpuid);				\
180 			}						\
181 		}							\
182 	}								\
183 }
184 
185 /*
186  * Re-targeting is allowed only for exclusive group or for primary.
187  */
188 #define	RETARGETABLE_CLIENT(group, mcip)				\
189 	((((group) != NULL) &&						\
190 	    ((group)->mrg_state == MAC_GROUP_STATE_RESERVED)) ||	\
191 	    mac_is_primary_client(mcip))
192 
193 #define	MAC_RING_RETARGETABLE(ring)					\
194 	(((ring) != NULL) &&						\
195 	    ((ring)->mr_info.mri_intr.mi_ddi_handle != NULL) &&		\
196 	    !((ring)->mr_info.mri_intr.mi_ddi_shared))
197 
198 
199 /* INIT and FINI ROUTINES */
200 
201 void
mac_soft_ring_init(void)202 mac_soft_ring_init(void)
203 {
204 	mac_soft_ring_cache = kmem_cache_create("mac_soft_ring_cache",
205 	    sizeof (mac_soft_ring_t), 64, NULL, NULL, NULL, NULL, NULL, 0);
206 
207 	mac_srs_cache = kmem_cache_create("mac_srs_cache",
208 	    sizeof (mac_soft_ring_set_t),
209 	    64, NULL, NULL, NULL, NULL, NULL, 0);
210 
211 	rw_init(&mac_srs_g_lock, NULL, RW_DEFAULT, NULL);
212 	mutex_enter(&cpu_lock);
213 	register_cpu_setup_func(mac_srs_cpu_setup, NULL);
214 	mutex_exit(&cpu_lock);
215 }
216 
217 void
mac_soft_ring_finish(void)218 mac_soft_ring_finish(void)
219 {
220 	mutex_enter(&cpu_lock);
221 	unregister_cpu_setup_func(mac_srs_cpu_setup, NULL);
222 	mutex_exit(&cpu_lock);
223 	rw_destroy(&mac_srs_g_lock);
224 	kmem_cache_destroy(mac_soft_ring_cache);
225 	kmem_cache_destroy(mac_srs_cache);
226 }
227 
228 static void
mac_srs_soft_rings_free(mac_soft_ring_set_t * mac_srs)229 mac_srs_soft_rings_free(mac_soft_ring_set_t *mac_srs)
230 {
231 	mac_soft_ring_t	*softring, *next, *head;
232 
233 	/*
234 	 * Synchronize with mac_walk_srs_bind/unbind which are callbacks from
235 	 * DR. The callbacks from DR are called with cpu_lock held, and hence
236 	 * can't wait to grab the mac perimeter. The soft ring list is hence
237 	 * protected for read access by srs_lock. Changing the soft ring list
238 	 * needs the mac perimeter and the srs_lock.
239 	 */
240 	mutex_enter(&mac_srs->srs_lock);
241 
242 	head = mac_srs->srs_soft_ring_head;
243 	mac_srs->srs_soft_ring_head = NULL;
244 	mac_srs->srs_soft_ring_tail = NULL;
245 	mac_srs->srs_soft_ring_count = 0;
246 
247 	mutex_exit(&mac_srs->srs_lock);
248 
249 	for (softring = head; softring != NULL; softring = next) {
250 		next = softring->s_ring_next;
251 		mac_soft_ring_free(softring);
252 	}
253 }
254 
255 static void
mac_srs_add_glist(mac_soft_ring_set_t * mac_srs)256 mac_srs_add_glist(mac_soft_ring_set_t *mac_srs)
257 {
258 	ASSERT(mac_srs->srs_next == NULL && mac_srs->srs_prev == NULL);
259 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip));
260 
261 	rw_enter(&mac_srs_g_lock, RW_WRITER);
262 	mutex_enter(&mac_srs->srs_lock);
263 
264 	ASSERT((mac_srs->srs_state & SRS_IN_GLIST) == 0);
265 
266 	if (mac_srs_g_list == NULL) {
267 		mac_srs_g_list = mac_srs;
268 	} else {
269 		mac_srs->srs_next = mac_srs_g_list;
270 		mac_srs_g_list->srs_prev = mac_srs;
271 		mac_srs->srs_prev = NULL;
272 		mac_srs_g_list = mac_srs;
273 	}
274 	mac_srs->srs_state |= SRS_IN_GLIST;
275 
276 	mutex_exit(&mac_srs->srs_lock);
277 	rw_exit(&mac_srs_g_lock);
278 }
279 
280 static void
mac_srs_remove_glist(mac_soft_ring_set_t * mac_srs)281 mac_srs_remove_glist(mac_soft_ring_set_t *mac_srs)
282 {
283 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip));
284 
285 	rw_enter(&mac_srs_g_lock, RW_WRITER);
286 	mutex_enter(&mac_srs->srs_lock);
287 
288 	ASSERT((mac_srs->srs_state & SRS_IN_GLIST) != 0);
289 
290 	if (mac_srs == mac_srs_g_list) {
291 		mac_srs_g_list = mac_srs->srs_next;
292 		if (mac_srs_g_list != NULL)
293 			mac_srs_g_list->srs_prev = NULL;
294 	} else {
295 		mac_srs->srs_prev->srs_next = mac_srs->srs_next;
296 		if (mac_srs->srs_next != NULL)
297 			mac_srs->srs_next->srs_prev = mac_srs->srs_prev;
298 	}
299 	mac_srs->srs_state &= ~SRS_IN_GLIST;
300 
301 	mutex_exit(&mac_srs->srs_lock);
302 	rw_exit(&mac_srs_g_lock);
303 }
304 
305 /* POLLING SETUP AND TEAR DOWN ROUTINES */
306 
307 /*
308  * mac_srs_client_poll_quiesce and mac_srs_client_poll_restart
309  *
310  * These routines are used to call back into the upper layer
311  * (primarily TCP squeue) to stop polling the soft rings or
312  * restart polling.
313  */
314 void
mac_srs_client_poll_quiesce(mac_client_impl_t * mcip,mac_soft_ring_set_t * mac_srs)315 mac_srs_client_poll_quiesce(mac_client_impl_t *mcip,
316     mac_soft_ring_set_t *mac_srs)
317 {
318 	mac_soft_ring_t	*softring;
319 
320 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip));
321 
322 	if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) {
323 		ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS));
324 		return;
325 	}
326 
327 	for (softring = mac_srs->srs_soft_ring_head;
328 	    softring != NULL; softring = softring->s_ring_next) {
329 		if ((softring->s_ring_type & ST_RING_TCP) &&
330 		    (softring->s_ring_rx_arg2 != NULL)) {
331 			mcip->mci_resource_quiesce(mcip->mci_resource_arg,
332 			    softring->s_ring_rx_arg2);
333 		}
334 	}
335 }
336 
337 void
mac_srs_client_poll_restart(mac_client_impl_t * mcip,mac_soft_ring_set_t * mac_srs)338 mac_srs_client_poll_restart(mac_client_impl_t *mcip,
339     mac_soft_ring_set_t *mac_srs)
340 {
341 	mac_soft_ring_t	*softring;
342 
343 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip));
344 
345 	if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) {
346 		ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS));
347 		return;
348 	}
349 
350 	for (softring = mac_srs->srs_soft_ring_head;
351 	    softring != NULL; softring = softring->s_ring_next) {
352 		if ((softring->s_ring_type & ST_RING_TCP) &&
353 		    (softring->s_ring_rx_arg2 != NULL)) {
354 			mcip->mci_resource_restart(mcip->mci_resource_arg,
355 			    softring->s_ring_rx_arg2);
356 		}
357 	}
358 }
359 
360 /*
361  * Register the given SRS and associated soft rings with the consumer and
362  * enable the polling interface used by the consumer.(i.e IP) over this
363  * SRS and associated soft rings.
364  */
365 void
mac_srs_client_poll_enable(mac_client_impl_t * mcip,mac_soft_ring_set_t * mac_srs)366 mac_srs_client_poll_enable(mac_client_impl_t *mcip,
367     mac_soft_ring_set_t *mac_srs)
368 {
369 	mac_rx_fifo_t		mrf;
370 	mac_soft_ring_t		*softring;
371 
372 	ASSERT(mac_srs->srs_mcip == mcip);
373 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip));
374 
375 	if (!(mcip->mci_state_flags & MCIS_CLIENT_POLL_CAPABLE))
376 		return;
377 
378 	bzero(&mrf, sizeof (mac_rx_fifo_t));
379 	mrf.mrf_type = MAC_RX_FIFO;
380 
381 	/*
382 	 * A SRS is capable of acting as a soft ring for cases
383 	 * where no fanout is needed. This is the case for userland
384 	 * flows.
385 	 */
386 	if (mac_srs->srs_type & SRST_NO_SOFT_RINGS)
387 		return;
388 
389 	mrf.mrf_receive = (mac_receive_t)mac_soft_ring_poll;
390 	mrf.mrf_intr_enable = (mac_intr_enable_t)mac_soft_ring_intr_enable;
391 	mrf.mrf_intr_disable = (mac_intr_disable_t)mac_soft_ring_intr_disable;
392 	mac_srs->srs_type |= SRST_CLIENT_POLL_ENABLED;
393 
394 	softring = mac_srs->srs_soft_ring_head;
395 	while (softring != NULL) {
396 		if (softring->s_ring_type & (ST_RING_TCP | ST_RING_UDP)) {
397 			/*
398 			 * TCP and UDP support DLS bypass. Squeue polling
399 			 * support implies DLS bypass since the squeue poll
400 			 * path does not have DLS processing.
401 			 */
402 			mac_soft_ring_dls_bypass(softring,
403 			    mcip->mci_direct_rx_fn, mcip->mci_direct_rx_arg);
404 		}
405 		/*
406 		 * Non-TCP protocols don't support squeues. Hence we don't
407 		 * make any ring addition callbacks for non-TCP rings
408 		 */
409 		if (!(softring->s_ring_type & ST_RING_TCP)) {
410 			softring->s_ring_rx_arg2 = NULL;
411 			softring = softring->s_ring_next;
412 			continue;
413 		}
414 		mrf.mrf_rx_arg = softring;
415 		mrf.mrf_intr_handle = (mac_intr_handle_t)softring;
416 		mrf.mrf_cpu_id = softring->s_ring_cpuid;
417 		mrf.mrf_flow_priority = mac_srs->srs_pri;
418 
419 		softring->s_ring_rx_arg2 = mcip->mci_resource_add(
420 		    mcip->mci_resource_arg, (mac_resource_t *)&mrf);
421 
422 		softring = softring->s_ring_next;
423 	}
424 }
425 
426 /*
427  * Unregister the given SRS and associated soft rings with the consumer and
428  * disable the polling interface used by the consumer.(i.e IP) over this
429  * SRS and associated soft rings.
430  */
431 void
mac_srs_client_poll_disable(mac_client_impl_t * mcip,mac_soft_ring_set_t * mac_srs)432 mac_srs_client_poll_disable(mac_client_impl_t *mcip,
433     mac_soft_ring_set_t *mac_srs)
434 {
435 	mac_soft_ring_t		*softring;
436 
437 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip));
438 
439 	/*
440 	 * A SRS is capable of acting as a soft ring for cases
441 	 * where no protocol fanout is needed. This is the case
442 	 * for userland flows. Nothing to do here.
443 	 */
444 	if (mac_srs->srs_type & SRST_NO_SOFT_RINGS)
445 		return;
446 
447 	mutex_enter(&mac_srs->srs_lock);
448 	if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) {
449 		ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS));
450 		mutex_exit(&mac_srs->srs_lock);
451 		return;
452 	}
453 	mac_srs->srs_type &= ~(SRST_CLIENT_POLL_ENABLED | SRST_DLS_BYPASS);
454 	mutex_exit(&mac_srs->srs_lock);
455 
456 	/*
457 	 * DLS bypass is now disabled in the case of both TCP and UDP.
458 	 * Reset the soft ring callbacks to the standard 'mac_rx_deliver'
459 	 * callback. In addition, in the case of TCP, invoke IP's callback
460 	 * for ring removal.
461 	 */
462 	for (softring = mac_srs->srs_soft_ring_head;
463 	    softring != NULL; softring = softring->s_ring_next) {
464 		if (!(softring->s_ring_type & (ST_RING_UDP | ST_RING_TCP)))
465 			continue;
466 
467 		if ((softring->s_ring_type & ST_RING_TCP) &&
468 		    softring->s_ring_rx_arg2 != NULL) {
469 			mcip->mci_resource_remove(mcip->mci_resource_arg,
470 			    softring->s_ring_rx_arg2);
471 		}
472 
473 		mutex_enter(&softring->s_ring_lock);
474 		while (softring->s_ring_state & S_RING_PROC) {
475 			softring->s_ring_state |= S_RING_CLIENT_WAIT;
476 			cv_wait(&softring->s_ring_client_cv,
477 			    &softring->s_ring_lock);
478 		}
479 		softring->s_ring_state &= ~S_RING_CLIENT_WAIT;
480 		softring->s_ring_rx_arg2 = NULL;
481 		softring->s_ring_rx_func = mac_rx_deliver;
482 		softring->s_ring_rx_arg1 = mcip;
483 		mutex_exit(&softring->s_ring_lock);
484 	}
485 }
486 
487 /*
488  * Enable or disable poll capability of the SRS on the underlying Rx ring.
489  *
490  * There is a need to enable or disable the poll capability of an SRS over an
491  * Rx ring depending on the number of mac clients sharing the ring and also
492  * whether user flows are configured on it. However the poll state is actively
493  * manipulated by the SRS worker and poll threads and uncoordinated changes by
494  * yet another thread to the underlying capability can surprise them leading
495  * to assert failures. Instead we quiesce the SRS, make the changes and then
496  * restart the SRS.
497  */
498 static void
mac_srs_poll_state_change(mac_soft_ring_set_t * mac_srs,boolean_t turn_off_poll_capab,mac_rx_func_t rx_func)499 mac_srs_poll_state_change(mac_soft_ring_set_t *mac_srs,
500     boolean_t turn_off_poll_capab, mac_rx_func_t rx_func)
501 {
502 	boolean_t	need_restart = B_FALSE;
503 	mac_srs_rx_t	*srs_rx = &mac_srs->srs_rx;
504 	mac_ring_t	*ring;
505 
506 	if (!SRS_QUIESCED(mac_srs)) {
507 		mac_rx_srs_quiesce(mac_srs, SRS_QUIESCE);
508 		need_restart = B_TRUE;
509 	}
510 
511 	ring = mac_srs->srs_ring;
512 	if ((ring != NULL) &&
513 	    (ring->mr_classify_type == MAC_HW_CLASSIFIER)) {
514 		if (turn_off_poll_capab)
515 			mac_srs->srs_state &= ~SRS_POLLING_CAPAB;
516 		else if (mac_poll_enable)
517 			mac_srs->srs_state |= SRS_POLLING_CAPAB;
518 	}
519 	srs_rx->sr_lower_proc = rx_func;
520 
521 	if (need_restart)
522 		mac_rx_srs_restart(mac_srs);
523 }
524 
525 /* CPU RECONFIGURATION AND FANOUT COMPUTATION ROUTINES */
526 
527 /*
528  * Return the next CPU to be used to bind a MAC kernel thread.
529  * If a cpupart is specified, the cpu chosen must be from that
530  * cpu partition.
531  */
532 static processorid_t
mac_next_bind_cpu(cpupart_t * cpupart)533 mac_next_bind_cpu(cpupart_t *cpupart)
534 {
535 	static cpu_t		*cp = NULL;
536 	cpu_t			*cp_start;
537 
538 	ASSERT(MUTEX_HELD(&cpu_lock));
539 
540 	if (cp == NULL)
541 		cp = cpu_list;
542 
543 	cp = cp->cpu_next_onln;
544 	cp_start = cp;
545 
546 	do {
547 		if ((cpupart == NULL) || (cp->cpu_part == cpupart))
548 			return (cp->cpu_id);
549 
550 	} while ((cp = cp->cpu_next_onln) != cp_start);
551 
552 	return (NULL);
553 }
554 
555 /* ARGSUSED */
556 static int
mac_srs_cpu_setup(cpu_setup_t what,int id,void * arg)557 mac_srs_cpu_setup(cpu_setup_t what, int id, void *arg)
558 {
559 	ASSERT(MUTEX_HELD(&cpu_lock));
560 	switch (what) {
561 	case CPU_CONFIG:
562 	case CPU_ON:
563 	case CPU_CPUPART_IN:
564 		mac_walk_srs_and_bind(id);
565 		break;
566 
567 	case CPU_UNCONFIG:
568 	case CPU_OFF:
569 	case CPU_CPUPART_OUT:
570 		mac_walk_srs_and_unbind(id);
571 		break;
572 
573 	default:
574 		break;
575 	}
576 	return (0);
577 }
578 
579 /*
580  * mac_compute_soft_ring_count():
581  *
582  * This routine computes the number of soft rings needed to handle incoming
583  * load given a flow_entry.
584  *
585  * The routine does the following:
586  * 1) soft rings will be created if mac_soft_ring_enable is set.
587  * 2) If the underlying link is a 10Gbps link, then soft rings will be
588  * created even if mac_soft_ring_enable is not set. The number of soft
589  * rings, so created,  will equal mac_rx_soft_ring_10gig_count.
590  * 3) On a sun4v platform (i.e., mac_soft_ring_enable is set), 2 times the
591  * mac_rx_soft_ring_10gig_count number of soft rings will be created for a
592  * 10Gbps link.
593  *
594  * If a bandwidth limit is specified, the number that gets computed is
595  * dependent upon CPU speed, the number of Rx rings configured, and
596  * the bandwidth limit.
597  * If more Rx rings are available, less number of soft rings is needed.
598  *
599  * mac_use_bw_heuristic is another "hidden" variable that can be used to
600  * override the default use of soft ring count computation. Depending upon
601  * the usefulness of it, mac_use_bw_heuristic can later be made into a
602  * data-link property or removed altogether.
603  *
604  * TODO: Cleanup and tighten some of the assumptions.
605  */
606 boolean_t mac_use_bw_heuristic = B_TRUE;
607 static int
mac_compute_soft_ring_count(flow_entry_t * flent,int rx_srs_cnt,int maxcpus)608 mac_compute_soft_ring_count(flow_entry_t *flent, int rx_srs_cnt, int maxcpus)
609 {
610 	uint64_t cpu_speed, bw = 0;
611 	int srings = 0;
612 	boolean_t bw_enabled = B_FALSE;
613 
614 	ASSERT(!(flent->fe_type & FLOW_USER));
615 	if (flent->fe_resource_props.mrp_mask & MRP_MAXBW &&
616 	    mac_use_bw_heuristic) {
617 		/* bandwidth enabled */
618 		bw_enabled = B_TRUE;
619 		bw = flent->fe_resource_props.mrp_maxbw;
620 	}
621 	if (!bw_enabled) {
622 		/* No bandwidth enabled */
623 		if (mac_soft_ring_enable)
624 			srings = mac_rx_soft_ring_count;
625 
626 		/* Is this a 10Gig link? */
627 		flent->fe_nic_speed = mac_client_stat_get(flent->fe_mcip,
628 		    MAC_STAT_IFSPEED);
629 		/* convert to Mbps */
630 		if (((flent->fe_nic_speed)/1000000) > 1000 &&
631 		    mac_rx_soft_ring_10gig_count > 0) {
632 			/* This is a 10Gig link */
633 			srings = mac_rx_soft_ring_10gig_count;
634 			/*
635 			 * Use 2 times mac_rx_soft_ring_10gig_count for
636 			 * sun4v systems.
637 			 */
638 			if (mac_soft_ring_enable)
639 				srings = srings * 2;
640 		}
641 	} else {
642 		/*
643 		 * Soft ring computation using CPU speed and specified
644 		 * bandwidth limit.
645 		 */
646 		/* Assumption: all CPUs have the same frequency */
647 		cpu_speed = (uint64_t)CPU->cpu_type_info.pi_clock;
648 
649 		/* cpu_speed is in MHz; make bw in units of Mbps.  */
650 		bw = bw/1000000;
651 
652 		if (bw >= 1000) {
653 			/*
654 			 * bw is greater than or equal to 1Gbps.
655 			 * The number of soft rings required is a function
656 			 * of bandwidth and CPU speed. To keep this simple,
657 			 * let's use this rule: 1GHz CPU can handle 1Gbps.
658 			 * If bw is less than 1 Gbps, then there is no need
659 			 * for soft rings. Assumption is that CPU speeds
660 			 * (on modern systems) are at least 1GHz.
661 			 */
662 			srings = bw/cpu_speed;
663 			if (srings <= 1 && mac_soft_ring_enable) {
664 				/*
665 				 * Give at least 2 soft rings
666 				 * for sun4v systems
667 				 */
668 				srings = 2;
669 			}
670 		}
671 	}
672 	/*
673 	 * If the flent has multiple Rx SRSs, then each SRS need not
674 	 * have that many soft rings on top of it. The number of
675 	 * soft rings for each Rx SRS is found by dividing srings by
676 	 * rx_srs_cnt.
677 	 */
678 	if (rx_srs_cnt > 1) {
679 		int remainder;
680 
681 		remainder = srings%rx_srs_cnt;
682 		srings = srings/rx_srs_cnt;
683 		if (remainder != 0)
684 			srings++;
685 		/*
686 		 * Fanning out to 1 soft ring is not very useful.
687 		 * Set it as well to 0 and mac_srs_fanout_init()
688 		 * will take care of creating a single soft ring
689 		 * for proto fanout.
690 		 */
691 		if (srings == 1)
692 			srings = 0;
693 	}
694 	/* Do some more massaging */
695 	srings = min(srings, maxcpus);
696 	srings = min(srings, MAX_SR_FANOUT);
697 	return (srings);
698 }
699 
700 /*
701  * mac_tx_cpu_init:
702  * set up CPUs for Tx interrupt re-targeting and Tx worker
703  * thread binding
704  */
705 static void
mac_tx_cpu_init(flow_entry_t * flent,mac_resource_props_t * mrp,cpupart_t * cpupart)706 mac_tx_cpu_init(flow_entry_t *flent, mac_resource_props_t *mrp,
707     cpupart_t *cpupart)
708 {
709 	mac_soft_ring_set_t *tx_srs = flent->fe_tx_srs;
710 	mac_srs_tx_t *srs_tx = &tx_srs->srs_tx;
711 	mac_cpus_t *srs_cpu = &tx_srs->srs_cpu;
712 	mac_soft_ring_t *sringp;
713 	mac_ring_t *ring;
714 	processorid_t worker_cpuid;
715 	boolean_t retargetable_client = B_FALSE;
716 	int i, j;
717 
718 	if (RETARGETABLE_CLIENT((mac_group_t *)flent->fe_tx_ring_group,
719 	    flent->fe_mcip)) {
720 		retargetable_client = B_TRUE;
721 	}
722 
723 	if (MAC_TX_SOFT_RINGS(tx_srs)) {
724 		if (mrp != NULL)
725 			j = mrp->mrp_ncpus - 1;
726 		for (i = 0; i < tx_srs->srs_tx_ring_count; i++) {
727 			if (mrp != NULL) {
728 				if (j < 0)
729 					j = mrp->mrp_ncpus - 1;
730 				worker_cpuid = mrp->mrp_cpu[j];
731 			} else {
732 				/*
733 				 * Bind interrupt to the next CPU available
734 				 * and leave the worker unbound.
735 				 */
736 				worker_cpuid = -1;
737 			}
738 			sringp = tx_srs->srs_tx_soft_rings[i];
739 			ring = (mac_ring_t *)sringp->s_ring_tx_arg2;
740 			srs_cpu->mc_tx_fanout_cpus[i] = worker_cpuid;
741 			if (MAC_RING_RETARGETABLE(ring) &&
742 			    retargetable_client) {
743 				mutex_enter(&cpu_lock);
744 				srs_cpu->mc_tx_intr_cpu[i] =
745 				    (mrp != NULL) ? mrp->mrp_cpu[j] :
746 				    (mac_tx_intr_retarget ?
747 				    mac_next_bind_cpu(cpupart) : -1);
748 				mutex_exit(&cpu_lock);
749 			} else {
750 				srs_cpu->mc_tx_intr_cpu[i] = -1;
751 			}
752 			if (mrp != NULL)
753 				j--;
754 		}
755 	} else {
756 		/* Tx mac_ring_handle_t is stored in st_arg2 */
757 		srs_cpu->mc_tx_fanout_cpus[0] =
758 		    (mrp != NULL) ? mrp->mrp_cpu[mrp->mrp_ncpus - 1] : -1;
759 		ring = (mac_ring_t *)srs_tx->st_arg2;
760 		if (MAC_RING_RETARGETABLE(ring) && retargetable_client) {
761 			mutex_enter(&cpu_lock);
762 			srs_cpu->mc_tx_intr_cpu[0] = (mrp != NULL) ?
763 			    mrp->mrp_cpu[mrp->mrp_ncpus - 1] :
764 			    (mac_tx_intr_retarget ?
765 			    mac_next_bind_cpu(cpupart) : -1);
766 			mutex_exit(&cpu_lock);
767 		} else {
768 			srs_cpu->mc_tx_intr_cpu[0] = -1;
769 		}
770 	}
771 }
772 
773 /*
774  * Assignment of user specified CPUs to a link.
775  *
776  * Minimum CPUs required to get an optimal assignmet:
777  * For each Rx SRS, atleast two CPUs are needed if mac_latency_optimize
778  * flag is set -- one for polling, one for fanout soft ring.
779  * If mac_latency_optimize is not set, then 3 CPUs are needed -- one
780  * for polling, one for SRS worker thread and one for fanout soft ring.
781  *
782  * The CPUs needed for Tx side is equal to the number of Tx rings
783  * the link is using.
784  *
785  * mac_flow_user_cpu_init() categorizes the CPU assignment depending
786  * upon the number of CPUs in 3 different buckets.
787  *
788  * In the first bucket, the most optimal case is handled. The user has
789  * passed enough number of CPUs and every thread gets its own CPU.
790  *
791  * The second and third are the sub-optimal cases. Enough CPUs are not
792  * available.
793  *
794  * The second bucket handles the case where atleast one distinct CPU is
795  * is available for each of the Rx rings (Rx SRSes) and Tx rings (Tx
796  * SRS or soft rings).
797  *
798  * In the third case (worst case scenario), specified CPU count is less
799  * than the Rx rings configured for the link. In this case, we round
800  * robin the CPUs among the Rx SRSes and Tx SRS/soft rings.
801  */
802 static void
mac_flow_user_cpu_init(flow_entry_t * flent,mac_resource_props_t * mrp)803 mac_flow_user_cpu_init(flow_entry_t *flent, mac_resource_props_t *mrp)
804 {
805 	mac_soft_ring_set_t *rx_srs, *tx_srs;
806 	int i, srs_cnt;
807 	mac_cpus_t *srs_cpu;
808 	int no_of_cpus, cpu_cnt;
809 	int rx_srs_cnt, reqd_rx_cpu_cnt;
810 	int fanout_cpu_cnt, reqd_tx_cpu_cnt;
811 	int reqd_poll_worker_cnt, fanout_cnt_per_srs;
812 	mac_resource_props_t *emrp = &flent->fe_effective_props;
813 
814 	ASSERT(mrp->mrp_fanout_mode == MCM_CPUS);
815 	/*
816 	 * The check for nbc_ncpus to be within limits for
817 	 * the user specified case was done earlier and if
818 	 * not within limits, an error would have been
819 	 * returned to the user.
820 	 */
821 	ASSERT(mrp->mrp_ncpus > 0);
822 
823 	no_of_cpus = mrp->mrp_ncpus;
824 
825 	if (mrp->mrp_rx_intr_cpu != -1) {
826 		/*
827 		 * interrupt has been re-targetted. Poll
828 		 * thread needs to be bound to interrupt
829 		 * CPU.
830 		 *
831 		 * Find where in the list is the intr
832 		 * CPU and swap it with the first one.
833 		 * We will be using the first CPU in the
834 		 * list for poll.
835 		 */
836 		for (i = 0; i < no_of_cpus; i++) {
837 			if (mrp->mrp_cpu[i] == mrp->mrp_rx_intr_cpu)
838 				break;
839 		}
840 		mrp->mrp_cpu[i] = mrp->mrp_cpu[0];
841 		mrp->mrp_cpu[0] = mrp->mrp_rx_intr_cpu;
842 	}
843 
844 	/*
845 	 * Requirements:
846 	 * The number of CPUs that each Rx ring needs is dependent
847 	 * upon mac_latency_optimize flag.
848 	 * 1) If set, atleast 2 CPUs are needed -- one for
849 	 * polling, one for fanout soft ring.
850 	 * 2) If not set, then atleast 3 CPUs are needed -- one
851 	 * for polling, one for srs worker thread, and one for
852 	 * fanout soft ring.
853 	 */
854 	rx_srs_cnt = (flent->fe_rx_srs_cnt > 1) ?
855 	    (flent->fe_rx_srs_cnt - 1) : flent->fe_rx_srs_cnt;
856 	reqd_rx_cpu_cnt = mac_latency_optimize ?
857 	    (rx_srs_cnt * 2) : (rx_srs_cnt * 3);
858 
859 	/* How many CPUs are needed for Tx side? */
860 	tx_srs = flent->fe_tx_srs;
861 	reqd_tx_cpu_cnt = MAC_TX_SOFT_RINGS(tx_srs) ?
862 	    tx_srs->srs_tx_ring_count : 1;
863 
864 	/* CPUs needed for Rx SRSes poll and worker threads */
865 	reqd_poll_worker_cnt = mac_latency_optimize ?
866 	    rx_srs_cnt : rx_srs_cnt * 2;
867 
868 	/* Has the user provided enough CPUs? */
869 	if (no_of_cpus >= (reqd_rx_cpu_cnt + reqd_tx_cpu_cnt)) {
870 		/*
871 		 * Best case scenario. There is enough CPUs. All
872 		 * Rx rings will get their own set of CPUs plus
873 		 * Tx soft rings will get their own.
874 		 */
875 		/*
876 		 * fanout_cpu_cnt is the number of CPUs available
877 		 * for Rx side fanout soft rings.
878 		 */
879 		fanout_cpu_cnt = no_of_cpus -
880 		    reqd_poll_worker_cnt - reqd_tx_cpu_cnt;
881 
882 		/*
883 		 * Divide fanout_cpu_cnt by rx_srs_cnt to find
884 		 * out how many fanout soft rings each Rx SRS
885 		 * can have.
886 		 */
887 		fanout_cnt_per_srs = fanout_cpu_cnt/rx_srs_cnt;
888 
889 		/* fanout_cnt_per_srs should not be >  MAX_SR_FANOUT */
890 		fanout_cnt_per_srs = min(fanout_cnt_per_srs, MAX_SR_FANOUT);
891 
892 		/* Do the assignment for the default Rx ring */
893 		cpu_cnt = 0;
894 		rx_srs = flent->fe_rx_srs[0];
895 		ASSERT(rx_srs->srs_ring == NULL);
896 		if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT)
897 			rx_srs->srs_fanout_state = SRS_FANOUT_REINIT;
898 		srs_cpu = &rx_srs->srs_cpu;
899 		srs_cpu->mc_ncpus = no_of_cpus;
900 		bcopy(mrp->mrp_cpu,
901 		    srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus));
902 		srs_cpu->mc_rx_fanout_cnt = fanout_cnt_per_srs;
903 		srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt++];
904 		/* Retarget the interrupt to the same CPU as the poll */
905 		srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid;
906 		srs_cpu->mc_rx_workerid = (mac_latency_optimize ?
907 		    srs_cpu->mc_rx_pollid : mrp->mrp_cpu[cpu_cnt++]);
908 		for (i = 0; i < fanout_cnt_per_srs; i++)
909 			srs_cpu->mc_rx_fanout_cpus[i] = mrp->mrp_cpu[cpu_cnt++];
910 
911 		/* Do the assignment for h/w Rx SRSes */
912 		if (flent->fe_rx_srs_cnt > 1) {
913 			cpu_cnt = 0;
914 			for (srs_cnt = 1;
915 			    srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) {
916 				rx_srs = flent->fe_rx_srs[srs_cnt];
917 				ASSERT(rx_srs->srs_ring != NULL);
918 				if (rx_srs->srs_fanout_state ==
919 				    SRS_FANOUT_INIT) {
920 					rx_srs->srs_fanout_state =
921 					    SRS_FANOUT_REINIT;
922 				}
923 				srs_cpu = &rx_srs->srs_cpu;
924 				srs_cpu->mc_ncpus = no_of_cpus;
925 				bcopy(mrp->mrp_cpu, srs_cpu->mc_cpus,
926 				    sizeof (srs_cpu->mc_cpus));
927 				srs_cpu->mc_rx_fanout_cnt = fanout_cnt_per_srs;
928 				/* The first CPU in the list is the intr CPU */
929 				srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt++];
930 				srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid;
931 				srs_cpu->mc_rx_workerid =
932 				    (mac_latency_optimize ?
933 				    srs_cpu->mc_rx_pollid :
934 				    mrp->mrp_cpu[cpu_cnt++]);
935 				for (i = 0; i < fanout_cnt_per_srs; i++) {
936 					srs_cpu->mc_rx_fanout_cpus[i] =
937 					    mrp->mrp_cpu[cpu_cnt++];
938 				}
939 				ASSERT(cpu_cnt <= no_of_cpus);
940 			}
941 		}
942 		goto tx_cpu_init;
943 	}
944 
945 	/*
946 	 * Sub-optimal case.
947 	 * We have the following information:
948 	 * no_of_cpus - no. of cpus that user passed.
949 	 * rx_srs_cnt - no. of rx rings.
950 	 * reqd_rx_cpu_cnt = mac_latency_optimize?rx_srs_cnt*2:rx_srs_cnt*3
951 	 * reqd_tx_cpu_cnt - no. of cpus reqd. for Tx side.
952 	 * reqd_poll_worker_cnt = mac_latency_optimize?rx_srs_cnt:rx_srs_cnt*2
953 	 */
954 	/*
955 	 * If we bind the Rx fanout soft rings to the same CPUs
956 	 * as poll/worker, would that be enough?
957 	 */
958 	if (no_of_cpus >= (rx_srs_cnt + reqd_tx_cpu_cnt)) {
959 		boolean_t worker_assign = B_FALSE;
960 
961 		/*
962 		 * If mac_latency_optimize is not set, are there
963 		 * enough CPUs to assign a CPU for worker also?
964 		 */
965 		if (no_of_cpus >= (reqd_poll_worker_cnt + reqd_tx_cpu_cnt))
966 			worker_assign = B_TRUE;
967 		/*
968 		 * Zero'th Rx SRS is the default Rx ring. It is not
969 		 * associated with h/w Rx ring.
970 		 */
971 		rx_srs = flent->fe_rx_srs[0];
972 		ASSERT(rx_srs->srs_ring == NULL);
973 		if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT)
974 			rx_srs->srs_fanout_state = SRS_FANOUT_REINIT;
975 		cpu_cnt = 0;
976 		srs_cpu = &rx_srs->srs_cpu;
977 		srs_cpu->mc_ncpus = no_of_cpus;
978 		bcopy(mrp->mrp_cpu,
979 		    srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus));
980 		srs_cpu->mc_rx_fanout_cnt = 1;
981 		srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt++];
982 		/* Retarget the interrupt to the same CPU as the poll */
983 		srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid;
984 		srs_cpu->mc_rx_workerid =
985 		    ((!mac_latency_optimize && worker_assign) ?
986 		    mrp->mrp_cpu[cpu_cnt++] : srs_cpu->mc_rx_pollid);
987 
988 		srs_cpu->mc_rx_fanout_cpus[0] = mrp->mrp_cpu[cpu_cnt];
989 
990 		/* Do CPU bindings for SRSes having h/w Rx rings */
991 		if (flent->fe_rx_srs_cnt > 1) {
992 			cpu_cnt = 0;
993 			for (srs_cnt = 1;
994 			    srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) {
995 				rx_srs = flent->fe_rx_srs[srs_cnt];
996 				ASSERT(rx_srs->srs_ring != NULL);
997 				if (rx_srs->srs_fanout_state ==
998 				    SRS_FANOUT_INIT) {
999 					rx_srs->srs_fanout_state =
1000 					    SRS_FANOUT_REINIT;
1001 				}
1002 				srs_cpu = &rx_srs->srs_cpu;
1003 				srs_cpu->mc_ncpus = no_of_cpus;
1004 				bcopy(mrp->mrp_cpu, srs_cpu->mc_cpus,
1005 				    sizeof (srs_cpu->mc_cpus));
1006 				srs_cpu->mc_rx_pollid =
1007 				    mrp->mrp_cpu[cpu_cnt];
1008 				srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid;
1009 				srs_cpu->mc_rx_workerid =
1010 				    ((!mac_latency_optimize && worker_assign) ?
1011 				    mrp->mrp_cpu[++cpu_cnt] :
1012 				    srs_cpu->mc_rx_pollid);
1013 				srs_cpu->mc_rx_fanout_cnt = 1;
1014 				srs_cpu->mc_rx_fanout_cpus[0] =
1015 				    mrp->mrp_cpu[cpu_cnt];
1016 				cpu_cnt++;
1017 				ASSERT(cpu_cnt <= no_of_cpus);
1018 			}
1019 		}
1020 		goto tx_cpu_init;
1021 	}
1022 
1023 	/*
1024 	 * Real sub-optimal case. Not enough CPUs for poll and
1025 	 * Tx soft rings. Do a round robin assignment where
1026 	 * each Rx SRS will get the same CPU for poll, worker
1027 	 * and fanout soft ring.
1028 	 */
1029 	cpu_cnt = 0;
1030 	for (srs_cnt = 0; srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) {
1031 		rx_srs = flent->fe_rx_srs[srs_cnt];
1032 		srs_cpu = &rx_srs->srs_cpu;
1033 		if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT)
1034 			rx_srs->srs_fanout_state = SRS_FANOUT_REINIT;
1035 		srs_cpu->mc_ncpus = no_of_cpus;
1036 		bcopy(mrp->mrp_cpu,
1037 		    srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus));
1038 		srs_cpu->mc_rx_fanout_cnt = 1;
1039 		srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt];
1040 		/* Retarget the interrupt to the same CPU as the poll */
1041 		srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid;
1042 		srs_cpu->mc_rx_workerid = mrp->mrp_cpu[cpu_cnt];
1043 		srs_cpu->mc_rx_fanout_cpus[0] = mrp->mrp_cpu[cpu_cnt];
1044 		if (++cpu_cnt >= no_of_cpus)
1045 			cpu_cnt = 0;
1046 	}
1047 
1048 tx_cpu_init:
1049 	mac_tx_cpu_init(flent, mrp, NULL);
1050 
1051 	/*
1052 	 * Copy the user specified CPUs to the effective CPUs
1053 	 */
1054 	for (i = 0; i < mrp->mrp_ncpus; i++) {
1055 		emrp->mrp_cpu[i] = mrp->mrp_cpu[i];
1056 	}
1057 	emrp->mrp_ncpus = mrp->mrp_ncpus;
1058 	emrp->mrp_mask = mrp->mrp_mask;
1059 	bzero(emrp->mrp_pool, MAXPATHLEN);
1060 }
1061 
1062 /*
1063  * mac_flow_cpu_init():
1064  *
1065  * Each SRS has a mac_cpu_t structure, srs_cpu. This routine fills in
1066  * the CPU binding information in srs_cpu for all Rx SRSes associated
1067  * with a flent.
1068  */
1069 static void
mac_flow_cpu_init(flow_entry_t * flent,cpupart_t * cpupart)1070 mac_flow_cpu_init(flow_entry_t *flent, cpupart_t *cpupart)
1071 {
1072 	mac_soft_ring_set_t *rx_srs;
1073 	processorid_t cpuid;
1074 	int i, j, k, srs_cnt, nscpus, maxcpus, soft_ring_cnt = 0;
1075 	mac_cpus_t *srs_cpu;
1076 	mac_resource_props_t *emrp = &flent->fe_effective_props;
1077 	uint32_t cpus[MRP_NCPUS];
1078 
1079 	/*
1080 	 * The maximum number of CPUs available can either be
1081 	 * the number of CPUs in the pool or the number of CPUs
1082 	 * in the system.
1083 	 */
1084 	maxcpus = (cpupart != NULL) ? cpupart->cp_ncpus : ncpus;
1085 
1086 	/*
1087 	 * Compute the number of soft rings needed on top for each Rx
1088 	 * SRS. "rx_srs_cnt-1" indicates the number of Rx SRS
1089 	 * associated with h/w Rx rings. Soft ring count needed for
1090 	 * each h/w Rx SRS is computed and the same is applied to
1091 	 * software classified Rx SRS. The first Rx SRS in fe_rx_srs[]
1092 	 * is the software classified Rx SRS.
1093 	 */
1094 	soft_ring_cnt = mac_compute_soft_ring_count(flent,
1095 	    flent->fe_rx_srs_cnt - 1, maxcpus);
1096 	if (soft_ring_cnt == 0) {
1097 		/*
1098 		 * Even when soft_ring_cnt is 0, we still need
1099 		 * to create a soft ring for TCP, UDP and
1100 		 * OTHER. So set it to 1.
1101 		 */
1102 		soft_ring_cnt = 1;
1103 	}
1104 	for (srs_cnt = 0; srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) {
1105 		rx_srs = flent->fe_rx_srs[srs_cnt];
1106 		srs_cpu = &rx_srs->srs_cpu;
1107 		if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT)
1108 			rx_srs->srs_fanout_state = SRS_FANOUT_REINIT;
1109 		srs_cpu->mc_ncpus = soft_ring_cnt;
1110 		srs_cpu->mc_rx_fanout_cnt = soft_ring_cnt;
1111 		mutex_enter(&cpu_lock);
1112 		for (j = 0; j < soft_ring_cnt; j++) {
1113 			cpuid = mac_next_bind_cpu(cpupart);
1114 			srs_cpu->mc_cpus[j] = cpuid;
1115 			srs_cpu->mc_rx_fanout_cpus[j] = cpuid;
1116 		}
1117 		cpuid = mac_next_bind_cpu(cpupart);
1118 		srs_cpu->mc_rx_pollid = cpuid;
1119 		srs_cpu->mc_rx_intr_cpu = (mac_rx_intr_retarget ?
1120 		    srs_cpu->mc_rx_pollid : -1);
1121 		/* increment ncpus to account for polling cpu */
1122 		srs_cpu->mc_ncpus++;
1123 		srs_cpu->mc_cpus[j++] = cpuid;
1124 		if (!mac_latency_optimize) {
1125 			cpuid = mac_next_bind_cpu(cpupart);
1126 			srs_cpu->mc_ncpus++;
1127 			srs_cpu->mc_cpus[j++] = cpuid;
1128 		}
1129 		srs_cpu->mc_rx_workerid = cpuid;
1130 		mutex_exit(&cpu_lock);
1131 	}
1132 
1133 	nscpus = 0;
1134 	for (srs_cnt = 0; srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) {
1135 		rx_srs = flent->fe_rx_srs[srs_cnt];
1136 		srs_cpu = &rx_srs->srs_cpu;
1137 		for (j = 0; j < srs_cpu->mc_ncpus; j++) {
1138 			cpus[nscpus++] = srs_cpu->mc_cpus[j];
1139 		}
1140 	}
1141 
1142 
1143 	/*
1144 	 * Copy cpu list to fe_effective_props
1145 	 * without duplicates.
1146 	 */
1147 	k = 0;
1148 	for (i = 0; i < nscpus; i++) {
1149 		for (j = 0; j < k; j++) {
1150 			if (emrp->mrp_cpu[j] == cpus[i])
1151 				break;
1152 		}
1153 		if (j == k)
1154 			emrp->mrp_cpu[k++] = cpus[i];
1155 	}
1156 	emrp->mrp_ncpus = k;
1157 
1158 	mac_tx_cpu_init(flent, NULL, cpupart);
1159 }
1160 
1161 /*
1162  * DATAPATH SETUP ROUTINES
1163  * (setup SRS and set/update FANOUT, B/W and PRIORITY)
1164  */
1165 
1166 /*
1167  * mac_srs_fanout_list_alloc:
1168  *
1169  * The underlying device can expose upto MAX_RINGS_PER_GROUP worth of
1170  * rings to a client. In such a case, MAX_RINGS_PER_GROUP worth of
1171  * array space is needed to store Tx soft rings. Thus we allocate so
1172  * much array space for srs_tx_soft_rings.
1173  *
1174  * And when it is an aggr, again we allocate MAX_RINGS_PER_GROUP worth
1175  * of space to st_soft_rings. This array is used for quick access to
1176  * soft ring associated with a pseudo Tx ring based on the pseudo
1177  * ring's index (mr_index).
1178  */
1179 static void
mac_srs_fanout_list_alloc(mac_soft_ring_set_t * mac_srs)1180 mac_srs_fanout_list_alloc(mac_soft_ring_set_t *mac_srs)
1181 {
1182 	mac_client_impl_t *mcip = mac_srs->srs_mcip;
1183 
1184 	if (mac_srs->srs_type & SRST_TX) {
1185 		mac_srs->srs_tx_soft_rings = (mac_soft_ring_t **)
1186 		    kmem_zalloc(sizeof (mac_soft_ring_t *) *
1187 		    MAX_RINGS_PER_GROUP, KM_SLEEP);
1188 		if (mcip->mci_state_flags & MCIS_IS_AGGR) {
1189 			mac_srs_tx_t *tx = &mac_srs->srs_tx;
1190 
1191 			tx->st_soft_rings = (mac_soft_ring_t **)
1192 			    kmem_zalloc(sizeof (mac_soft_ring_t *) *
1193 			    MAX_RINGS_PER_GROUP, KM_SLEEP);
1194 		}
1195 	} else {
1196 		mac_srs->srs_tcp_soft_rings = (mac_soft_ring_t **)
1197 		    kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT,
1198 		    KM_SLEEP);
1199 		mac_srs->srs_udp_soft_rings = (mac_soft_ring_t **)
1200 		    kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT,
1201 		    KM_SLEEP);
1202 		mac_srs->srs_oth_soft_rings = (mac_soft_ring_t **)
1203 		    kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT,
1204 		    KM_SLEEP);
1205 	}
1206 }
1207 
1208 static void
mac_srs_worker_bind(mac_soft_ring_set_t * mac_srs,processorid_t cpuid)1209 mac_srs_worker_bind(mac_soft_ring_set_t *mac_srs, processorid_t cpuid)
1210 {
1211 	cpu_t *cp;
1212 	boolean_t clear = B_FALSE;
1213 
1214 	ASSERT(MUTEX_HELD(&cpu_lock));
1215 
1216 	if (!mac_srs_thread_bind)
1217 		return;
1218 
1219 	cp = cpu_get(cpuid);
1220 	if (cp == NULL || !cpu_is_online(cp))
1221 		return;
1222 
1223 	mutex_enter(&mac_srs->srs_lock);
1224 	mac_srs->srs_state |= SRS_WORKER_BOUND;
1225 	if (mac_srs->srs_worker_cpuid != -1)
1226 		clear = B_TRUE;
1227 	mac_srs->srs_worker_cpuid = cpuid;
1228 	mutex_exit(&mac_srs->srs_lock);
1229 
1230 	if (clear)
1231 		thread_affinity_clear(mac_srs->srs_worker);
1232 
1233 	thread_affinity_set(mac_srs->srs_worker, cpuid);
1234 	DTRACE_PROBE1(worker__CPU, processorid_t, cpuid);
1235 }
1236 
1237 static void
mac_srs_poll_bind(mac_soft_ring_set_t * mac_srs,processorid_t cpuid)1238 mac_srs_poll_bind(mac_soft_ring_set_t *mac_srs, processorid_t cpuid)
1239 {
1240 	cpu_t *cp;
1241 	boolean_t clear = B_FALSE;
1242 
1243 	ASSERT(MUTEX_HELD(&cpu_lock));
1244 
1245 	if (!mac_srs_thread_bind || mac_srs->srs_poll_thr == NULL)
1246 		return;
1247 
1248 	cp = cpu_get(cpuid);
1249 	if (cp == NULL || !cpu_is_online(cp))
1250 		return;
1251 
1252 	mutex_enter(&mac_srs->srs_lock);
1253 	mac_srs->srs_state |= SRS_POLL_BOUND;
1254 	if (mac_srs->srs_poll_cpuid != -1)
1255 		clear = B_TRUE;
1256 	mac_srs->srs_poll_cpuid = cpuid;
1257 	mutex_exit(&mac_srs->srs_lock);
1258 
1259 	if (clear)
1260 		thread_affinity_clear(mac_srs->srs_poll_thr);
1261 
1262 	thread_affinity_set(mac_srs->srs_poll_thr, cpuid);
1263 	DTRACE_PROBE1(poll__CPU, processorid_t, cpuid);
1264 }
1265 
1266 /*
1267  * Re-target interrupt to the passed CPU. If re-target is successful,
1268  * set mc_rx_intr_cpu to the re-targeted CPU. Otherwise set it to -1.
1269  */
1270 void
mac_rx_srs_retarget_intr(mac_soft_ring_set_t * mac_srs,processorid_t cpuid)1271 mac_rx_srs_retarget_intr(mac_soft_ring_set_t *mac_srs, processorid_t cpuid)
1272 {
1273 	cpu_t *cp;
1274 	mac_ring_t *ring = mac_srs->srs_ring;
1275 	mac_intr_t *mintr = &ring->mr_info.mri_intr;
1276 	flow_entry_t *flent = mac_srs->srs_flent;
1277 	boolean_t primary = mac_is_primary_client(mac_srs->srs_mcip);
1278 
1279 	ASSERT(MUTEX_HELD(&cpu_lock));
1280 
1281 	/*
1282 	 * Don't re-target the interrupt for these cases:
1283 	 * 1) ring is NULL
1284 	 * 2) the interrupt is shared (mi_ddi_shared)
1285 	 * 3) ddi_handle is NULL and !primary
1286 	 * 4) primary, ddi_handle is NULL but fe_rx_srs_cnt > 2
1287 	 * Case 3 & 4 are because of mac_client_intr_cpu() routine.
1288 	 * This routine will re-target fixed interrupt for primary
1289 	 * mac client if the client has only one ring. In that
1290 	 * case, mc_rx_intr_cpu will already have the correct value.
1291 	 */
1292 	if (ring == NULL || mintr->mi_ddi_shared || cpuid == -1 ||
1293 	    (mintr->mi_ddi_handle == NULL && !primary) || (primary &&
1294 	    mintr->mi_ddi_handle == NULL && flent->fe_rx_srs_cnt > 2)) {
1295 		mac_srs->srs_cpu.mc_rx_intr_cpu = -1;
1296 		return;
1297 	}
1298 
1299 	if (mintr->mi_ddi_handle == NULL)
1300 		return;
1301 
1302 	cp = cpu_get(cpuid);
1303 	if (cp == NULL || !cpu_is_online(cp))
1304 		return;
1305 
1306 	/* Drop the cpu_lock as set_intr_affinity() holds it */
1307 	mutex_exit(&cpu_lock);
1308 	if (set_intr_affinity(mintr->mi_ddi_handle, cpuid) == DDI_SUCCESS)
1309 		mac_srs->srs_cpu.mc_rx_intr_cpu = cpuid;
1310 	else
1311 		mac_srs->srs_cpu.mc_rx_intr_cpu = -1;
1312 	mutex_enter(&cpu_lock);
1313 }
1314 
1315 /*
1316  * Re-target Tx interrupts
1317  */
1318 void
mac_tx_srs_retarget_intr(mac_soft_ring_set_t * mac_srs)1319 mac_tx_srs_retarget_intr(mac_soft_ring_set_t *mac_srs)
1320 {
1321 	cpu_t *cp;
1322 	mac_ring_t *ring;
1323 	mac_intr_t *mintr;
1324 	mac_soft_ring_t *sringp;
1325 	mac_srs_tx_t *srs_tx;
1326 	mac_cpus_t *srs_cpu;
1327 	processorid_t cpuid;
1328 	int i;
1329 
1330 	ASSERT(MUTEX_HELD(&cpu_lock));
1331 
1332 	srs_cpu = &mac_srs->srs_cpu;
1333 	if (MAC_TX_SOFT_RINGS(mac_srs)) {
1334 		for (i = 0; i < mac_srs->srs_tx_ring_count; i++) {
1335 			sringp = mac_srs->srs_tx_soft_rings[i];
1336 			ring = (mac_ring_t *)sringp->s_ring_tx_arg2;
1337 			cpuid = srs_cpu->mc_tx_intr_cpu[i];
1338 			cp = cpu_get(cpuid);
1339 			if (cp == NULL || !cpu_is_online(cp) ||
1340 			    !MAC_RING_RETARGETABLE(ring)) {
1341 				srs_cpu->mc_tx_retargeted_cpu[i] = -1;
1342 				continue;
1343 			}
1344 			mintr = &ring->mr_info.mri_intr;
1345 			/*
1346 			 * Drop the cpu_lock as set_intr_affinity()
1347 			 * holds it
1348 			 */
1349 			mutex_exit(&cpu_lock);
1350 			if (set_intr_affinity(mintr->mi_ddi_handle,
1351 			    cpuid) == DDI_SUCCESS) {
1352 				srs_cpu->mc_tx_retargeted_cpu[i] = cpuid;
1353 			} else {
1354 				srs_cpu->mc_tx_retargeted_cpu[i] = -1;
1355 			}
1356 			mutex_enter(&cpu_lock);
1357 		}
1358 	} else {
1359 		cpuid = srs_cpu->mc_tx_intr_cpu[0];
1360 		cp = cpu_get(cpuid);
1361 		if (cp == NULL || !cpu_is_online(cp)) {
1362 			srs_cpu->mc_tx_retargeted_cpu[0] = -1;
1363 			return;
1364 		}
1365 		srs_tx = &mac_srs->srs_tx;
1366 		ring = (mac_ring_t *)srs_tx->st_arg2;
1367 		if (MAC_RING_RETARGETABLE(ring)) {
1368 			mintr = &ring->mr_info.mri_intr;
1369 			mutex_exit(&cpu_lock);
1370 			if ((set_intr_affinity(mintr->mi_ddi_handle,
1371 			    cpuid) == DDI_SUCCESS)) {
1372 				srs_cpu->mc_tx_retargeted_cpu[0] = cpuid;
1373 			} else {
1374 				srs_cpu->mc_tx_retargeted_cpu[0] = -1;
1375 			}
1376 			mutex_enter(&cpu_lock);
1377 		}
1378 	}
1379 }
1380 
1381 /*
1382  * When a CPU comes back online, bind the MAC kernel threads which
1383  * were previously bound to that CPU, and had to be unbound because
1384  * the CPU was going away.
1385  *
1386  * These functions are called with cpu_lock held and hence we can't
1387  * cv_wait to grab the mac perimeter. Since these functions walk the soft
1388  * ring list of an SRS without being in the perimeter, the list itself
1389  * is protected by the SRS lock.
1390  */
1391 static void
mac_walk_srs_and_bind(int cpuid)1392 mac_walk_srs_and_bind(int cpuid)
1393 {
1394 	mac_soft_ring_set_t *mac_srs;
1395 	mac_soft_ring_t *soft_ring;
1396 
1397 	rw_enter(&mac_srs_g_lock, RW_READER);
1398 
1399 	if ((mac_srs = mac_srs_g_list) == NULL)
1400 		goto done;
1401 
1402 	for (; mac_srs != NULL; mac_srs = mac_srs->srs_next) {
1403 		if (mac_srs->srs_worker_cpuid == -1 &&
1404 		    mac_srs->srs_worker_cpuid_save == cpuid) {
1405 			mac_srs->srs_worker_cpuid_save = -1;
1406 			mac_srs_worker_bind(mac_srs, cpuid);
1407 		}
1408 
1409 		if (!(mac_srs->srs_type & SRST_TX)) {
1410 			if (mac_srs->srs_poll_cpuid == -1 &&
1411 			    mac_srs->srs_poll_cpuid_save == cpuid) {
1412 				mac_srs->srs_poll_cpuid_save = -1;
1413 				mac_srs_poll_bind(mac_srs, cpuid);
1414 			}
1415 		}
1416 
1417 		/* Next tackle the soft rings associated with the srs */
1418 		mutex_enter(&mac_srs->srs_lock);
1419 		for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL;
1420 		    soft_ring = soft_ring->s_ring_next) {
1421 			if (soft_ring->s_ring_cpuid == -1 &&
1422 			    soft_ring->s_ring_cpuid_save == cpuid) {
1423 				soft_ring->s_ring_cpuid_save = -1;
1424 				(void) mac_soft_ring_bind(soft_ring, cpuid);
1425 			}
1426 		}
1427 		mutex_exit(&mac_srs->srs_lock);
1428 	}
1429 done:
1430 	rw_exit(&mac_srs_g_lock);
1431 }
1432 
1433 /*
1434  * Change the priority of the SRS's poll and worker thread. Additionally,
1435  * update the priority of the worker threads for the SRS's soft rings.
1436  * Need to modify any associated squeue threads.
1437  */
1438 void
mac_update_srs_priority(mac_soft_ring_set_t * mac_srs,pri_t prival)1439 mac_update_srs_priority(mac_soft_ring_set_t *mac_srs, pri_t prival)
1440 {
1441 	mac_soft_ring_t		*ringp;
1442 
1443 	mac_srs->srs_pri = prival;
1444 	thread_lock(mac_srs->srs_worker);
1445 	(void) thread_change_pri(mac_srs->srs_worker, mac_srs->srs_pri, 0);
1446 	thread_unlock(mac_srs->srs_worker);
1447 	if (mac_srs->srs_poll_thr != NULL) {
1448 		thread_lock(mac_srs->srs_poll_thr);
1449 		(void) thread_change_pri(mac_srs->srs_poll_thr,
1450 		    mac_srs->srs_pri, 0);
1451 		thread_unlock(mac_srs->srs_poll_thr);
1452 	}
1453 	if ((ringp = mac_srs->srs_soft_ring_head) == NULL)
1454 		return;
1455 	while (ringp != mac_srs->srs_soft_ring_tail) {
1456 		thread_lock(ringp->s_ring_worker);
1457 		(void) thread_change_pri(ringp->s_ring_worker,
1458 		    mac_srs->srs_pri, 0);
1459 		thread_unlock(ringp->s_ring_worker);
1460 		ringp = ringp->s_ring_next;
1461 	}
1462 	ASSERT(ringp == mac_srs->srs_soft_ring_tail);
1463 	thread_lock(ringp->s_ring_worker);
1464 	(void) thread_change_pri(ringp->s_ring_worker, mac_srs->srs_pri, 0);
1465 	thread_unlock(ringp->s_ring_worker);
1466 }
1467 
1468 /*
1469  * Change the receive bandwidth limit.
1470  */
1471 static void
mac_rx_srs_update_bwlimit(mac_soft_ring_set_t * srs,mac_resource_props_t * mrp)1472 mac_rx_srs_update_bwlimit(mac_soft_ring_set_t *srs, mac_resource_props_t *mrp)
1473 {
1474 	mac_soft_ring_t		*softring;
1475 
1476 	mutex_enter(&srs->srs_lock);
1477 	mutex_enter(&srs->srs_bw->mac_bw_lock);
1478 
1479 	if (mrp->mrp_maxbw == MRP_MAXBW_RESETVAL) {
1480 		/* Reset bandwidth limit */
1481 		if (srs->srs_type & SRST_BW_CONTROL) {
1482 			softring = srs->srs_soft_ring_head;
1483 			while (softring != NULL) {
1484 				softring->s_ring_type &= ~ST_RING_BW_CTL;
1485 				softring = softring->s_ring_next;
1486 			}
1487 			srs->srs_type &= ~SRST_BW_CONTROL;
1488 			srs->srs_drain_func = mac_rx_srs_drain;
1489 		}
1490 	} else {
1491 		/* Set/Modify bandwidth limit */
1492 		srs->srs_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw);
1493 		/*
1494 		 * Give twice the queuing capability before
1495 		 * dropping packets. The unit is bytes/tick.
1496 		 */
1497 		srs->srs_bw->mac_bw_drop_threshold =
1498 		    srs->srs_bw->mac_bw_limit << 1;
1499 		if (!(srs->srs_type & SRST_BW_CONTROL)) {
1500 			softring = srs->srs_soft_ring_head;
1501 			while (softring != NULL) {
1502 				softring->s_ring_type |= ST_RING_BW_CTL;
1503 				softring = softring->s_ring_next;
1504 			}
1505 			srs->srs_type |= SRST_BW_CONTROL;
1506 			srs->srs_drain_func = mac_rx_srs_drain_bw;
1507 		}
1508 	}
1509 done:
1510 	mutex_exit(&srs->srs_bw->mac_bw_lock);
1511 	mutex_exit(&srs->srs_lock);
1512 }
1513 
1514 /* Change the transmit bandwidth limit */
1515 static void
mac_tx_srs_update_bwlimit(mac_soft_ring_set_t * srs,mac_resource_props_t * mrp)1516 mac_tx_srs_update_bwlimit(mac_soft_ring_set_t *srs, mac_resource_props_t *mrp)
1517 {
1518 	uint32_t		tx_mode, ring_info = 0;
1519 	mac_srs_tx_t		*srs_tx = &srs->srs_tx;
1520 	mac_client_impl_t	*mcip = srs->srs_mcip;
1521 
1522 	/*
1523 	 * We need to quiesce/restart the client here because mac_tx() and
1524 	 * srs->srs_tx->st_func do not hold srs->srs_lock while accessing
1525 	 * st_mode and related fields, which are modified by the code below.
1526 	 */
1527 	mac_tx_client_quiesce((mac_client_handle_t)mcip);
1528 
1529 	mutex_enter(&srs->srs_lock);
1530 	mutex_enter(&srs->srs_bw->mac_bw_lock);
1531 
1532 	tx_mode = srs_tx->st_mode;
1533 	if (mrp->mrp_maxbw == MRP_MAXBW_RESETVAL) {
1534 		/* Reset bandwidth limit */
1535 		if (tx_mode == SRS_TX_BW) {
1536 			if (srs_tx->st_arg2 != NULL)
1537 				ring_info = mac_hwring_getinfo(srs_tx->st_arg2);
1538 			if (mac_tx_serialize ||
1539 			    (ring_info & MAC_RING_TX_SERIALIZE)) {
1540 				srs_tx->st_mode = SRS_TX_SERIALIZE;
1541 			} else {
1542 				srs_tx->st_mode = SRS_TX_DEFAULT;
1543 			}
1544 		} else if (tx_mode == SRS_TX_BW_FANOUT) {
1545 			srs_tx->st_mode = SRS_TX_FANOUT;
1546 		} else if (tx_mode == SRS_TX_BW_AGGR) {
1547 			srs_tx->st_mode = SRS_TX_AGGR;
1548 		}
1549 		srs->srs_type &= ~SRST_BW_CONTROL;
1550 	} else {
1551 		/* Set/Modify bandwidth limit */
1552 		srs->srs_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw);
1553 		/*
1554 		 * Give twice the queuing capability before
1555 		 * dropping packets. The unit is bytes/tick.
1556 		 */
1557 		srs->srs_bw->mac_bw_drop_threshold =
1558 		    srs->srs_bw->mac_bw_limit << 1;
1559 		srs->srs_type |= SRST_BW_CONTROL;
1560 		if (tx_mode != SRS_TX_BW && tx_mode != SRS_TX_BW_FANOUT &&
1561 		    tx_mode != SRS_TX_BW_AGGR) {
1562 			if (tx_mode == SRS_TX_SERIALIZE ||
1563 			    tx_mode == SRS_TX_DEFAULT) {
1564 				srs_tx->st_mode = SRS_TX_BW;
1565 			} else if (tx_mode == SRS_TX_FANOUT) {
1566 				srs_tx->st_mode = SRS_TX_BW_FANOUT;
1567 			} else if (tx_mode == SRS_TX_AGGR) {
1568 				srs_tx->st_mode = SRS_TX_BW_AGGR;
1569 			} else {
1570 				ASSERT(0);
1571 			}
1572 		}
1573 	}
1574 done:
1575 	srs_tx->st_func = mac_tx_get_func(srs_tx->st_mode);
1576 	mutex_exit(&srs->srs_bw->mac_bw_lock);
1577 	mutex_exit(&srs->srs_lock);
1578 
1579 	mac_tx_client_restart((mac_client_handle_t)mcip);
1580 }
1581 
1582 /*
1583  * The uber function that deals with any update to bandwidth limits.
1584  */
1585 void
mac_srs_update_bwlimit(flow_entry_t * flent,mac_resource_props_t * mrp)1586 mac_srs_update_bwlimit(flow_entry_t *flent, mac_resource_props_t *mrp)
1587 {
1588 	int			count;
1589 
1590 	for (count = 0; count < flent->fe_rx_srs_cnt; count++)
1591 		mac_rx_srs_update_bwlimit(flent->fe_rx_srs[count], mrp);
1592 	mac_tx_srs_update_bwlimit(flent->fe_tx_srs, mrp);
1593 }
1594 
1595 void
mac_srs_change_upcall(void * arg,mac_direct_rx_t rx_func,void * rx_arg1)1596 mac_srs_change_upcall(void *arg, mac_direct_rx_t rx_func, void *rx_arg1)
1597 {
1598 	mac_soft_ring_set_t	*mac_srs = arg;
1599 	mac_srs_rx_t		*srs_rx = &mac_srs->srs_rx;
1600 	mac_soft_ring_t		*softring;
1601 
1602 	mutex_enter(&mac_srs->srs_lock);
1603 	ASSERT((mac_srs->srs_type & SRST_TX) == 0);
1604 	srs_rx->sr_func = rx_func;
1605 	srs_rx->sr_arg1 = rx_arg1;
1606 
1607 	softring = mac_srs->srs_soft_ring_head;
1608 	while (softring != NULL) {
1609 		mutex_enter(&softring->s_ring_lock);
1610 		softring->s_ring_rx_func = rx_func;
1611 		softring->s_ring_rx_arg1 = rx_arg1;
1612 		mutex_exit(&softring->s_ring_lock);
1613 		softring = softring->s_ring_next;
1614 	}
1615 
1616 	mutex_exit(&mac_srs->srs_lock);
1617 }
1618 
1619 /*
1620  * When the first sub-flow is added to a link, we disable polling on the
1621  * link and also modify the entry point to mac_rx_srs_subflow_process.
1622  * (polling is disabled because with the subflow added, accounting
1623  * for polling needs additional logic, it is assumed that when a subflow is
1624  * added, we can take some hit as a result of disabling polling rather than
1625  * adding more complexity - if this becomes a perf. issue we need to
1626  * re-rvaluate this logic).  When the last subflow is removed, we turn back
1627  * polling and also reset the entry point to mac_rx_srs_process.
1628  *
1629  * In the future if there are multiple SRS, we can simply
1630  * take one and give it to the flow rather than disabling polling and
1631  * resetting the entry point.
1632  */
1633 void
mac_client_update_classifier(mac_client_impl_t * mcip,boolean_t enable)1634 mac_client_update_classifier(mac_client_impl_t *mcip, boolean_t enable)
1635 {
1636 	flow_entry_t		*flent = mcip->mci_flent;
1637 	int			i;
1638 	mac_impl_t		*mip = mcip->mci_mip;
1639 	mac_rx_func_t		rx_func;
1640 	uint_t			rx_srs_cnt;
1641 	boolean_t		enable_classifier;
1642 
1643 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mip));
1644 
1645 	enable_classifier = !FLOW_TAB_EMPTY(mcip->mci_subflow_tab) && enable;
1646 
1647 	rx_func = enable_classifier ? mac_rx_srs_subflow_process :
1648 	    mac_rx_srs_process;
1649 
1650 	/* Tell mac_srs_poll_state_change to disable polling if necessary */
1651 	if (mip->mi_state_flags & MIS_POLL_DISABLE)
1652 		enable_classifier = B_TRUE;
1653 
1654 	/*
1655 	 * If receive function has already been configured correctly for
1656 	 * current subflow configuration, do nothing.
1657 	 */
1658 	if (flent->fe_cb_fn == (flow_fn_t)rx_func)
1659 		return;
1660 
1661 	rx_srs_cnt = flent->fe_rx_srs_cnt;
1662 	for (i = 0; i < rx_srs_cnt; i++) {
1663 		ASSERT(flent->fe_rx_srs[i] != NULL);
1664 		mac_srs_poll_state_change(flent->fe_rx_srs[i],
1665 		    enable_classifier, rx_func);
1666 	}
1667 
1668 	/*
1669 	 * Change the S/W classifier so that we can land in the
1670 	 * correct processing function with correct argument.
1671 	 * If all subflows have been removed we can revert to
1672 	 * mac_rx_srsprocess, else we need mac_rx_srs_subflow_process.
1673 	 */
1674 	mutex_enter(&flent->fe_lock);
1675 	flent->fe_cb_fn = (flow_fn_t)rx_func;
1676 	flent->fe_cb_arg1 = (void *)mip;
1677 	flent->fe_cb_arg2 = flent->fe_rx_srs[0];
1678 	mutex_exit(&flent->fe_lock);
1679 }
1680 
1681 static void
mac_srs_update_fanout_list(mac_soft_ring_set_t * mac_srs)1682 mac_srs_update_fanout_list(mac_soft_ring_set_t *mac_srs)
1683 {
1684 	int tcp_count = 0, udp_count = 0, oth_count = 0, tx_count = 0;
1685 	mac_soft_ring_t *softring;
1686 
1687 	softring = mac_srs->srs_soft_ring_head;
1688 	if (softring == NULL) {
1689 		ASSERT(mac_srs->srs_soft_ring_count == 0);
1690 		mac_srs->srs_tcp_ring_count = 0;
1691 		mac_srs->srs_udp_ring_count = 0;
1692 		mac_srs->srs_oth_ring_count = 0;
1693 		mac_srs->srs_tx_ring_count = 0;
1694 		return;
1695 	}
1696 
1697 	while (softring != NULL) {
1698 		if (softring->s_ring_type & ST_RING_TCP) {
1699 			mac_srs->srs_tcp_soft_rings[tcp_count++] = softring;
1700 		} else if (softring->s_ring_type & ST_RING_UDP) {
1701 			mac_srs->srs_udp_soft_rings[udp_count++] = softring;
1702 		} else if (softring->s_ring_type & ST_RING_OTH) {
1703 			mac_srs->srs_oth_soft_rings[oth_count++] = softring;
1704 		} else {
1705 			ASSERT(softring->s_ring_type & ST_RING_TX);
1706 			mac_srs->srs_tx_soft_rings[tx_count++] = softring;
1707 		}
1708 		softring = softring->s_ring_next;
1709 	}
1710 
1711 	ASSERT(mac_srs->srs_soft_ring_count ==
1712 	    (tcp_count + udp_count + oth_count + tx_count));
1713 	mac_srs->srs_tcp_ring_count = tcp_count;
1714 	mac_srs->srs_udp_ring_count = udp_count;
1715 	mac_srs->srs_oth_ring_count = oth_count;
1716 	mac_srs->srs_tx_ring_count = tx_count;
1717 }
1718 
1719 void
mac_srs_create_proto_softrings(int id,uint16_t type,pri_t pri,mac_client_impl_t * mcip,mac_soft_ring_set_t * mac_srs,processorid_t cpuid,mac_direct_rx_t rx_func,void * x_arg1,mac_resource_handle_t x_arg2,boolean_t set_bypass)1720 mac_srs_create_proto_softrings(int id, uint16_t type, pri_t pri,
1721     mac_client_impl_t *mcip, mac_soft_ring_set_t *mac_srs,
1722     processorid_t cpuid, mac_direct_rx_t rx_func, void *x_arg1,
1723     mac_resource_handle_t x_arg2, boolean_t set_bypass)
1724 {
1725 	mac_soft_ring_t	*softring;
1726 	mac_rx_fifo_t	mrf;
1727 
1728 	bzero(&mrf, sizeof (mac_rx_fifo_t));
1729 	mrf.mrf_type = MAC_RX_FIFO;
1730 	mrf.mrf_receive = (mac_receive_t)mac_soft_ring_poll;
1731 	mrf.mrf_intr_enable =
1732 	    (mac_intr_enable_t)mac_soft_ring_intr_enable;
1733 	mrf.mrf_intr_disable =
1734 	    (mac_intr_disable_t)mac_soft_ring_intr_disable;
1735 	mrf.mrf_flow_priority = pri;
1736 
1737 	softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait,
1738 	    (type|ST_RING_TCP), pri, mcip, mac_srs,
1739 	    cpuid, rx_func, x_arg1, x_arg2);
1740 	softring->s_ring_rx_arg2 = NULL;
1741 
1742 	/*
1743 	 * TCP and UDP support DLS bypass. In addition TCP
1744 	 * squeue can also poll their corresponding soft rings.
1745 	 */
1746 	if (set_bypass && (mcip->mci_resource_arg != NULL)) {
1747 		mac_soft_ring_dls_bypass(softring,
1748 		    mcip->mci_direct_rx_fn,
1749 		    mcip->mci_direct_rx_arg);
1750 
1751 		mrf.mrf_rx_arg = softring;
1752 		mrf.mrf_intr_handle = (mac_intr_handle_t)softring;
1753 
1754 		/*
1755 		 * Make a call in IP to get a TCP squeue assigned to
1756 		 * this softring to maintain full CPU locality through
1757 		 * the stack and allow the squeue to be able to poll
1758 		 * the softring so the flow control can be pushed
1759 		 * all the way to H/W.
1760 		 */
1761 		softring->s_ring_rx_arg2 =
1762 		    mcip->mci_resource_add((void *)mcip->mci_resource_arg,
1763 		    (mac_resource_t *)&mrf);
1764 	}
1765 
1766 	/*
1767 	 * Non-TCP protocols don't support squeues. Hence we
1768 	 * don't make any ring addition callbacks for non-TCP
1769 	 * rings. Now create the UDP softring and allow it to
1770 	 * bypass the DLS layer.
1771 	 */
1772 	softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait,
1773 	    (type|ST_RING_UDP), pri, mcip, mac_srs,
1774 	    cpuid, rx_func, x_arg1, x_arg2);
1775 	softring->s_ring_rx_arg2 = NULL;
1776 
1777 	if (set_bypass && (mcip->mci_resource_arg != NULL)) {
1778 		mac_soft_ring_dls_bypass(softring,
1779 		    mcip->mci_direct_rx_fn,
1780 		    mcip->mci_direct_rx_arg);
1781 	}
1782 
1783 	/* Create the Oth softrings which has to go through the DLS */
1784 	softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait,
1785 	    (type|ST_RING_OTH), pri, mcip, mac_srs,
1786 	    cpuid, rx_func, x_arg1, x_arg2);
1787 	softring->s_ring_rx_arg2 = NULL;
1788 }
1789 
1790 /*
1791  * This routine associates a CPU or a set of CPU to process incoming
1792  * traffic from a mac client. If multiple CPUs are specified, then
1793  * so many soft rings are created with each soft ring worker thread
1794  * bound to a CPU in the set. Each soft ring in turn will be
1795  * associated with an squeue and the squeue will be moved to the
1796  * same CPU as that of the soft ring's.
1797  */
1798 static void
mac_srs_fanout_modify(mac_client_impl_t * mcip,mac_direct_rx_t rx_func,void * x_arg1,mac_resource_handle_t x_arg2,mac_soft_ring_set_t * mac_rx_srs,mac_soft_ring_set_t * mac_tx_srs)1799 mac_srs_fanout_modify(mac_client_impl_t *mcip, mac_direct_rx_t rx_func,
1800     void *x_arg1, mac_resource_handle_t x_arg2,
1801     mac_soft_ring_set_t *mac_rx_srs, mac_soft_ring_set_t *mac_tx_srs)
1802 {
1803 	mac_soft_ring_t *softring;
1804 	uint32_t soft_ring_flag = 0;
1805 	processorid_t cpuid = -1;
1806 	int i, srings_present, new_fanout_cnt;
1807 	mac_cpus_t *srs_cpu;
1808 
1809 	/* fanout state is REINIT. Set it back to INIT */
1810 	ASSERT(mac_rx_srs->srs_fanout_state == SRS_FANOUT_REINIT);
1811 	mac_rx_srs->srs_fanout_state = SRS_FANOUT_INIT;
1812 
1813 	/* how many are present right now */
1814 	srings_present = mac_rx_srs->srs_tcp_ring_count;
1815 	/* new request */
1816 	srs_cpu = &mac_rx_srs->srs_cpu;
1817 	new_fanout_cnt = srs_cpu->mc_rx_fanout_cnt;
1818 
1819 	mutex_enter(&mac_rx_srs->srs_lock);
1820 	if (mac_rx_srs->srs_type & SRST_BW_CONTROL)
1821 		soft_ring_flag |= ST_RING_BW_CTL;
1822 	mutex_exit(&mac_rx_srs->srs_lock);
1823 
1824 	if (new_fanout_cnt > srings_present) {
1825 		/* soft rings increased */
1826 		mutex_enter(&mac_rx_srs->srs_lock);
1827 		mac_rx_srs->srs_type |= SRST_FANOUT_SRC_IP;
1828 		mutex_exit(&mac_rx_srs->srs_lock);
1829 
1830 		for (i = mac_rx_srs->srs_tcp_ring_count;
1831 		    i < new_fanout_cnt; i++) {
1832 			/*
1833 			 * Create the protocol softrings and set the
1834 			 * DLS bypass where possible.
1835 			 */
1836 			mac_srs_create_proto_softrings(i, soft_ring_flag,
1837 			    mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid,
1838 			    rx_func, x_arg1, x_arg2, B_TRUE);
1839 		}
1840 		mac_srs_update_fanout_list(mac_rx_srs);
1841 	} else if (new_fanout_cnt < srings_present) {
1842 		/* soft rings decreased */
1843 		if (new_fanout_cnt == 1) {
1844 			mutex_enter(&mac_rx_srs->srs_lock);
1845 			mac_rx_srs->srs_type &= ~SRST_FANOUT_SRC_IP;
1846 			ASSERT(mac_rx_srs->srs_type & SRST_FANOUT_PROTO);
1847 			mutex_exit(&mac_rx_srs->srs_lock);
1848 		}
1849 		/* Get rid of extra soft rings */
1850 		for (i = new_fanout_cnt;
1851 		    i < mac_rx_srs->srs_tcp_ring_count; i++) {
1852 			softring = mac_rx_srs->srs_tcp_soft_rings[i];
1853 			if (softring->s_ring_rx_arg2 != NULL) {
1854 				mcip->mci_resource_remove(
1855 				    (void *)mcip->mci_resource_arg,
1856 				    softring->s_ring_rx_arg2);
1857 			}
1858 			mac_soft_ring_remove(mac_rx_srs,
1859 			    mac_rx_srs->srs_tcp_soft_rings[i]);
1860 			mac_soft_ring_remove(mac_rx_srs,
1861 			    mac_rx_srs->srs_udp_soft_rings[i]);
1862 			mac_soft_ring_remove(mac_rx_srs,
1863 			    mac_rx_srs->srs_oth_soft_rings[i]);
1864 		}
1865 		mac_srs_update_fanout_list(mac_rx_srs);
1866 	}
1867 
1868 	ASSERT(new_fanout_cnt == mac_rx_srs->srs_tcp_ring_count);
1869 	mutex_enter(&cpu_lock);
1870 	for (i = 0; i < mac_rx_srs->srs_tcp_ring_count; i++) {
1871 		cpuid = srs_cpu->mc_rx_fanout_cpus[i];
1872 		(void) mac_soft_ring_bind(mac_rx_srs->srs_udp_soft_rings[i],
1873 		    cpuid);
1874 		(void) mac_soft_ring_bind(mac_rx_srs->srs_oth_soft_rings[i],
1875 		    cpuid);
1876 		(void) mac_soft_ring_bind(mac_rx_srs->srs_tcp_soft_rings[i],
1877 		    cpuid);
1878 		softring = mac_rx_srs->srs_tcp_soft_rings[i];
1879 		if (softring->s_ring_rx_arg2 != NULL) {
1880 			mcip->mci_resource_bind((void *)mcip->mci_resource_arg,
1881 			    softring->s_ring_rx_arg2, cpuid);
1882 		}
1883 	}
1884 
1885 	mac_srs_worker_bind(mac_rx_srs, srs_cpu->mc_rx_workerid);
1886 	mac_srs_poll_bind(mac_rx_srs, srs_cpu->mc_rx_pollid);
1887 	mac_rx_srs_retarget_intr(mac_rx_srs, srs_cpu->mc_rx_intr_cpu);
1888 	/*
1889 	 * Bind Tx srs and soft ring threads too. Let's bind tx
1890 	 * srs to the last cpu in mrp list.
1891 	 */
1892 	if (mac_tx_srs != NULL) {
1893 		BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp);
1894 		mac_tx_srs_retarget_intr(mac_tx_srs);
1895 	}
1896 	mutex_exit(&cpu_lock);
1897 }
1898 
1899 /*
1900  * Bind SRS threads and soft rings to CPUs/create fanout list.
1901  */
1902 void
mac_srs_fanout_init(mac_client_impl_t * mcip,mac_resource_props_t * mrp,mac_direct_rx_t rx_func,void * x_arg1,mac_resource_handle_t x_arg2,mac_soft_ring_set_t * mac_rx_srs,mac_soft_ring_set_t * mac_tx_srs,cpupart_t * cpupart)1903 mac_srs_fanout_init(mac_client_impl_t *mcip, mac_resource_props_t *mrp,
1904     mac_direct_rx_t rx_func, void *x_arg1, mac_resource_handle_t x_arg2,
1905     mac_soft_ring_set_t *mac_rx_srs, mac_soft_ring_set_t *mac_tx_srs,
1906     cpupart_t *cpupart)
1907 {
1908 	int		i;
1909 	processorid_t	cpuid;
1910 	uint32_t	soft_ring_flag = 0;
1911 	int soft_ring_cnt;
1912 	mac_cpus_t *srs_cpu = &mac_rx_srs->srs_cpu;
1913 
1914 	/*
1915 	 * Remove the no soft ring flag and we will adjust it
1916 	 * appropriately further down.
1917 	 */
1918 	mutex_enter(&mac_rx_srs->srs_lock);
1919 	mac_rx_srs->srs_type &= ~SRST_NO_SOFT_RINGS;
1920 	mutex_exit(&mac_rx_srs->srs_lock);
1921 
1922 	ASSERT(mac_rx_srs->srs_soft_ring_head == NULL);
1923 
1924 	if (mac_rx_srs->srs_type & SRST_BW_CONTROL)
1925 		soft_ring_flag |= ST_RING_BW_CTL;
1926 
1927 	ASSERT(mac_rx_srs->srs_fanout_state == SRS_FANOUT_UNINIT);
1928 	mac_rx_srs->srs_fanout_state = SRS_FANOUT_INIT;
1929 	/*
1930 	 * Ring count can be 0 if no fanout is required and no cpu
1931 	 * were specified. Leave the SRS worker and poll thread
1932 	 * unbound
1933 	 */
1934 	ASSERT(mrp != NULL);
1935 	soft_ring_cnt = srs_cpu->mc_rx_fanout_cnt;
1936 
1937 	/* Step 1: bind cpu contains cpu list where threads need to bind */
1938 	if (soft_ring_cnt > 0) {
1939 		mutex_enter(&cpu_lock);
1940 		for (i = 0; i < soft_ring_cnt; i++) {
1941 			cpuid = srs_cpu->mc_rx_fanout_cpus[i];
1942 			/* Create the protocol softrings */
1943 			mac_srs_create_proto_softrings(i, soft_ring_flag,
1944 			    mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid,
1945 			    rx_func, x_arg1, x_arg2, B_FALSE);
1946 		}
1947 		mac_srs_worker_bind(mac_rx_srs, srs_cpu->mc_rx_workerid);
1948 		mac_srs_poll_bind(mac_rx_srs, srs_cpu->mc_rx_pollid);
1949 		mac_rx_srs_retarget_intr(mac_rx_srs, srs_cpu->mc_rx_intr_cpu);
1950 		/*
1951 		 * Bind Tx srs and soft ring threads too.
1952 		 * Let's bind tx srs to the last cpu in
1953 		 * mrp list.
1954 		 */
1955 		if (mac_tx_srs == NULL) {
1956 			mutex_exit(&cpu_lock);
1957 			goto alldone;
1958 		}
1959 
1960 		BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp);
1961 		mac_tx_srs_retarget_intr(mac_tx_srs);
1962 		mutex_exit(&cpu_lock);
1963 	} else {
1964 		mutex_enter(&cpu_lock);
1965 		/*
1966 		 * For a subflow, mrp_workerid and mrp_pollid
1967 		 * is not set.
1968 		 */
1969 		mac_srs_worker_bind(mac_rx_srs, mrp->mrp_rx_workerid);
1970 		mac_srs_poll_bind(mac_rx_srs, mrp->mrp_rx_pollid);
1971 		mutex_exit(&cpu_lock);
1972 		goto no_softrings;
1973 	}
1974 
1975 alldone:
1976 	if (soft_ring_cnt > 1)
1977 		mac_rx_srs->srs_type |= SRST_FANOUT_SRC_IP;
1978 	mac_srs_update_fanout_list(mac_rx_srs);
1979 	mac_srs_client_poll_enable(mcip, mac_rx_srs);
1980 	return;
1981 
1982 no_softrings:
1983 	if (mac_rx_srs->srs_type & SRST_FANOUT_PROTO) {
1984 		mutex_enter(&cpu_lock);
1985 		cpuid = mac_next_bind_cpu(cpupart);
1986 		/* Create the protocol softrings */
1987 		mac_srs_create_proto_softrings(0, soft_ring_flag,
1988 		    mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid,
1989 		    rx_func, x_arg1, x_arg2, B_FALSE);
1990 		mutex_exit(&cpu_lock);
1991 	} else {
1992 		/*
1993 		 * This is the case when there is no fanout which is
1994 		 * true for subflows.
1995 		 */
1996 		mac_rx_srs->srs_type |= SRST_NO_SOFT_RINGS;
1997 	}
1998 	mac_srs_update_fanout_list(mac_rx_srs);
1999 	mac_srs_client_poll_enable(mcip, mac_rx_srs);
2000 }
2001 
2002 /*
2003  * mac_fanout_setup:
2004  *
2005  * Calls mac_srs_fanout_init() or modify() depending upon whether
2006  * the SRS is getting initialized or re-initialized.
2007  */
2008 void
mac_fanout_setup(mac_client_impl_t * mcip,flow_entry_t * flent,mac_resource_props_t * mrp,mac_direct_rx_t rx_func,void * x_arg1,mac_resource_handle_t x_arg2,cpupart_t * cpupart)2009 mac_fanout_setup(mac_client_impl_t *mcip, flow_entry_t *flent,
2010     mac_resource_props_t *mrp, mac_direct_rx_t rx_func, void *x_arg1,
2011     mac_resource_handle_t x_arg2, cpupart_t *cpupart)
2012 {
2013 	mac_soft_ring_set_t *mac_rx_srs, *mac_tx_srs;
2014 	int i, rx_srs_cnt;
2015 
2016 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip));
2017 	/*
2018 	 * This is an aggregation port. Fanout will be setup
2019 	 * over the aggregation itself.
2020 	 */
2021 	if (mcip->mci_state_flags & MCIS_EXCLUSIVE)
2022 		return;
2023 
2024 	mac_rx_srs = flent->fe_rx_srs[0];
2025 	/*
2026 	 * Set up the fanout on the tx side only once, with the
2027 	 * first rx SRS. The CPU binding, fanout, and bandwidth
2028 	 * criteria are common to both RX and TX, so
2029 	 * initializing them along side avoids redundant code.
2030 	 */
2031 	mac_tx_srs = flent->fe_tx_srs;
2032 	rx_srs_cnt = flent->fe_rx_srs_cnt;
2033 
2034 	/* No fanout for subflows */
2035 	if (flent->fe_type & FLOW_USER) {
2036 		mac_srs_fanout_init(mcip, mrp, rx_func,
2037 		    x_arg1, x_arg2, mac_rx_srs, mac_tx_srs,
2038 		    cpupart);
2039 		return;
2040 	}
2041 
2042 	if (mrp->mrp_mask & MRP_CPUS_USERSPEC)
2043 		mac_flow_user_cpu_init(flent, mrp);
2044 	else
2045 		mac_flow_cpu_init(flent, cpupart);
2046 
2047 	mrp->mrp_rx_fanout_cnt = mac_rx_srs->srs_cpu.mc_rx_fanout_cnt;
2048 
2049 	/*
2050 	 * Set up fanout for both SW (0th SRS) and HW classified
2051 	 * SRS (the rest of Rx SRSs in flent).
2052 	 */
2053 	for (i = 0; i < rx_srs_cnt; i++) {
2054 		mac_rx_srs = flent->fe_rx_srs[i];
2055 		if (i != 0)
2056 			mac_tx_srs = NULL;
2057 		switch (mac_rx_srs->srs_fanout_state) {
2058 		case SRS_FANOUT_UNINIT:
2059 			mac_srs_fanout_init(mcip, mrp, rx_func,
2060 			    x_arg1, x_arg2, mac_rx_srs, mac_tx_srs,
2061 			    cpupart);
2062 			break;
2063 		case SRS_FANOUT_INIT:
2064 			break;
2065 		case SRS_FANOUT_REINIT:
2066 			mac_rx_srs_quiesce(mac_rx_srs, SRS_QUIESCE);
2067 			mac_srs_fanout_modify(mcip, rx_func, x_arg1,
2068 			    x_arg2, mac_rx_srs, mac_tx_srs);
2069 			mac_rx_srs_restart(mac_rx_srs);
2070 			break;
2071 		default:
2072 			VERIFY(mac_rx_srs->srs_fanout_state <=
2073 			    SRS_FANOUT_REINIT);
2074 			break;
2075 		}
2076 	}
2077 }
2078 
2079 /*
2080  * mac_srs_create:
2081  *
2082  * Create a mac_soft_ring_set_t (SRS). If soft_ring_fanout_type is
2083  * SRST_TX, an SRS for Tx side is created. Otherwise an SRS for Rx side
2084  * processing is created.
2085  *
2086  * Details on Rx SRS:
2087  * Create a SRS and also add the necessary soft rings for TCP and
2088  * non-TCP based on fanout type and count specified.
2089  *
2090  * mac_soft_ring_fanout, mac_srs_fanout_modify (?),
2091  * mac_soft_ring_stop_workers, mac_soft_ring_set_destroy, etc need
2092  * to be heavily modified.
2093  *
2094  * mi_soft_ring_list_size, mi_soft_ring_size, etc need to disappear.
2095  */
2096 mac_soft_ring_set_t *
mac_srs_create(mac_client_impl_t * mcip,flow_entry_t * flent,uint32_t srs_type,mac_direct_rx_t rx_func,void * x_arg1,mac_resource_handle_t x_arg2,mac_ring_t * ring)2097 mac_srs_create(mac_client_impl_t *mcip, flow_entry_t *flent, uint32_t srs_type,
2098     mac_direct_rx_t rx_func, void *x_arg1, mac_resource_handle_t x_arg2,
2099     mac_ring_t *ring)
2100 {
2101 	mac_soft_ring_set_t 	*mac_srs;
2102 	mac_srs_rx_t		*srs_rx;
2103 	mac_srs_tx_t		*srs_tx;
2104 	mac_bw_ctl_t		*mac_bw;
2105 	mac_resource_props_t	*mrp;
2106 	boolean_t		is_tx_srs = ((srs_type & SRST_TX) != 0);
2107 
2108 	mac_srs = kmem_cache_alloc(mac_srs_cache, KM_SLEEP);
2109 	bzero(mac_srs, sizeof (mac_soft_ring_set_t));
2110 	srs_rx = &mac_srs->srs_rx;
2111 	srs_tx = &mac_srs->srs_tx;
2112 
2113 	mutex_enter(&flent->fe_lock);
2114 
2115 	/*
2116 	 * Get the bandwidth control structure from the flent. Get
2117 	 * rid of any residual values in the control structure for
2118 	 * the tx bw struct and also for the rx, if the rx srs is
2119 	 * the 1st one being brought up (the rx bw ctl struct may
2120 	 * be shared by multiple SRSs)
2121 	 */
2122 	if (is_tx_srs) {
2123 		mac_srs->srs_bw = &flent->fe_tx_bw;
2124 		bzero(mac_srs->srs_bw, sizeof (mac_bw_ctl_t));
2125 		flent->fe_tx_srs = mac_srs;
2126 	} else {
2127 		/*
2128 		 * The bw counter (stored in the flent) is shared
2129 		 * by SRS's within an rx group.
2130 		 */
2131 		mac_srs->srs_bw = &flent->fe_rx_bw;
2132 		/* First rx SRS, clear the bw structure */
2133 		if (flent->fe_rx_srs_cnt == 0)
2134 			bzero(mac_srs->srs_bw, sizeof (mac_bw_ctl_t));
2135 
2136 		/*
2137 		 * It is better to panic here rather than just assert because
2138 		 * on a non-debug kernel we might end up courrupting memory
2139 		 * and making it difficult to debug.
2140 		 */
2141 		if (flent->fe_rx_srs_cnt >= MAX_RINGS_PER_GROUP) {
2142 			panic("Array Overrun detected due to MAC client %p "
2143 			    " having more rings than %d", (void *)mcip,
2144 			    MAX_RINGS_PER_GROUP);
2145 		}
2146 		flent->fe_rx_srs[flent->fe_rx_srs_cnt] = mac_srs;
2147 		flent->fe_rx_srs_cnt++;
2148 	}
2149 	mac_srs->srs_flent = flent;
2150 	mutex_exit(&flent->fe_lock);
2151 
2152 	mac_srs->srs_state = 0;
2153 	mac_srs->srs_type = (srs_type | SRST_NO_SOFT_RINGS);
2154 	mac_srs->srs_worker_cpuid = mac_srs->srs_worker_cpuid_save = -1;
2155 	mac_srs->srs_poll_cpuid = mac_srs->srs_poll_cpuid_save = -1;
2156 	mac_srs->srs_mcip = mcip;
2157 	mac_srs_fanout_list_alloc(mac_srs);
2158 
2159 	/*
2160 	 * For a flow we use the underlying MAC client's priority range with
2161 	 * the priority value to find an absolute priority value. For a MAC
2162 	 * client we use the MAC client's maximum priority as the value.
2163 	 */
2164 	mrp = &flent->fe_effective_props;
2165 	if ((mac_srs->srs_type & SRST_FLOW) != 0) {
2166 		mac_srs->srs_pri = FLOW_PRIORITY(mcip->mci_min_pri,
2167 		    mcip->mci_max_pri, mrp->mrp_priority);
2168 	} else {
2169 		mac_srs->srs_pri = mcip->mci_max_pri;
2170 	}
2171 	/*
2172 	 * We need to insert the SRS in the global list before
2173 	 * binding the SRS and SR threads. Otherwise there is a
2174 	 * is a small window where the cpu reconfig callbacks
2175 	 * may miss the SRS in the list walk and DR could fail
2176 	 * as there are bound threads.
2177 	 */
2178 	mac_srs_add_glist(mac_srs);
2179 
2180 	/* Initialize bw limit */
2181 	if ((mrp->mrp_mask & MRP_MAXBW) != 0) {
2182 		mac_srs->srs_drain_func = mac_rx_srs_drain_bw;
2183 
2184 		mac_bw = mac_srs->srs_bw;
2185 		mutex_enter(&mac_bw->mac_bw_lock);
2186 		mac_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw);
2187 
2188 		/*
2189 		 * Give twice the queuing capability before
2190 		 * dropping packets. The unit is bytes/tick.
2191 		 */
2192 		mac_bw->mac_bw_drop_threshold = mac_bw->mac_bw_limit << 1;
2193 		mutex_exit(&mac_bw->mac_bw_lock);
2194 		mac_srs->srs_type |= SRST_BW_CONTROL;
2195 	} else {
2196 		mac_srs->srs_drain_func = mac_rx_srs_drain;
2197 	}
2198 
2199 	/*
2200 	 * We use the following policy to control Receive
2201 	 * Side Dynamic Polling:
2202 	 * 1) We switch to poll mode anytime the processing thread causes
2203 	 *    a backlog to build up in SRS and its associated Soft Rings
2204 	 *    (sr_poll_pkt_cnt > 0).
2205 	 * 2) As long as the backlog stays under the low water mark
2206 	 *    (sr_lowat), we poll the H/W for more packets.
2207 	 * 3) If the backlog (sr_poll_pkt_cnt) exceeds low water mark, we
2208 	 *    stay in poll mode but don't poll the H/W for more packets.
2209 	 * 4) Anytime in polling mode, if we poll the H/W for packets and
2210 	 *    find nothing plus we have an existing backlog
2211 	 *    (sr_poll_pkt_cnt > 0), we stay in polling mode but don't poll
2212 	 *    the H/W for packets anymore (let the polling thread go to sleep).
2213 	 * 5) Once the backlog is relived (packets are processed) we reenable
2214 	 *    polling (by signalling the poll thread) only when the backlog
2215 	 *    dips below sr_poll_thres.
2216 	 * 6) sr_hiwat is used exclusively when we are not polling capable
2217 	 *    and is used to decide when to drop packets so the SRS queue
2218 	 *    length doesn't grow infinitely.
2219 	 */
2220 	if (!is_tx_srs) {
2221 		srs_rx->sr_hiwat = mac_soft_ring_max_q_cnt;
2222 		/* Low water mark needs to be less than high water mark */
2223 		srs_rx->sr_lowat = mac_soft_ring_min_q_cnt <=
2224 		    mac_soft_ring_max_q_cnt ? mac_soft_ring_min_q_cnt :
2225 		    (mac_soft_ring_max_q_cnt >> 2);
2226 		/* Poll threshold need to be half of low water mark or less */
2227 		srs_rx->sr_poll_thres = mac_soft_ring_poll_thres <=
2228 		    (srs_rx->sr_lowat >> 1) ? mac_soft_ring_poll_thres :
2229 		    (srs_rx->sr_lowat >> 1);
2230 		if (mac_latency_optimize)
2231 			mac_srs->srs_state |= SRS_LATENCY_OPT;
2232 		else
2233 			mac_srs->srs_state |= SRS_SOFTRING_QUEUE;
2234 	}
2235 
2236 	mac_srs->srs_worker = thread_create(NULL, 0,
2237 	    mac_srs_worker, mac_srs, 0, &p0, TS_RUN, mac_srs->srs_pri);
2238 
2239 	if (is_tx_srs) {
2240 		/* Handle everything about Tx SRS and return */
2241 		mac_srs->srs_drain_func = mac_tx_srs_drain;
2242 		srs_tx->st_max_q_cnt = mac_tx_srs_max_q_cnt;
2243 		srs_tx->st_hiwat =
2244 		    (mac_tx_srs_hiwat > mac_tx_srs_max_q_cnt) ?
2245 		    mac_tx_srs_max_q_cnt : mac_tx_srs_hiwat;
2246 		srs_tx->st_arg1 = x_arg1;
2247 		srs_tx->st_arg2 = x_arg2;
2248 		goto done;
2249 	}
2250 
2251 	if ((srs_type & SRST_FLOW) != 0 ||
2252 	    FLOW_TAB_EMPTY(mcip->mci_subflow_tab))
2253 		srs_rx->sr_lower_proc = mac_rx_srs_process;
2254 	else
2255 		srs_rx->sr_lower_proc = mac_rx_srs_subflow_process;
2256 
2257 	srs_rx->sr_func = rx_func;
2258 	srs_rx->sr_arg1 = x_arg1;
2259 	srs_rx->sr_arg2 = x_arg2;
2260 
2261 	if (ring != NULL) {
2262 		uint_t ring_info;
2263 
2264 		/* Is the mac_srs created over the RX default group? */
2265 		if (ring->mr_gh == (mac_group_handle_t)
2266 		    MAC_DEFAULT_RX_GROUP(mcip->mci_mip)) {
2267 			mac_srs->srs_type |= SRST_DEFAULT_GRP;
2268 		}
2269 		mac_srs->srs_ring = ring;
2270 		ring->mr_srs = mac_srs;
2271 		ring->mr_classify_type = MAC_HW_CLASSIFIER;
2272 		ring->mr_flag |= MR_INCIPIENT;
2273 
2274 		if (!(mcip->mci_mip->mi_state_flags & MIS_POLL_DISABLE) &&
2275 		    FLOW_TAB_EMPTY(mcip->mci_subflow_tab) && mac_poll_enable)
2276 			mac_srs->srs_state |= SRS_POLLING_CAPAB;
2277 
2278 		mac_srs->srs_poll_thr = thread_create(NULL, 0,
2279 		    mac_rx_srs_poll_ring, mac_srs, 0, &p0, TS_RUN,
2280 		    mac_srs->srs_pri);
2281 		/*
2282 		 * Some drivers require serialization and don't send
2283 		 * packet chains in interrupt context. For such
2284 		 * drivers, we should always queue in soft ring
2285 		 * so that we get a chance to switch into a polling
2286 		 * mode under backlog.
2287 		 */
2288 		ring_info = mac_hwring_getinfo((mac_ring_handle_t)ring);
2289 		if (ring_info & MAC_RING_RX_ENQUEUE)
2290 			mac_srs->srs_state |= SRS_SOFTRING_QUEUE;
2291 	}
2292 done:
2293 	mac_srs_stat_create(mac_srs);
2294 	return (mac_srs);
2295 }
2296 
2297 /*
2298  * Figure out the number of soft rings required. Its dependant on
2299  * if protocol fanout is required (for LINKs), global settings
2300  * require us to do fanout for performance (based on mac_soft_ring_enable),
2301  * or user has specifically requested fanout.
2302  */
2303 static uint32_t
mac_find_fanout(flow_entry_t * flent,uint32_t link_type)2304 mac_find_fanout(flow_entry_t *flent, uint32_t link_type)
2305 {
2306 	uint32_t			fanout_type;
2307 	mac_resource_props_t		*mrp = &flent->fe_effective_props;
2308 
2309 	/* no fanout for subflows */
2310 	switch (link_type) {
2311 	case SRST_FLOW:
2312 		fanout_type = SRST_NO_SOFT_RINGS;
2313 		break;
2314 	case SRST_LINK:
2315 		fanout_type = SRST_FANOUT_PROTO;
2316 		break;
2317 	}
2318 
2319 	/* A primary NIC/link is being plumbed */
2320 	if (flent->fe_type & FLOW_PRIMARY_MAC) {
2321 		if (mac_soft_ring_enable && mac_rx_soft_ring_count > 1) {
2322 			fanout_type |= SRST_FANOUT_SRC_IP;
2323 		}
2324 	} else if (flent->fe_type & FLOW_VNIC) {
2325 		/* A VNIC is being created */
2326 		if (mrp != NULL && mrp->mrp_ncpus > 0) {
2327 			fanout_type |= SRST_FANOUT_SRC_IP;
2328 		}
2329 	}
2330 
2331 	return (fanout_type);
2332 }
2333 
2334 /*
2335  * Change a group from h/w to s/w classification.
2336  */
2337 void
mac_rx_switch_grp_to_sw(mac_group_t * group)2338 mac_rx_switch_grp_to_sw(mac_group_t *group)
2339 {
2340 	mac_ring_t		*ring;
2341 	mac_soft_ring_set_t	*mac_srs;
2342 
2343 	for (ring = group->mrg_rings; ring != NULL; ring = ring->mr_next) {
2344 		if (ring->mr_classify_type == MAC_HW_CLASSIFIER) {
2345 			/*
2346 			 * Remove the SRS associated with the HW ring.
2347 			 * As a result, polling will be disabled.
2348 			 */
2349 			mac_srs = ring->mr_srs;
2350 			ASSERT(mac_srs != NULL);
2351 			mac_rx_srs_remove(mac_srs);
2352 			ring->mr_srs = NULL;
2353 		}
2354 
2355 		if (ring->mr_state != MR_INUSE)
2356 			(void) mac_start_ring(ring);
2357 
2358 		/*
2359 		 * We need to perform SW classification
2360 		 * for packets landing in these rings
2361 		 */
2362 		ring->mr_flag = 0;
2363 		ring->mr_classify_type = MAC_SW_CLASSIFIER;
2364 	}
2365 }
2366 
2367 /*
2368  * Create the Rx SRS for S/W classifier and for each ring in the
2369  * group (if exclusive group). Also create the Tx SRS.
2370  */
2371 void
mac_srs_group_setup(mac_client_impl_t * mcip,flow_entry_t * flent,uint32_t link_type)2372 mac_srs_group_setup(mac_client_impl_t *mcip, flow_entry_t *flent,
2373     uint32_t link_type)
2374 {
2375 	cpupart_t		*cpupart;
2376 	mac_resource_props_t	*mrp = MCIP_RESOURCE_PROPS(mcip);
2377 	mac_resource_props_t	*emrp = MCIP_EFFECTIVE_PROPS(mcip);
2378 	boolean_t		use_default = B_FALSE;
2379 
2380 	mac_rx_srs_group_setup(mcip, flent, link_type);
2381 	mac_tx_srs_group_setup(mcip, flent, link_type);
2382 
2383 	pool_lock();
2384 	cpupart = mac_pset_find(mrp, &use_default);
2385 	mac_fanout_setup(mcip, flent, MCIP_RESOURCE_PROPS(mcip),
2386 	    mac_rx_deliver, mcip, NULL, cpupart);
2387 	mac_set_pool_effective(use_default, cpupart, mrp, emrp);
2388 	pool_unlock();
2389 }
2390 
2391 /*
2392  * Set up the RX SRSs. If the S/W SRS is not set, set  it up, if there
2393  * is a group associated with this MAC client, set up SRSs for individual
2394  * h/w rings.
2395  */
2396 void
mac_rx_srs_group_setup(mac_client_impl_t * mcip,flow_entry_t * flent,uint32_t link_type)2397 mac_rx_srs_group_setup(mac_client_impl_t *mcip, flow_entry_t *flent,
2398     uint32_t link_type)
2399 {
2400 	mac_impl_t		*mip = mcip->mci_mip;
2401 	mac_soft_ring_set_t	*mac_srs;
2402 	mac_ring_t 		*ring;
2403 	uint32_t		fanout_type;
2404 	mac_group_t		*rx_group = flent->fe_rx_ring_group;
2405 
2406 	fanout_type = mac_find_fanout(flent, link_type);
2407 
2408 	/* Create the SRS for S/W classification if none exists */
2409 	if (flent->fe_rx_srs[0] == NULL) {
2410 		ASSERT(flent->fe_rx_srs_cnt == 0);
2411 		/* Setup the Rx SRS */
2412 		mac_srs = mac_srs_create(mcip, flent, fanout_type | link_type,
2413 		    mac_rx_deliver, mcip, NULL, NULL);
2414 		mutex_enter(&flent->fe_lock);
2415 		flent->fe_cb_fn = (flow_fn_t)mac_srs->srs_rx.sr_lower_proc;
2416 		flent->fe_cb_arg1 = (void *)mip;
2417 		flent->fe_cb_arg2 = (void *)mac_srs;
2418 		mutex_exit(&flent->fe_lock);
2419 	}
2420 
2421 	if (rx_group == NULL)
2422 		return;
2423 	/*
2424 	 * fanout for default SRS is done when default SRS are created
2425 	 * above. As each ring is added to the group, we setup the
2426 	 * SRS and fanout to it.
2427 	 */
2428 	switch (rx_group->mrg_state) {
2429 	case MAC_GROUP_STATE_RESERVED:
2430 		for (ring = rx_group->mrg_rings; ring != NULL;
2431 		    ring = ring->mr_next) {
2432 			switch (ring->mr_state) {
2433 			case MR_INUSE:
2434 			case MR_FREE:
2435 				if (ring->mr_srs != NULL)
2436 					break;
2437 				if (ring->mr_state != MR_INUSE)
2438 					(void) mac_start_ring(ring);
2439 
2440 				/*
2441 				 * Since the group is exclusively ours create
2442 				 * an SRS for this ring to allow the
2443 				 * individual SRS to dynamically poll the
2444 				 * ring. Do this only if the  client is not
2445 				 * a VLAN MAC client, since for VLAN we do
2446 				 * s/w classification for the VID check, and
2447 				 * if it has a unicast address.
2448 				 */
2449 				if ((mcip->mci_state_flags &
2450 				    MCIS_NO_UNICAST_ADDR) ||
2451 				    i_mac_flow_vid(mcip->mci_flent) !=
2452 				    VLAN_ID_NONE) {
2453 					break;
2454 				}
2455 				mac_srs = mac_srs_create(mcip, flent,
2456 				    fanout_type | link_type,
2457 				    mac_rx_deliver, mcip, NULL, ring);
2458 				break;
2459 			default:
2460 				cmn_err(CE_PANIC,
2461 				    "srs_setup: mcip = %p "
2462 				    "trying to add UNKNOWN ring = %p\n",
2463 				    (void *)mcip, (void *)ring);
2464 				break;
2465 			}
2466 		}
2467 		break;
2468 	case MAC_GROUP_STATE_SHARED:
2469 		/*
2470 		 * Set all rings of this group to software classified.
2471 		 *
2472 		 * If the group is current RESERVED, the existing mac
2473 		 * client (the only client on this group) is using
2474 		 * this group exclusively.  In that case we need to
2475 		 * disable polling on the rings of the group (if it
2476 		 * was enabled), and free the SRS associated with the
2477 		 * rings.
2478 		 */
2479 		mac_rx_switch_grp_to_sw(rx_group);
2480 		break;
2481 	default:
2482 		ASSERT(B_FALSE);
2483 		break;
2484 	}
2485 }
2486 
2487 /*
2488  * Set up the TX SRS.
2489  */
2490 void
mac_tx_srs_group_setup(mac_client_impl_t * mcip,flow_entry_t * flent,uint32_t link_type)2491 mac_tx_srs_group_setup(mac_client_impl_t *mcip, flow_entry_t *flent,
2492     uint32_t link_type)
2493 {
2494 	int			cnt;
2495 	int			ringcnt;
2496 	mac_ring_t		*ring;
2497 	mac_group_t		*grp;
2498 
2499 	/*
2500 	 * If we are opened exclusively (like aggr does for aggr_ports),
2501 	 * don't set up Tx SRS and Tx soft rings as they won't be used.
2502 	 * The same thing has to be done for Rx side also. See bug:
2503 	 * 6880080
2504 	 */
2505 	if (mcip->mci_state_flags & MCIS_EXCLUSIVE) {
2506 		/*
2507 		 * If we have rings, start them here.
2508 		 */
2509 		if (flent->fe_tx_ring_group == NULL)
2510 			return;
2511 		grp = (mac_group_t *)flent->fe_tx_ring_group;
2512 		ringcnt = grp->mrg_cur_count;
2513 		ring = grp->mrg_rings;
2514 		for (cnt = 0; cnt < ringcnt; cnt++) {
2515 			if (ring->mr_state != MR_INUSE) {
2516 				(void) mac_start_ring(ring);
2517 			}
2518 			ring = ring->mr_next;
2519 		}
2520 		return;
2521 	}
2522 	if (flent->fe_tx_srs == NULL) {
2523 		(void) mac_srs_create(mcip, flent, SRST_TX | link_type,
2524 		    NULL, mcip, NULL, NULL);
2525 	}
2526 	mac_tx_srs_setup(mcip, flent);
2527 }
2528 
2529 /*
2530  * Remove all the RX SRSs. If we want to remove only the SRSs associated
2531  * with h/w rings, leave the S/W SRS alone. This is used when we want to
2532  * move the MAC client from one group to another, so we need to teardown
2533  * on the h/w SRSs.
2534  */
2535 void
mac_rx_srs_group_teardown(flow_entry_t * flent,boolean_t hwonly)2536 mac_rx_srs_group_teardown(flow_entry_t *flent, boolean_t hwonly)
2537 {
2538 	mac_soft_ring_set_t	*mac_srs;
2539 	int			i;
2540 	int			count = flent->fe_rx_srs_cnt;
2541 
2542 	for (i = 0; i < count; i++) {
2543 		if (i == 0 && hwonly)
2544 			continue;
2545 		mac_srs = flent->fe_rx_srs[i];
2546 		mac_rx_srs_quiesce(mac_srs, SRS_CONDEMNED);
2547 		mac_srs_free(mac_srs);
2548 		flent->fe_rx_srs[i] = NULL;
2549 		flent->fe_rx_srs_cnt--;
2550 	}
2551 	ASSERT(!hwonly || flent->fe_rx_srs_cnt == 1);
2552 	ASSERT(hwonly || flent->fe_rx_srs_cnt == 0);
2553 }
2554 
2555 /*
2556  * Remove the TX SRS.
2557  */
2558 void
mac_tx_srs_group_teardown(mac_client_impl_t * mcip,flow_entry_t * flent,uint32_t link_type)2559 mac_tx_srs_group_teardown(mac_client_impl_t *mcip, flow_entry_t *flent,
2560     uint32_t link_type)
2561 {
2562 	mac_soft_ring_set_t	*tx_srs;
2563 	mac_srs_tx_t		*tx;
2564 
2565 	if ((tx_srs = flent->fe_tx_srs) == NULL)
2566 		return;
2567 
2568 	tx = &tx_srs->srs_tx;
2569 	switch (link_type) {
2570 	case SRST_FLOW:
2571 		/*
2572 		 * For flows, we need to work with passed
2573 		 * flent to find the Rx/Tx SRS.
2574 		 */
2575 		mac_tx_srs_quiesce(tx_srs, SRS_CONDEMNED);
2576 		break;
2577 	case SRST_LINK:
2578 		mac_tx_client_condemn((mac_client_handle_t)mcip);
2579 		if (tx->st_arg2 != NULL) {
2580 			ASSERT(tx_srs->srs_type & SRST_TX);
2581 			/*
2582 			 * The ring itself will be stopped when
2583 			 * we release the group or in the
2584 			 * mac_datapath_teardown (for the default
2585 			 * group)
2586 			 */
2587 			tx->st_arg2 = NULL;
2588 		}
2589 		break;
2590 	default:
2591 		ASSERT(B_FALSE);
2592 		break;
2593 	}
2594 	mac_srs_free(tx_srs);
2595 	flent->fe_tx_srs = NULL;
2596 }
2597 
2598 /*
2599  * This is the group state machine.
2600  *
2601  * The state of an Rx group is given by
2602  * the following table. The default group and its rings are started in
2603  * mac_start itself and the default group stays in SHARED state until
2604  * mac_stop at which time the group and rings are stopped and and it
2605  * reverts to the Registered state.
2606  *
2607  * Typically this function is called on a group after adding or removing a
2608  * client from it, to find out what should be the new state of the group.
2609  * If the new state is RESERVED, then the client that owns this group
2610  * exclusively is also returned. Note that adding or removing a client from
2611  * a group could also impact the default group and the caller needs to
2612  * evaluate the effect on the default group.
2613  *
2614  * Group type		# of clients	mi_nactiveclients	Group State
2615  *			in the group
2616  *
2617  * Non-default		0		N.A.			REGISTERED
2618  * Non-default		1		N.A.			RESERVED
2619  *
2620  * Default		0		N.A.			SHARED
2621  * Default		1		1			RESERVED
2622  * Default		1		> 1			SHARED
2623  * Default		> 1		N.A.			SHARED
2624  *
2625  * For a TX group, the following is the state table.
2626  *
2627  * Group type		# of clients	Group State
2628  *			in the group
2629  *
2630  * Non-default		0		REGISTERED
2631  * Non-default		1		RESERVED
2632  *
2633  * Default		0		REGISTERED
2634  * Default		1		RESERVED
2635  * Default		> 1		SHARED
2636  */
2637 mac_group_state_t
mac_group_next_state(mac_group_t * grp,mac_client_impl_t ** group_only_mcip,mac_group_t * defgrp,boolean_t rx_group)2638 mac_group_next_state(mac_group_t *grp, mac_client_impl_t **group_only_mcip,
2639     mac_group_t *defgrp, boolean_t rx_group)
2640 {
2641 	mac_impl_t		*mip = (mac_impl_t *)grp->mrg_mh;
2642 
2643 	*group_only_mcip = NULL;
2644 
2645 	/* Non-default group */
2646 
2647 	if (grp != defgrp) {
2648 		if (MAC_GROUP_NO_CLIENT(grp))
2649 			return (MAC_GROUP_STATE_REGISTERED);
2650 
2651 		*group_only_mcip = MAC_GROUP_ONLY_CLIENT(grp);
2652 		if (*group_only_mcip != NULL)
2653 			return (MAC_GROUP_STATE_RESERVED);
2654 
2655 		return (MAC_GROUP_STATE_SHARED);
2656 	}
2657 
2658 	/* Default group */
2659 
2660 	if (MAC_GROUP_NO_CLIENT(grp)) {
2661 		if (rx_group)
2662 			return (MAC_GROUP_STATE_SHARED);
2663 		else
2664 			return (MAC_GROUP_STATE_REGISTERED);
2665 	}
2666 	*group_only_mcip = MAC_GROUP_ONLY_CLIENT(grp);
2667 	if (*group_only_mcip == NULL)
2668 		return (MAC_GROUP_STATE_SHARED);
2669 
2670 	if (rx_group && mip->mi_nactiveclients != 1)
2671 		return (MAC_GROUP_STATE_SHARED);
2672 
2673 	ASSERT(*group_only_mcip != NULL);
2674 	return (MAC_GROUP_STATE_RESERVED);
2675 }
2676 
2677 /*
2678  * OVERVIEW NOTES FOR DATAPATH
2679  * ===========================
2680  *
2681  * Create an SRS and setup the corresponding flow function and args.
2682  * Add a classification rule for the flow specified by 'flent' and program
2683  * the hardware classifier when applicable.
2684  *
2685  * Rx ring assignment, SRS, polling and B/W enforcement
2686  * ----------------------------------------------------
2687  *
2688  * We try to use H/W classification on NIC and assign traffic to a
2689  * MAC address to a particular Rx ring. There is a 1-1 mapping
2690  * between a SRS and a Rx ring. The SRS (short for soft ring set)
2691  * dynamically switches the underlying Rx ring between interrupt
2692  * and polling mode and enforces any specified B/W control.
2693  *
2694  * There is always a SRS created and tied to each H/W and S/W rule.
2695  * Whenever we create a H/W rule, we always add the the same rule to
2696  * S/W classifier and tie a SRS to it.
2697  *
2698  * In case a B/W control is specified, its broken into bytes
2699  * per ticks and as soon as the quota for a tick is exhausted,
2700  * the underlying Rx ring is forced into poll mode for remianing
2701  * tick. The SRS poll thread only polls for bytes that are
2702  * allowed to come in the SRS. We typically let 4x the configured
2703  * B/W worth of packets to come in the SRS (to prevent unnecessary
2704  * drops due to bursts) but only process the specified amount.
2705  *
2706  * A Link (primary NIC, VNIC, VLAN or aggr) can have 1 or more
2707  * Rx rings (and corresponding SRSs) assigned to it. The SRS
2708  * in turn can have softrings to do protocol level fanout or
2709  * softrings to do S/W based fanout or both. In case the NIC
2710  * has no Rx rings, we do S/W classification to respective SRS.
2711  * The S/W classification rule is always setup and ready. This
2712  * allows the MAC layer to reassign Rx rings whenever needed
2713  * but packets still continue to flow via the default path and
2714  * getting S/W classified to correct SRS.
2715  *
2716  * In other cases where a NIC or VNIC is plumbed, our goal is use
2717  * H/W classifier and get two Rx ring assigned for the Link. One
2718  * for TCP and one for UDP|SCTP. The respective SRS still do the
2719  * polling on the Rx ring. For Link that is plumbed for IP, there
2720  * is a TCP squeue which also does polling and can control the
2721  * the Rx ring directly (where SRS is just pass through). For
2722  * the following cases, the SRS does the polling underneath.
2723  * 1) non IP based Links (Links which are not plumbed via ifconfig)
2724  *    and paths which have no IP squeues (UDP & SCTP)
2725  * 2) If B/W control is specified on the Link
2726  * 3) If S/W fanout is secified
2727  *
2728  * Note1: As of current implementation, we try to assign only 1 Rx
2729  * ring per Link and more than 1 Rx ring for primary Link for
2730  * H/W based fanout. We always create following softrings per SRS:
2731  * 1) TCP softring which is polled by TCP squeue where possible
2732  *    (and also bypasses DLS)
2733  * 2) UDP/SCTP based which bypasses DLS
2734  * 3) OTH softring which goes via DLS (currently deal with IPv6
2735  *    and non TCP/UDP/SCTP for IPv4 packets).
2736  *
2737  * It is necessary to create 3 softrings since SRS has to poll
2738  * the single Rx ring underneath and enforce any link level B/W
2739  * control (we can't switch the Rx ring in poll mode just based
2740  * on TCP squeue if the same Rx ring is sharing UDP and other
2741  * traffic as well). Once polling is done and any Link level B/W
2742  * control is specified, the packets are assigned to respective
2743  * softring based on protocol. Since TCP has IP based squeue
2744  * which benefits by polling, we separate TCP packets into
2745  * its own softring which can be polled by IP squeue. We need
2746  * to separate out UDP/SCTP to UDP softring since it can bypass
2747  * the DLS layer which has heavy performance advanatges and we
2748  * need a softring (OTH) for rest.
2749  *
2750  * ToDo: The 3 softrings for protocol are needed only till we can
2751  * get rid of DLS from datapath, make IPv4 and IPv6 paths
2752  * symmetric (deal with mac_header_info for v6 and polling for
2753  * IPv4 TCP - ip_accept_tcp is IPv4 specific although squeues
2754  * are generic), and bring SAP based classification to MAC layer
2755  *
2756  * H/W and S/W based fanout and multiple Rx rings per Link
2757  * -------------------------------------------------------
2758  *
2759  * In case, fanout is requested (or determined automatically based
2760  * on Link speed and processor speed), we try to assign multiple
2761  * Rx rings per Link with their respective SRS. In this case
2762  * the NIC should be capable of fanning out incoming packets between
2763  * the assigned Rx rings (H/W based fanout). All the SRS
2764  * individually switch their Rx ring between interrupt and polling
2765  * mode but share a common B/W control counter in case of Link
2766  * level B/W is specified.
2767  *
2768  * If S/W based fanout is specified in lieu of H/W based fanout,
2769  * the Link SRS creates the specified number of softrings for
2770  * each protocol (TCP, UDP, OTH). Incoming packets are fanned
2771  * out to the correct softring based on their protocol and
2772  * protocol specific hash function.
2773  *
2774  * Primary and non primary MAC clients
2775  * -----------------------------------
2776  *
2777  * The NICs, VNICs, Vlans, and Aggrs are typically termed as Links
2778  * and are a Layer 2 construct.
2779  *
2780  * Primary NIC:
2781  *	The Link that owns the primary MAC address and typically
2782  *	is used as the data NIC in non virtualized cases. As such
2783  *	H/W resources are preferntially given to primary NIC. As
2784  *	far as code is concerned, there is no difference in the
2785  *	primary NIC vs VNICs. They are all treated as Links.
2786  *	At the very first call to mac_unicast_add() we program the S/W
2787  *	classifier for the primary MAC address, get a soft ring set
2788  *	(and soft rings based on 'ip_soft_ring_cnt')
2789  *	and a Rx ring assigned for polling to get enabled.
2790  *	When IP get plumbed and negotiates polling, we can
2791  *	let squeue do the polling on TCP softring.
2792  *
2793  * VNICs:
2794  *	Same as any other Link. As long as the H/W resource assignments
2795  *	are equal, the data path and setup for all Links is same.
2796  *
2797  * Flows:
2798  *	Can be configured on Links. They have their own SRS and the
2799  *	S/W classifier is programmed appropriately based on the flow.
2800  *	The flows typically deal with layer 3 and above and
2801  *	creates a soft ring set specific to the flow. The receive
2802  *	side function is switched from mac_rx_srs_process to
2803  *	mac_rx_srs_subflow_process which first tries to assign the
2804  *	packet to appropriate flow SRS and failing which assigns it
2805  *	to link SRS. This allows us to avoid the layered approach
2806  *	which gets complex.
2807  *
2808  * By the time mac_datapath_setup() completes, we already have the
2809  * soft rings set, Rx rings, soft rings, etc figured out and both H/W
2810  * and S/W classifiers programmed. IP is not plumbed yet (and might
2811  * never be for Virtual Machines guest OS path). When IP is plumbed
2812  * (for both NIC and VNIC), we do a capability negotiation for polling
2813  * and upcall functions etc.
2814  *
2815  * Rx ring Assignement NOTES
2816  * -------------------------
2817  *
2818  * For NICs which have only 1 Rx ring (we treat  NICs with no Rx rings
2819  * as NIC with a single default ring), we assign the only ring to
2820  * primary Link. The primary Link SRS can do polling on it as long as
2821  * it is the only link in use and we compare the MAC address for unicast
2822  * packets before accepting an incoming packet (there is no need for S/W
2823  * classification in this case). We disable polling on the only ring the
2824  * moment 2nd link gets created (the polling remains enabled even though
2825  * there are broadcast and * multicast flows created).
2826  *
2827  * If the NIC has more than 1 Rx ring, we assign the default ring (the
2828  * 1st ring) to deal with broadcast, multicast and traffic for other
2829  * NICs which needs S/W classification. We assign the primary mac
2830  * addresses to another ring by specifiying a classification rule for
2831  * primary unicast MAC address to the selected ring. The primary Link
2832  * (and its SRS) can continue to poll the assigned Rx ring at all times
2833  * independantly.
2834  *
2835  * Note: In future, if no fanout is specified, we try to assign 2 Rx
2836  * rings for the primary Link with the primary MAC address + TCP going
2837  * to one ring and primary MAC address + UDP|SCTP going to other ring.
2838  * Any remaining traffic for primary MAC address can go to the default
2839  * Rx ring and get S/W classified. This way the respective SRSs don't
2840  * need to do proto fanout and don't need to have softrings at all and
2841  * can poll their respective Rx rings.
2842  *
2843  * As an optimization, when a new NIC or VNIC is created, we can get
2844  * only one Rx ring and make it a TCP specific Rx ring and use the
2845  * H/W default Rx ring for the rest (this Rx ring is never polled).
2846  *
2847  * For clients that don't have MAC address, but want to receive and
2848  * transmit packets (e.g, bpf, gvrp etc.), we need to setup the datapath.
2849  * For such clients (identified by the MCIS_NO_UNICAST_ADDR flag) we
2850  * always give the default group and use software classification (i.e.
2851  * even if this is the only client in the default group, we will
2852  * leave group as shared).
2853  */
2854 int
mac_datapath_setup(mac_client_impl_t * mcip,flow_entry_t * flent,uint32_t link_type)2855 mac_datapath_setup(mac_client_impl_t *mcip, flow_entry_t *flent,
2856     uint32_t link_type)
2857 {
2858 	mac_impl_t		*mip = mcip->mci_mip;
2859 	mac_group_t		*rgroup = NULL;
2860 	mac_group_t		*tgroup = NULL;
2861 	mac_group_t		*default_rgroup;
2862 	mac_group_t		*default_tgroup;
2863 	int			err;
2864 	uint8_t 		*mac_addr;
2865 	mac_group_state_t	next_state;
2866 	mac_client_impl_t	*group_only_mcip;
2867 	mac_resource_props_t	*mrp = MCIP_RESOURCE_PROPS(mcip);
2868 	mac_resource_props_t	*emrp = MCIP_EFFECTIVE_PROPS(mcip);
2869 	boolean_t		rxhw;
2870 	boolean_t		txhw;
2871 	boolean_t		use_default = B_FALSE;
2872 	cpupart_t		*cpupart;
2873 	boolean_t		no_unicast;
2874 	boolean_t		isprimary = flent->fe_type & FLOW_PRIMARY_MAC;
2875 	mac_client_impl_t	*reloc_pmcip = NULL;
2876 
2877 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mip));
2878 
2879 	switch (link_type) {
2880 	case SRST_FLOW:
2881 		mac_srs_group_setup(mcip, flent, link_type);
2882 		return (0);
2883 
2884 	case SRST_LINK:
2885 		no_unicast = mcip->mci_state_flags & MCIS_NO_UNICAST_ADDR;
2886 		mac_addr = flent->fe_flow_desc.fd_dst_mac;
2887 
2888 		/* Default RX group */
2889 		default_rgroup = MAC_DEFAULT_RX_GROUP(mip);
2890 
2891 		/* Default TX group */
2892 		default_tgroup = MAC_DEFAULT_TX_GROUP(mip);
2893 
2894 		if (no_unicast) {
2895 			rgroup = default_rgroup;
2896 			tgroup = default_tgroup;
2897 			goto grp_found;
2898 		}
2899 		rxhw = (mrp->mrp_mask & MRP_RX_RINGS) &&
2900 		    (mrp->mrp_nrxrings > 0 ||
2901 		    (mrp->mrp_mask & MRP_RXRINGS_UNSPEC));
2902 		txhw = (mrp->mrp_mask & MRP_TX_RINGS) &&
2903 		    (mrp->mrp_ntxrings > 0 ||
2904 		    (mrp->mrp_mask & MRP_TXRINGS_UNSPEC));
2905 
2906 		/*
2907 		 * By default we have given the primary all the rings
2908 		 * i.e. the default group. Let's see if the primary
2909 		 * needs to be relocated so that the addition of this
2910 		 * client doesn't impact the primary's performance,
2911 		 * i.e. if the primary is in the default group and
2912 		 * we add this client, the primary will lose polling.
2913 		 * We do this only for NICs supporting dynamic ring
2914 		 * grouping and only when this is the first client
2915 		 * after the primary (i.e. nactiveclients is 2)
2916 		 */
2917 		if (!isprimary && mip->mi_nactiveclients == 2 &&
2918 		    (group_only_mcip = mac_primary_client_handle(mip)) !=
2919 		    NULL && mip->mi_rx_group_type == MAC_GROUP_TYPE_DYNAMIC) {
2920 			reloc_pmcip = mac_check_primary_relocation(
2921 			    group_only_mcip, rxhw);
2922 		}
2923 		/*
2924 		 * Check to see if we can get an exclusive group for
2925 		 * this mac address or if there already exists a
2926 		 * group that has this mac address (case of VLANs).
2927 		 * If no groups are available, use the default group.
2928 		 */
2929 		rgroup = mac_reserve_rx_group(mcip, mac_addr, B_FALSE);
2930 		if (rgroup == NULL && rxhw) {
2931 			err = ENOSPC;
2932 			goto setup_failed;
2933 		} else if (rgroup == NULL) {
2934 			rgroup = default_rgroup;
2935 		}
2936 		/*
2937 		 * Check to see if we can get an exclusive group for
2938 		 * this mac client. If no groups are available, use
2939 		 * the default group.
2940 		 */
2941 		tgroup = mac_reserve_tx_group(mcip, B_FALSE);
2942 		if (tgroup == NULL && txhw) {
2943 			if (rgroup != NULL && rgroup != default_rgroup)
2944 				mac_release_rx_group(mcip, rgroup);
2945 			err = ENOSPC;
2946 			goto setup_failed;
2947 		} else if (tgroup == NULL) {
2948 			tgroup = default_tgroup;
2949 		}
2950 
2951 		/*
2952 		 * Some NICs don't support any Rx rings, so there may not
2953 		 * even be a default group.
2954 		 */
2955 	grp_found:
2956 		if (rgroup != NULL) {
2957 			if (rgroup != default_rgroup &&
2958 			    MAC_GROUP_NO_CLIENT(rgroup) &&
2959 			    (rxhw || mcip->mci_share != NULL)) {
2960 				MAC_RX_GRP_RESERVED(mip);
2961 				if (mip->mi_rx_group_type ==
2962 				    MAC_GROUP_TYPE_DYNAMIC) {
2963 					MAC_RX_RING_RESERVED(mip,
2964 					    rgroup->mrg_cur_count);
2965 				}
2966 			}
2967 			flent->fe_rx_ring_group = rgroup;
2968 			/*
2969 			 * Add the client to the group. This could cause
2970 			 * either this group to move to the shared state or
2971 			 * cause the default group to move to the shared state.
2972 			 * The actions on this group are done here, while the
2973 			 * actions on the default group are postponed to
2974 			 * the end of this function.
2975 			 */
2976 			mac_group_add_client(rgroup, mcip);
2977 			next_state = mac_group_next_state(rgroup,
2978 			    &group_only_mcip, default_rgroup, B_TRUE);
2979 			mac_set_group_state(rgroup, next_state);
2980 		}
2981 
2982 		if (tgroup != NULL) {
2983 			if (tgroup != default_tgroup &&
2984 			    MAC_GROUP_NO_CLIENT(tgroup) &&
2985 			    (txhw || mcip->mci_share != NULL)) {
2986 				MAC_TX_GRP_RESERVED(mip);
2987 				if (mip->mi_tx_group_type ==
2988 				    MAC_GROUP_TYPE_DYNAMIC) {
2989 					MAC_TX_RING_RESERVED(mip,
2990 					    tgroup->mrg_cur_count);
2991 				}
2992 			}
2993 			flent->fe_tx_ring_group = tgroup;
2994 			mac_group_add_client(tgroup, mcip);
2995 			next_state = mac_group_next_state(tgroup,
2996 			    &group_only_mcip, default_tgroup, B_FALSE);
2997 			tgroup->mrg_state = next_state;
2998 		}
2999 		/*
3000 		 * Setup the Rx and Tx SRSes. If we got a pristine group
3001 		 * exclusively above, mac_srs_group_setup would simply create
3002 		 * the required SRSes. If we ended up sharing a previously
3003 		 * reserved group, mac_srs_group_setup would also dismantle the
3004 		 * SRSes of the previously exclusive group
3005 		 */
3006 		mac_srs_group_setup(mcip, flent, link_type);
3007 
3008 		/* We are setting up minimal datapath only */
3009 		if (no_unicast)
3010 			break;
3011 		/* Program the S/W Classifer */
3012 		if ((err = mac_flow_add(mip->mi_flow_tab, flent)) != 0)
3013 			goto setup_failed;
3014 
3015 		/* Program the H/W Classifier */
3016 		if ((err = mac_add_macaddr(mip, rgroup, mac_addr,
3017 		    (mcip->mci_state_flags & MCIS_UNICAST_HW) != 0)) != 0)
3018 			goto setup_failed;
3019 		mcip->mci_unicast = mac_find_macaddr(mip, mac_addr);
3020 		ASSERT(mcip->mci_unicast != NULL);
3021 		/* Initialize the v6 local addr used by link protection */
3022 		mac_protect_update_v6_local_addr(mcip);
3023 		break;
3024 
3025 	default:
3026 		ASSERT(B_FALSE);
3027 		break;
3028 	}
3029 
3030 	/*
3031 	 * All broadcast and multicast traffic is received only on the default
3032 	 * group. If we have setup the datapath for a non-default group above
3033 	 * then move the default group to shared state to allow distribution of
3034 	 * incoming broadcast traffic to the other groups and dismantle the
3035 	 * SRSes over the default group.
3036 	 */
3037 	if (rgroup != NULL) {
3038 		if (rgroup != default_rgroup) {
3039 			if (default_rgroup->mrg_state ==
3040 			    MAC_GROUP_STATE_RESERVED) {
3041 				group_only_mcip = MAC_GROUP_ONLY_CLIENT(
3042 				    default_rgroup);
3043 				ASSERT(group_only_mcip != NULL &&
3044 				    mip->mi_nactiveclients > 1);
3045 
3046 				mac_set_group_state(default_rgroup,
3047 				    MAC_GROUP_STATE_SHARED);
3048 				mac_rx_srs_group_setup(group_only_mcip,
3049 				    group_only_mcip->mci_flent, SRST_LINK);
3050 				pool_lock();
3051 				cpupart = mac_pset_find(mrp, &use_default);
3052 				mac_fanout_setup(group_only_mcip,
3053 				    group_only_mcip->mci_flent,
3054 				    MCIP_RESOURCE_PROPS(group_only_mcip),
3055 				    mac_rx_deliver, group_only_mcip, NULL,
3056 				    cpupart);
3057 				mac_set_pool_effective(use_default, cpupart,
3058 				    mrp, emrp);
3059 				pool_unlock();
3060 			}
3061 			ASSERT(default_rgroup->mrg_state ==
3062 			    MAC_GROUP_STATE_SHARED);
3063 		}
3064 		/*
3065 		 * If we get an exclusive group for a VLAN MAC client we
3066 		 * need to take the s/w path to make the additional check for
3067 		 * the vid. Disable polling and set it to s/w classification.
3068 		 * Similarly for clients that don't have a unicast address.
3069 		 */
3070 		if (rgroup->mrg_state == MAC_GROUP_STATE_RESERVED &&
3071 		    (i_mac_flow_vid(flent) != VLAN_ID_NONE || no_unicast)) {
3072 			mac_rx_switch_grp_to_sw(rgroup);
3073 		}
3074 	}
3075 	mac_set_rings_effective(mcip);
3076 	return (0);
3077 
3078 setup_failed:
3079 	/* Switch the primary back to default group */
3080 	if (reloc_pmcip != NULL) {
3081 		(void) mac_rx_switch_group(reloc_pmcip,
3082 		    reloc_pmcip->mci_flent->fe_rx_ring_group, default_rgroup);
3083 	}
3084 	mac_datapath_teardown(mcip, flent, link_type);
3085 	return (err);
3086 }
3087 
3088 void
mac_datapath_teardown(mac_client_impl_t * mcip,flow_entry_t * flent,uint32_t link_type)3089 mac_datapath_teardown(mac_client_impl_t *mcip, flow_entry_t *flent,
3090     uint32_t link_type)
3091 {
3092 	mac_impl_t		*mip = mcip->mci_mip;
3093 	mac_group_t		*group = NULL;
3094 	mac_client_impl_t	*grp_only_mcip;
3095 	flow_entry_t		*group_only_flent;
3096 	mac_group_t		*default_group;
3097 	boolean_t		check_default_group = B_FALSE;
3098 	mac_group_state_t	next_state;
3099 	mac_resource_props_t	*mrp = MCIP_RESOURCE_PROPS(mcip);
3100 
3101 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mip));
3102 
3103 	switch (link_type) {
3104 	case SRST_FLOW:
3105 		mac_rx_srs_group_teardown(flent, B_FALSE);
3106 		mac_tx_srs_group_teardown(mcip, flent, SRST_FLOW);
3107 		return;
3108 
3109 	case SRST_LINK:
3110 		/* Stop sending packets */
3111 		mac_tx_client_block(mcip);
3112 
3113 		/* Stop the packets coming from the H/W */
3114 		if (mcip->mci_unicast != NULL) {
3115 			int err;
3116 			err = mac_remove_macaddr(mcip->mci_unicast);
3117 			if (err != 0) {
3118 				cmn_err(CE_WARN, "%s: failed to remove a MAC"
3119 				    " address because of error 0x%x",
3120 				    mip->mi_name, err);
3121 			}
3122 			mcip->mci_unicast = NULL;
3123 		}
3124 
3125 		/* Stop the packets coming from the S/W classifier */
3126 		mac_flow_remove(mip->mi_flow_tab, flent, B_FALSE);
3127 		mac_flow_wait(flent, FLOW_DRIVER_UPCALL);
3128 
3129 		/* Now quiesce and destroy all SRS and soft rings */
3130 		mac_rx_srs_group_teardown(flent, B_FALSE);
3131 		mac_tx_srs_group_teardown(mcip, flent, SRST_LINK);
3132 
3133 		ASSERT((mcip->mci_flent == flent) &&
3134 		    (flent->fe_next == NULL));
3135 
3136 		/*
3137 		 * Release our hold on the group as well. We need
3138 		 * to check if the shared group has only one client
3139 		 * left who can use it exclusively. Also, if we
3140 		 * were the last client, release the group.
3141 		 */
3142 		group = flent->fe_rx_ring_group;
3143 		default_group = MAC_DEFAULT_RX_GROUP(mip);
3144 		if (group != NULL) {
3145 			mac_group_remove_client(group, mcip);
3146 			next_state = mac_group_next_state(group,
3147 			    &grp_only_mcip, default_group, B_TRUE);
3148 			if (next_state == MAC_GROUP_STATE_RESERVED) {
3149 				/*
3150 				 * Only one client left on this RX group.
3151 				 */
3152 				ASSERT(grp_only_mcip != NULL);
3153 				mac_set_group_state(group,
3154 				    MAC_GROUP_STATE_RESERVED);
3155 				group_only_flent = grp_only_mcip->mci_flent;
3156 
3157 				/*
3158 				 * The only remaining client has exclusive
3159 				 * access on the group. Allow it to
3160 				 * dynamically poll the H/W rings etc.
3161 				 */
3162 				mac_rx_srs_group_setup(grp_only_mcip,
3163 				    group_only_flent, SRST_LINK);
3164 				mac_fanout_setup(grp_only_mcip,
3165 				    group_only_flent,
3166 				    MCIP_RESOURCE_PROPS(grp_only_mcip),
3167 				    mac_rx_deliver, grp_only_mcip, NULL, NULL);
3168 				mac_rx_group_unmark(group, MR_INCIPIENT);
3169 				mac_set_rings_effective(grp_only_mcip);
3170 			} else if (next_state == MAC_GROUP_STATE_REGISTERED) {
3171 				/*
3172 				 * This is a non-default group being freed up.
3173 				 * We need to reevaluate the default group
3174 				 * to see if the primary client can get
3175 				 * exclusive access to the default group.
3176 				 */
3177 				ASSERT(group != MAC_DEFAULT_RX_GROUP(mip));
3178 				if (mrp->mrp_mask & MRP_RX_RINGS) {
3179 					MAC_RX_GRP_RELEASED(mip);
3180 					if (mip->mi_rx_group_type ==
3181 					    MAC_GROUP_TYPE_DYNAMIC) {
3182 						MAC_RX_RING_RELEASED(mip,
3183 						    group->mrg_cur_count);
3184 					}
3185 				}
3186 				mac_release_rx_group(mcip, group);
3187 				mac_set_group_state(group,
3188 				    MAC_GROUP_STATE_REGISTERED);
3189 				check_default_group = B_TRUE;
3190 			} else {
3191 				ASSERT(next_state == MAC_GROUP_STATE_SHARED);
3192 				mac_set_group_state(group,
3193 				    MAC_GROUP_STATE_SHARED);
3194 				mac_rx_group_unmark(group, MR_CONDEMNED);
3195 			}
3196 			flent->fe_rx_ring_group = NULL;
3197 		}
3198 		/*
3199 		 * Remove the client from the TX group. Additionally, if
3200 		 * this a non-default group, then we also need to release
3201 		 * the group.
3202 		 */
3203 		group = flent->fe_tx_ring_group;
3204 		default_group = MAC_DEFAULT_TX_GROUP(mip);
3205 		if (group != NULL) {
3206 			mac_group_remove_client(group, mcip);
3207 			next_state = mac_group_next_state(group,
3208 			    &grp_only_mcip, default_group, B_FALSE);
3209 			if (next_state == MAC_GROUP_STATE_REGISTERED) {
3210 				if (group != default_group) {
3211 					if (mrp->mrp_mask & MRP_TX_RINGS) {
3212 						MAC_TX_GRP_RELEASED(mip);
3213 						if (mip->mi_tx_group_type ==
3214 						    MAC_GROUP_TYPE_DYNAMIC) {
3215 							MAC_TX_RING_RELEASED(
3216 							    mip, group->
3217 							    mrg_cur_count);
3218 						}
3219 					}
3220 					mac_release_tx_group(mcip, group);
3221 					/*
3222 					 * If the default group is reserved,
3223 					 * then we need to set the effective
3224 					 * rings as we would have given
3225 					 * back some rings when the group
3226 					 * was released
3227 					 */
3228 					if (mip->mi_tx_group_type ==
3229 					    MAC_GROUP_TYPE_DYNAMIC &&
3230 					    default_group->mrg_state ==
3231 					    MAC_GROUP_STATE_RESERVED) {
3232 						grp_only_mcip =
3233 						    MAC_GROUP_ONLY_CLIENT
3234 						    (default_group);
3235 						mac_set_rings_effective(
3236 						    grp_only_mcip);
3237 					}
3238 				} else {
3239 					mac_ring_t	*ring;
3240 					int		cnt;
3241 					int		ringcnt;
3242 
3243 					/*
3244 					 * Stop all the rings except the
3245 					 * default ring.
3246 					 */
3247 					ringcnt = group->mrg_cur_count;
3248 					ring = group->mrg_rings;
3249 					for (cnt = 0; cnt < ringcnt; cnt++) {
3250 						if (ring->mr_state ==
3251 						    MR_INUSE && ring !=
3252 						    (mac_ring_t *)
3253 						    mip->mi_default_tx_ring) {
3254 							mac_stop_ring(ring);
3255 							ring->mr_flag = 0;
3256 						}
3257 						ring = ring->mr_next;
3258 					}
3259 				}
3260 			} else if (next_state == MAC_GROUP_STATE_RESERVED) {
3261 				mac_set_rings_effective(grp_only_mcip);
3262 			}
3263 			flent->fe_tx_ring_group = NULL;
3264 			group->mrg_state = next_state;
3265 		}
3266 		break;
3267 	default:
3268 		ASSERT(B_FALSE);
3269 		break;
3270 	}
3271 
3272 	/*
3273 	 * The mac client using the default group gets exclusive access to the
3274 	 * default group if and only if it is the sole client on the entire
3275 	 * mip. If so set the group state to reserved, and set up the SRSes
3276 	 * over the default group.
3277 	 */
3278 	if (check_default_group) {
3279 		default_group = MAC_DEFAULT_RX_GROUP(mip);
3280 		ASSERT(default_group->mrg_state == MAC_GROUP_STATE_SHARED);
3281 		next_state = mac_group_next_state(default_group,
3282 		    &grp_only_mcip, default_group, B_TRUE);
3283 		if (next_state == MAC_GROUP_STATE_RESERVED) {
3284 			ASSERT(grp_only_mcip != NULL &&
3285 			    mip->mi_nactiveclients == 1);
3286 			mac_set_group_state(default_group,
3287 			    MAC_GROUP_STATE_RESERVED);
3288 			mac_rx_srs_group_setup(grp_only_mcip,
3289 			    grp_only_mcip->mci_flent, SRST_LINK);
3290 			mac_fanout_setup(grp_only_mcip,
3291 			    grp_only_mcip->mci_flent,
3292 			    MCIP_RESOURCE_PROPS(grp_only_mcip), mac_rx_deliver,
3293 			    grp_only_mcip, NULL, NULL);
3294 			mac_rx_group_unmark(default_group, MR_INCIPIENT);
3295 			mac_set_rings_effective(grp_only_mcip);
3296 		}
3297 	}
3298 
3299 	/*
3300 	 * If the primary is the only one left and the MAC supports
3301 	 * dynamic grouping, we need to see if the primary needs to
3302 	 * be moved to the default group so that it can use all the
3303 	 * H/W rings.
3304 	 */
3305 	if (!(flent->fe_type & FLOW_PRIMARY_MAC) &&
3306 	    mip->mi_nactiveclients == 1 &&
3307 	    mip->mi_rx_group_type == MAC_GROUP_TYPE_DYNAMIC) {
3308 		default_group = MAC_DEFAULT_RX_GROUP(mip);
3309 		grp_only_mcip = mac_primary_client_handle(mip);
3310 		if (grp_only_mcip == NULL)
3311 			return;
3312 		group_only_flent = grp_only_mcip->mci_flent;
3313 		mrp = MCIP_RESOURCE_PROPS(grp_only_mcip);
3314 		/*
3315 		 * If the primary has an explicit property set, leave it
3316 		 * alone.
3317 		 */
3318 		if (mrp->mrp_mask & MRP_RX_RINGS)
3319 			return;
3320 		/*
3321 		 * Switch the primary to the default group.
3322 		 */
3323 		(void) mac_rx_switch_group(grp_only_mcip,
3324 		    group_only_flent->fe_rx_ring_group, default_group);
3325 	}
3326 }
3327 
3328 /* DATAPATH TEAR DOWN ROUTINES (SRS and FANOUT teardown) */
3329 
3330 static void
mac_srs_fanout_list_free(mac_soft_ring_set_t * mac_srs)3331 mac_srs_fanout_list_free(mac_soft_ring_set_t *mac_srs)
3332 {
3333 	if (mac_srs->srs_type & SRST_TX) {
3334 		mac_srs_tx_t *tx;
3335 
3336 		ASSERT(mac_srs->srs_tcp_soft_rings == NULL);
3337 		ASSERT(mac_srs->srs_udp_soft_rings == NULL);
3338 		ASSERT(mac_srs->srs_oth_soft_rings == NULL);
3339 		ASSERT(mac_srs->srs_tx_soft_rings != NULL);
3340 		kmem_free(mac_srs->srs_tx_soft_rings,
3341 		    sizeof (mac_soft_ring_t *) * MAX_RINGS_PER_GROUP);
3342 		mac_srs->srs_tx_soft_rings = NULL;
3343 		tx = &mac_srs->srs_tx;
3344 		if (tx->st_soft_rings != NULL) {
3345 			kmem_free(tx->st_soft_rings,
3346 			    sizeof (mac_soft_ring_t *) * MAX_RINGS_PER_GROUP);
3347 		}
3348 	} else {
3349 		ASSERT(mac_srs->srs_tx_soft_rings == NULL);
3350 		ASSERT(mac_srs->srs_tcp_soft_rings != NULL);
3351 		kmem_free(mac_srs->srs_tcp_soft_rings,
3352 		    sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT);
3353 		mac_srs->srs_tcp_soft_rings = NULL;
3354 		ASSERT(mac_srs->srs_udp_soft_rings != NULL);
3355 		kmem_free(mac_srs->srs_udp_soft_rings,
3356 		    sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT);
3357 		mac_srs->srs_udp_soft_rings = NULL;
3358 		ASSERT(mac_srs->srs_oth_soft_rings != NULL);
3359 		kmem_free(mac_srs->srs_oth_soft_rings,
3360 		    sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT);
3361 		mac_srs->srs_oth_soft_rings = NULL;
3362 	}
3363 }
3364 
3365 /*
3366  * An RX SRS is attached to at most one mac_ring.
3367  * A TX SRS  has no  rings.
3368  */
3369 static void
mac_srs_ring_free(mac_soft_ring_set_t * mac_srs)3370 mac_srs_ring_free(mac_soft_ring_set_t *mac_srs)
3371 {
3372 	mac_client_impl_t	*mcip;
3373 	mac_ring_t		*ring;
3374 	flow_entry_t		*flent;
3375 
3376 	ring = mac_srs->srs_ring;
3377 	if (mac_srs->srs_type & SRST_TX) {
3378 		ASSERT(ring == NULL);
3379 		return;
3380 	}
3381 
3382 	if (ring == NULL)
3383 		return;
3384 
3385 	/*
3386 	 * Broadcast flows don't have a client impl association, but they
3387 	 * use only soft rings.
3388 	 */
3389 	flent = mac_srs->srs_flent;
3390 	mcip = flent->fe_mcip;
3391 	ASSERT(mcip != NULL);
3392 
3393 	ring->mr_classify_type = MAC_NO_CLASSIFIER;
3394 	ring->mr_srs = NULL;
3395 }
3396 
3397 /*
3398  * Physical unlink and free of the data structures happen below. This is
3399  * driven from mac_flow_destroy(), on the last refrele of a flow.
3400  *
3401  * Assumes Rx srs is 1-1 mapped with an ring.
3402  */
3403 void
mac_srs_free(mac_soft_ring_set_t * mac_srs)3404 mac_srs_free(mac_soft_ring_set_t *mac_srs)
3405 {
3406 	ASSERT(mac_srs->srs_mcip == NULL ||
3407 	    MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip));
3408 	ASSERT((mac_srs->srs_state & (SRS_CONDEMNED | SRS_CONDEMNED_DONE |
3409 	    SRS_PROC | SRS_PROC_FAST)) == (SRS_CONDEMNED | SRS_CONDEMNED_DONE));
3410 
3411 	mac_pkt_drop(NULL, NULL, mac_srs->srs_first, B_FALSE);
3412 	mac_srs_ring_free(mac_srs);
3413 	mac_srs_soft_rings_free(mac_srs);
3414 	mac_srs_fanout_list_free(mac_srs);
3415 
3416 	mac_srs->srs_bw = NULL;
3417 	mac_srs_stat_delete(mac_srs);
3418 	kmem_cache_free(mac_srs_cache, mac_srs);
3419 }
3420 
3421 static void
mac_srs_soft_rings_quiesce(mac_soft_ring_set_t * mac_srs,uint_t s_ring_flag)3422 mac_srs_soft_rings_quiesce(mac_soft_ring_set_t *mac_srs, uint_t s_ring_flag)
3423 {
3424 	mac_soft_ring_t	*softring;
3425 
3426 	ASSERT(MUTEX_HELD(&mac_srs->srs_lock));
3427 
3428 	mac_srs_soft_rings_signal(mac_srs, s_ring_flag);
3429 	if (s_ring_flag == S_RING_CONDEMNED) {
3430 		while (mac_srs->srs_soft_ring_condemned_count !=
3431 		    mac_srs->srs_soft_ring_count)
3432 			cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock);
3433 	} else {
3434 		while (mac_srs->srs_soft_ring_quiesced_count !=
3435 		    mac_srs->srs_soft_ring_count)
3436 			cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock);
3437 	}
3438 	mutex_exit(&mac_srs->srs_lock);
3439 
3440 	for (softring = mac_srs->srs_soft_ring_head; softring != NULL;
3441 	    softring = softring->s_ring_next)
3442 		(void) untimeout(softring->s_ring_tid);
3443 
3444 	(void) untimeout(mac_srs->srs_tid);
3445 
3446 	mutex_enter(&mac_srs->srs_lock);
3447 }
3448 
3449 /*
3450  * The block comment above mac_rx_classify_flow_state_change explains the
3451  * background. At this point upcalls from the driver (both hardware classified
3452  * and software classified) have been cut off. We now need to quiesce the
3453  * SRS worker, poll, and softring threads. The SRS worker thread serves as
3454  * the master controller. The steps involved are described below in the function
3455  */
3456 void
mac_srs_worker_quiesce(mac_soft_ring_set_t * mac_srs)3457 mac_srs_worker_quiesce(mac_soft_ring_set_t *mac_srs)
3458 {
3459 	uint_t			s_ring_flag;
3460 	uint_t			srs_poll_wait_flag;
3461 
3462 	ASSERT(MUTEX_HELD(&mac_srs->srs_lock));
3463 	ASSERT(mac_srs->srs_state & (SRS_CONDEMNED | SRS_QUIESCE));
3464 
3465 	if (mac_srs->srs_state & SRS_CONDEMNED) {
3466 		s_ring_flag = S_RING_CONDEMNED;
3467 		srs_poll_wait_flag = SRS_POLL_THR_EXITED;
3468 	} else {
3469 		s_ring_flag = S_RING_QUIESCE;
3470 		srs_poll_wait_flag = SRS_POLL_THR_QUIESCED;
3471 	}
3472 
3473 	/*
3474 	 * In the case of Rx SRS wait till the poll thread is done.
3475 	 */
3476 	if ((mac_srs->srs_type & SRST_TX) == 0 &&
3477 	    mac_srs->srs_poll_thr != NULL) {
3478 		while (!(mac_srs->srs_state & srs_poll_wait_flag))
3479 			cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock);
3480 
3481 		/*
3482 		 * Turn off polling as part of the quiesce operation.
3483 		 */
3484 		MAC_SRS_POLLING_OFF(mac_srs);
3485 		mac_srs->srs_state &= ~(SRS_POLLING | SRS_GET_PKTS);
3486 	}
3487 
3488 	/*
3489 	 * Then signal the soft ring worker threads to quiesce or quit
3490 	 * as needed and then wait till that happens.
3491 	 */
3492 	mac_srs_soft_rings_quiesce(mac_srs, s_ring_flag);
3493 
3494 	if (mac_srs->srs_state & SRS_CONDEMNED)
3495 		mac_srs->srs_state |= (SRS_QUIESCE_DONE | SRS_CONDEMNED_DONE);
3496 	else
3497 		mac_srs->srs_state |= SRS_QUIESCE_DONE;
3498 	cv_signal(&mac_srs->srs_quiesce_done_cv);
3499 }
3500 
3501 /*
3502  * Signal an SRS to start a temporary quiesce, or permanent removal, or restart
3503  * a quiesced SRS by setting the appropriate flags and signaling the SRS worker
3504  * or poll thread. This function is internal to the quiescing logic and is
3505  * called internally from the SRS quiesce or flow quiesce or client quiesce
3506  * higher level functions.
3507  */
3508 void
mac_srs_signal(mac_soft_ring_set_t * mac_srs,uint_t srs_flag)3509 mac_srs_signal(mac_soft_ring_set_t *mac_srs, uint_t srs_flag)
3510 {
3511 	mac_ring_t	*ring;
3512 
3513 	ring = mac_srs->srs_ring;
3514 	ASSERT(ring == NULL || ring->mr_refcnt == 0);
3515 
3516 	if (srs_flag == SRS_CONDEMNED) {
3517 		/*
3518 		 * The SRS is going away. We need to unbind the SRS and SR
3519 		 * threads before removing from the global SRS list. Otherwise
3520 		 * there is a small window where the cpu reconfig callbacks
3521 		 * may miss the SRS in the list walk and DR could fail since
3522 		 * there are still bound threads.
3523 		 */
3524 		mac_srs_threads_unbind(mac_srs);
3525 		mac_srs_remove_glist(mac_srs);
3526 	}
3527 	/*
3528 	 * Wakeup the SRS worker and poll threads.
3529 	 */
3530 	mutex_enter(&mac_srs->srs_lock);
3531 	mac_srs->srs_state |= srs_flag;
3532 	cv_signal(&mac_srs->srs_async);
3533 	cv_signal(&mac_srs->srs_cv);
3534 	mutex_exit(&mac_srs->srs_lock);
3535 }
3536 
3537 /*
3538  * In the Rx side, the quiescing is done bottom up. After the Rx upcalls
3539  * from the driver are done, then the Rx SRS is quiesced and only then can
3540  * we signal the soft rings. Thus this function can't be called arbitrarily
3541  * without satisfying the prerequisites. On the Tx side, the threads from
3542  * top need to quiesced, then the Tx SRS and only then can we signal the
3543  * Tx soft rings.
3544  */
3545 static void
mac_srs_soft_rings_signal(mac_soft_ring_set_t * mac_srs,uint_t sr_flag)3546 mac_srs_soft_rings_signal(mac_soft_ring_set_t *mac_srs, uint_t sr_flag)
3547 {
3548 	mac_soft_ring_t		*softring;
3549 
3550 	for (softring = mac_srs->srs_soft_ring_head; softring != NULL;
3551 	    softring = softring->s_ring_next)
3552 		mac_soft_ring_signal(softring, sr_flag);
3553 }
3554 
3555 /*
3556  * The block comment above mac_rx_classify_flow_state_change explains the
3557  * background. At this point the SRS is quiesced and we need to restart the
3558  * SRS worker, poll, and softring threads. The SRS worker thread serves as
3559  * the master controller. The steps involved are described below in the function
3560  */
3561 void
mac_srs_worker_restart(mac_soft_ring_set_t * mac_srs)3562 mac_srs_worker_restart(mac_soft_ring_set_t *mac_srs)
3563 {
3564 	boolean_t	iam_rx_srs;
3565 	mac_soft_ring_t	*softring;
3566 
3567 	ASSERT(MUTEX_HELD(&mac_srs->srs_lock));
3568 	if ((mac_srs->srs_type & SRST_TX) != 0) {
3569 		iam_rx_srs = B_FALSE;
3570 		ASSERT((mac_srs->srs_state &
3571 		    (SRS_POLL_THR_QUIESCED | SRS_QUIESCE_DONE | SRS_QUIESCE)) ==
3572 		    (SRS_QUIESCE_DONE | SRS_QUIESCE));
3573 	} else {
3574 		iam_rx_srs = B_TRUE;
3575 		ASSERT((mac_srs->srs_state &
3576 		    (SRS_QUIESCE_DONE | SRS_QUIESCE)) ==
3577 		    (SRS_QUIESCE_DONE | SRS_QUIESCE));
3578 		if (mac_srs->srs_poll_thr != NULL) {
3579 			ASSERT((mac_srs->srs_state & SRS_POLL_THR_QUIESCED) ==
3580 			    SRS_POLL_THR_QUIESCED);
3581 		}
3582 	}
3583 
3584 	/*
3585 	 * Signal any quiesced soft ring workers to restart and wait for the
3586 	 * soft ring down count to come down to zero.
3587 	 */
3588 	if (mac_srs->srs_soft_ring_quiesced_count != 0) {
3589 		for (softring = mac_srs->srs_soft_ring_head; softring != NULL;
3590 		    softring = softring->s_ring_next) {
3591 			if (!(softring->s_ring_state & S_RING_QUIESCE))
3592 				continue;
3593 			mac_soft_ring_signal(softring, S_RING_RESTART);
3594 		}
3595 		while (mac_srs->srs_soft_ring_quiesced_count != 0)
3596 			cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock);
3597 	}
3598 
3599 	mac_srs->srs_state &= ~(SRS_QUIESCE_DONE | SRS_QUIESCE | SRS_RESTART);
3600 	if (iam_rx_srs && mac_srs->srs_poll_thr != NULL) {
3601 		/*
3602 		 * Signal the poll thread and ask it to restart. Wait till it
3603 		 * actually restarts and the SRS_POLL_THR_QUIESCED flag gets
3604 		 * cleared.
3605 		 */
3606 		mac_srs->srs_state |= SRS_POLL_THR_RESTART;
3607 		cv_signal(&mac_srs->srs_cv);
3608 		while (mac_srs->srs_state & SRS_POLL_THR_QUIESCED)
3609 			cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock);
3610 		ASSERT(!(mac_srs->srs_state & SRS_POLL_THR_RESTART));
3611 	}
3612 	/* Wake up any waiter waiting for the restart to complete */
3613 	mac_srs->srs_state |= SRS_RESTART_DONE;
3614 	cv_signal(&mac_srs->srs_quiesce_done_cv);
3615 }
3616 
3617 static void
mac_srs_worker_unbind(mac_soft_ring_set_t * mac_srs)3618 mac_srs_worker_unbind(mac_soft_ring_set_t *mac_srs)
3619 {
3620 	mutex_enter(&mac_srs->srs_lock);
3621 	if (!(mac_srs->srs_state & SRS_WORKER_BOUND)) {
3622 		ASSERT(mac_srs->srs_worker_cpuid == -1);
3623 		mutex_exit(&mac_srs->srs_lock);
3624 		return;
3625 	}
3626 
3627 	mac_srs->srs_worker_cpuid = -1;
3628 	mac_srs->srs_state &= ~SRS_WORKER_BOUND;
3629 	thread_affinity_clear(mac_srs->srs_worker);
3630 	mutex_exit(&mac_srs->srs_lock);
3631 }
3632 
3633 static void
mac_srs_poll_unbind(mac_soft_ring_set_t * mac_srs)3634 mac_srs_poll_unbind(mac_soft_ring_set_t *mac_srs)
3635 {
3636 	mutex_enter(&mac_srs->srs_lock);
3637 	if (mac_srs->srs_poll_thr == NULL ||
3638 	    (mac_srs->srs_state & SRS_POLL_BOUND) == 0) {
3639 		ASSERT(mac_srs->srs_poll_cpuid == -1);
3640 		mutex_exit(&mac_srs->srs_lock);
3641 		return;
3642 	}
3643 
3644 	mac_srs->srs_poll_cpuid = -1;
3645 	mac_srs->srs_state &= ~SRS_POLL_BOUND;
3646 	thread_affinity_clear(mac_srs->srs_poll_thr);
3647 	mutex_exit(&mac_srs->srs_lock);
3648 }
3649 
3650 static void
mac_srs_threads_unbind(mac_soft_ring_set_t * mac_srs)3651 mac_srs_threads_unbind(mac_soft_ring_set_t *mac_srs)
3652 {
3653 	mac_soft_ring_t	*soft_ring;
3654 
3655 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip));
3656 
3657 	mutex_enter(&cpu_lock);
3658 	mac_srs_worker_unbind(mac_srs);
3659 	if (!(mac_srs->srs_type & SRST_TX))
3660 		mac_srs_poll_unbind(mac_srs);
3661 
3662 	for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL;
3663 	    soft_ring = soft_ring->s_ring_next) {
3664 		mac_soft_ring_unbind(soft_ring);
3665 	}
3666 	mutex_exit(&cpu_lock);
3667 }
3668 
3669 /*
3670  * When a CPU is going away, unbind all MAC threads which are bound
3671  * to that CPU. The affinity of the thread to the CPU is saved to allow
3672  * the thread to be rebound to the CPU if it comes back online.
3673  */
3674 static void
mac_walk_srs_and_unbind(int cpuid)3675 mac_walk_srs_and_unbind(int cpuid)
3676 {
3677 	mac_soft_ring_set_t *mac_srs;
3678 	mac_soft_ring_t *soft_ring;
3679 
3680 	rw_enter(&mac_srs_g_lock, RW_READER);
3681 
3682 	if ((mac_srs = mac_srs_g_list) == NULL)
3683 		goto done;
3684 
3685 	for (; mac_srs != NULL; mac_srs = mac_srs->srs_next) {
3686 		if (mac_srs->srs_worker_cpuid == cpuid) {
3687 			mac_srs->srs_worker_cpuid_save = cpuid;
3688 			mac_srs_worker_unbind(mac_srs);
3689 		}
3690 
3691 		if (!(mac_srs->srs_type & SRST_TX)) {
3692 			if (mac_srs->srs_poll_cpuid == cpuid) {
3693 				mac_srs->srs_poll_cpuid_save = cpuid;
3694 				mac_srs_poll_unbind(mac_srs);
3695 			}
3696 		}
3697 
3698 		/* Next tackle the soft rings associated with the srs */
3699 		mutex_enter(&mac_srs->srs_lock);
3700 		for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL;
3701 		    soft_ring = soft_ring->s_ring_next) {
3702 			if (soft_ring->s_ring_cpuid == cpuid) {
3703 				soft_ring->s_ring_cpuid_save = cpuid;
3704 				mac_soft_ring_unbind(soft_ring);
3705 			}
3706 		}
3707 		mutex_exit(&mac_srs->srs_lock);
3708 	}
3709 done:
3710 	rw_exit(&mac_srs_g_lock);
3711 }
3712 
3713 /* TX SETUP and TEARDOWN ROUTINES */
3714 
3715 /*
3716  * XXXHIO need to make sure the two mac_tx_srs_{add,del}_ring()
3717  * handle the case where the number of rings is one. I.e. there is
3718  * a ring pointed to by mac_srs->srs_tx_arg2.
3719  */
3720 void
mac_tx_srs_add_ring(mac_soft_ring_set_t * mac_srs,mac_ring_t * tx_ring)3721 mac_tx_srs_add_ring(mac_soft_ring_set_t *mac_srs, mac_ring_t *tx_ring)
3722 {
3723 	mac_client_impl_t *mcip = mac_srs->srs_mcip;
3724 	mac_soft_ring_t *soft_ring;
3725 	int count = mac_srs->srs_tx_ring_count;
3726 	uint32_t soft_ring_type = ST_RING_TX;
3727 	uint_t ring_info;
3728 
3729 	ASSERT(mac_srs->srs_state & SRS_QUIESCE);
3730 	ring_info = mac_hwring_getinfo((mac_ring_handle_t)tx_ring);
3731 	if (mac_tx_serialize || (ring_info & MAC_RING_TX_SERIALIZE))
3732 		soft_ring_type |= ST_RING_WORKER_ONLY;
3733 	soft_ring = mac_soft_ring_create(count, 0,
3734 	    soft_ring_type, maxclsyspri, mcip, mac_srs, -1,
3735 	    NULL, mcip, (mac_resource_handle_t)tx_ring);
3736 	mac_srs->srs_tx_ring_count++;
3737 	mac_srs_update_fanout_list(mac_srs);
3738 	/*
3739 	 * put this soft ring in quiesce mode too so when we restart
3740 	 * all soft rings in the srs are in the same state.
3741 	 */
3742 	mac_soft_ring_signal(soft_ring, S_RING_QUIESCE);
3743 }
3744 
3745 static void
mac_soft_ring_remove(mac_soft_ring_set_t * mac_srs,mac_soft_ring_t * softring)3746 mac_soft_ring_remove(mac_soft_ring_set_t *mac_srs, mac_soft_ring_t *softring)
3747 {
3748 	int sringcnt;
3749 
3750 	mutex_enter(&mac_srs->srs_lock);
3751 	sringcnt = mac_srs->srs_soft_ring_count;
3752 	ASSERT(sringcnt > 0);
3753 	mac_soft_ring_signal(softring, S_RING_CONDEMNED);
3754 
3755 	ASSERT(mac_srs->srs_soft_ring_condemned_count == 0);
3756 	while (mac_srs->srs_soft_ring_condemned_count != 1)
3757 		cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock);
3758 
3759 	if (softring == mac_srs->srs_soft_ring_head) {
3760 		mac_srs->srs_soft_ring_head = softring->s_ring_next;
3761 		if (mac_srs->srs_soft_ring_head != NULL) {
3762 			mac_srs->srs_soft_ring_head->s_ring_prev = NULL;
3763 		} else {
3764 			mac_srs->srs_soft_ring_tail = NULL;
3765 		}
3766 	} else {
3767 		softring->s_ring_prev->s_ring_next =
3768 		    softring->s_ring_next;
3769 		if (softring->s_ring_next != NULL) {
3770 			softring->s_ring_next->s_ring_prev =
3771 			    softring->s_ring_prev;
3772 		} else {
3773 			mac_srs->srs_soft_ring_tail =
3774 			    softring->s_ring_prev;
3775 		}
3776 	}
3777 	mac_srs->srs_soft_ring_count--;
3778 
3779 	mac_srs->srs_soft_ring_condemned_count--;
3780 	mutex_exit(&mac_srs->srs_lock);
3781 
3782 	mac_soft_ring_free(softring);
3783 }
3784 
3785 void
mac_tx_srs_del_ring(mac_soft_ring_set_t * mac_srs,mac_ring_t * tx_ring)3786 mac_tx_srs_del_ring(mac_soft_ring_set_t *mac_srs, mac_ring_t *tx_ring)
3787 {
3788 	int i;
3789 	mac_soft_ring_t *soft_ring, *remove_sring;
3790 	mac_client_impl_t *mcip = mac_srs->srs_mcip;
3791 
3792 	mutex_enter(&mac_srs->srs_lock);
3793 	for (i = 0; i < mac_srs->srs_tx_ring_count; i++) {
3794 		soft_ring =  mac_srs->srs_tx_soft_rings[i];
3795 		if (soft_ring->s_ring_tx_arg2 == tx_ring)
3796 			break;
3797 	}
3798 	mutex_exit(&mac_srs->srs_lock);
3799 	ASSERT(i < mac_srs->srs_tx_ring_count);
3800 	remove_sring = soft_ring;
3801 	/*
3802 	 * In the case of aggr, the soft ring associated with a Tx ring
3803 	 * is also stored in st_soft_rings[] array. That entry should
3804 	 * be removed.
3805 	 */
3806 	if (mcip->mci_state_flags & MCIS_IS_AGGR) {
3807 		mac_srs_tx_t *tx = &mac_srs->srs_tx;
3808 
3809 		ASSERT(tx->st_soft_rings[tx_ring->mr_index] == remove_sring);
3810 		tx->st_soft_rings[tx_ring->mr_index] = NULL;
3811 	}
3812 	mac_soft_ring_remove(mac_srs, remove_sring);
3813 	mac_srs_update_fanout_list(mac_srs);
3814 }
3815 
3816 /*
3817  * mac_tx_srs_setup():
3818  * Used to setup Tx rings. If no free Tx ring is available, then default
3819  * Tx ring is used.
3820  */
3821 void
mac_tx_srs_setup(mac_client_impl_t * mcip,flow_entry_t * flent)3822 mac_tx_srs_setup(mac_client_impl_t *mcip, flow_entry_t *flent)
3823 {
3824 	mac_impl_t		*mip = mcip->mci_mip;
3825 	mac_soft_ring_set_t	*tx_srs = flent->fe_tx_srs;
3826 	int			i;
3827 	int			tx_ring_count = 0;
3828 	uint32_t		soft_ring_type;
3829 	mac_group_t		*grp = NULL;
3830 	mac_ring_t		*ring;
3831 	mac_srs_tx_t		*tx = &tx_srs->srs_tx;
3832 	boolean_t		is_aggr;
3833 	uint_t			ring_info = 0;
3834 
3835 	is_aggr = (mcip->mci_state_flags & MCIS_IS_AGGR) != 0;
3836 	grp = flent->fe_tx_ring_group;
3837 	if (grp == NULL) {
3838 		ring = (mac_ring_t *)mip->mi_default_tx_ring;
3839 		goto no_group;
3840 	}
3841 	tx_ring_count = grp->mrg_cur_count;
3842 	ring = grp->mrg_rings;
3843 	/*
3844 	 * An attempt is made to reserve 'tx_ring_count' number
3845 	 * of Tx rings. If tx_ring_count is 0, default Tx ring
3846 	 * is used. If it is 1, an attempt is made to reserve one
3847 	 * Tx ring. In both the cases, the ring information is
3848 	 * stored in Tx SRS. If multiple Tx rings are specified,
3849 	 * then each Tx ring will have a Tx-side soft ring. All
3850 	 * these soft rings will be hang off Tx SRS.
3851 	 */
3852 	switch (grp->mrg_state) {
3853 		case MAC_GROUP_STATE_SHARED:
3854 		case MAC_GROUP_STATE_RESERVED:
3855 			if (tx_ring_count <= 1 && !is_aggr) {
3856 no_group:
3857 				if (ring != NULL &&
3858 				    ring->mr_state != MR_INUSE) {
3859 					(void) mac_start_ring(ring);
3860 					ring_info = mac_hwring_getinfo(
3861 					    (mac_ring_handle_t)ring);
3862 				}
3863 				tx->st_arg2 = (void *)ring;
3864 				mac_tx_srs_stat_recreate(tx_srs, B_FALSE);
3865 				if (tx_srs->srs_type & SRST_BW_CONTROL) {
3866 					tx->st_mode = SRS_TX_BW;
3867 				} else if (mac_tx_serialize ||
3868 				    (ring_info & MAC_RING_TX_SERIALIZE)) {
3869 					tx->st_mode = SRS_TX_SERIALIZE;
3870 				} else {
3871 					tx->st_mode = SRS_TX_DEFAULT;
3872 				}
3873 				break;
3874 			}
3875 			soft_ring_type = ST_RING_TX;
3876 			if (tx_srs->srs_type & SRST_BW_CONTROL) {
3877 				tx->st_mode = is_aggr ?
3878 				    SRS_TX_BW_AGGR : SRS_TX_BW_FANOUT;
3879 			} else {
3880 				tx->st_mode = is_aggr ? SRS_TX_AGGR :
3881 				    SRS_TX_FANOUT;
3882 			}
3883 			for (i = 0; i < tx_ring_count; i++) {
3884 				ASSERT(ring != NULL);
3885 				switch (ring->mr_state) {
3886 				case MR_INUSE:
3887 				case MR_FREE:
3888 					ASSERT(ring->mr_srs == NULL);
3889 
3890 					if (ring->mr_state != MR_INUSE)
3891 						(void) mac_start_ring(ring);
3892 					ring_info = mac_hwring_getinfo(
3893 					    (mac_ring_handle_t)ring);
3894 					if (mac_tx_serialize || (ring_info &
3895 					    MAC_RING_TX_SERIALIZE)) {
3896 						soft_ring_type |=
3897 						    ST_RING_WORKER_ONLY;
3898 					}
3899 					(void) mac_soft_ring_create(i, 0,
3900 					    soft_ring_type, maxclsyspri,
3901 					    mcip, tx_srs, -1, NULL, mcip,
3902 					    (mac_resource_handle_t)ring);
3903 					break;
3904 				default:
3905 					cmn_err(CE_PANIC,
3906 					    "srs_setup: mcip = %p "
3907 					    "trying to add UNKNOWN ring = %p\n",
3908 					    (void *)mcip, (void *)ring);
3909 					break;
3910 				}
3911 				ring = ring->mr_next;
3912 			}
3913 			mac_srs_update_fanout_list(tx_srs);
3914 			break;
3915 		default:
3916 			ASSERT(B_FALSE);
3917 			break;
3918 	}
3919 	tx->st_func = mac_tx_get_func(tx->st_mode);
3920 	if (is_aggr) {
3921 		VERIFY(i_mac_capab_get((mac_handle_t)mip,
3922 		    MAC_CAPAB_AGGR, &tx->st_capab_aggr));
3923 	}
3924 	DTRACE_PROBE3(tx__srs___setup__return, mac_soft_ring_set_t *, tx_srs,
3925 	    int, tx->st_mode, int, tx_srs->srs_tx_ring_count);
3926 }
3927 
3928 /*
3929  * Update the fanout of a client if its recorded link speed doesn't match
3930  * its current link speed.
3931  */
3932 void
mac_fanout_recompute_client(mac_client_impl_t * mcip,cpupart_t * cpupart)3933 mac_fanout_recompute_client(mac_client_impl_t *mcip, cpupart_t *cpupart)
3934 {
3935 	uint64_t link_speed;
3936 	mac_resource_props_t *mcip_mrp;
3937 	flow_entry_t *flent = mcip->mci_flent;
3938 	mac_soft_ring_set_t *rx_srs;
3939 	mac_cpus_t *srs_cpu;
3940 	int soft_ring_count, maxcpus;
3941 
3942 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip));
3943 
3944 	link_speed = mac_client_stat_get(mcip->mci_flent->fe_mcip,
3945 	    MAC_STAT_IFSPEED);
3946 
3947 	if ((link_speed != 0) &&
3948 	    (link_speed != mcip->mci_flent->fe_nic_speed)) {
3949 		mcip_mrp = MCIP_RESOURCE_PROPS(mcip);
3950 		/*
3951 		 * Before calling mac_fanout_setup(), check to see if
3952 		 * the SRSes already have the right number of soft
3953 		 * rings. mac_fanout_setup() is a heavy duty operation
3954 		 * where new cpu bindings are done for SRS and soft
3955 		 * ring threads and interrupts re-targeted.
3956 		 */
3957 		maxcpus = (cpupart != NULL) ? cpupart->cp_ncpus : ncpus;
3958 		soft_ring_count = mac_compute_soft_ring_count(flent,
3959 		    flent->fe_rx_srs_cnt - 1, maxcpus);
3960 		/*
3961 		 * If soft_ring_count returned by
3962 		 * mac_compute_soft_ring_count() is 0, bump it
3963 		 * up by 1 because we always have atleast one
3964 		 * TCP, UDP, and OTH soft ring associated with
3965 		 * an SRS.
3966 		 */
3967 		soft_ring_count = (soft_ring_count == 0) ?
3968 		    1 : soft_ring_count;
3969 		rx_srs = flent->fe_rx_srs[0];
3970 		srs_cpu = &rx_srs->srs_cpu;
3971 		if (soft_ring_count != srs_cpu->mc_rx_fanout_cnt) {
3972 			mac_fanout_setup(mcip, flent, mcip_mrp,
3973 			    mac_rx_deliver, mcip, NULL, cpupart);
3974 		}
3975 	}
3976 }
3977 
3978 /*
3979  * Walk through the list of mac clients for the MAC.
3980  * For each active mac client, recompute the number of soft rings
3981  * associated with every client, only if current speed is different
3982  * from the speed that was previously used for soft ring computation.
3983  * If the cable is disconnected whlie the NIC is started, we would get
3984  * notification with speed set to 0. We do not recompute in that case.
3985  */
3986 void
mac_fanout_recompute(mac_impl_t * mip)3987 mac_fanout_recompute(mac_impl_t *mip)
3988 {
3989 	mac_client_impl_t	*mcip;
3990 	cpupart_t		*cpupart;
3991 	boolean_t		use_default;
3992 	mac_resource_props_t	*mrp, *emrp;
3993 
3994 	i_mac_perim_enter(mip);
3995 	if ((mip->mi_state_flags & MIS_IS_VNIC) != 0 ||
3996 	    mip->mi_linkstate != LINK_STATE_UP) {
3997 		i_mac_perim_exit(mip);
3998 		return;
3999 	}
4000 
4001 	for (mcip = mip->mi_clients_list; mcip != NULL;
4002 	    mcip = mcip->mci_client_next) {
4003 		if ((mcip->mci_state_flags & MCIS_SHARE_BOUND) != 0 ||
4004 		    !MCIP_DATAPATH_SETUP(mcip))
4005 			continue;
4006 		mrp = MCIP_RESOURCE_PROPS(mcip);
4007 		emrp = MCIP_EFFECTIVE_PROPS(mcip);
4008 		use_default = B_FALSE;
4009 		pool_lock();
4010 		cpupart = mac_pset_find(mrp, &use_default);
4011 		mac_fanout_recompute_client(mcip, cpupart);
4012 		mac_set_pool_effective(use_default, cpupart, mrp, emrp);
4013 		pool_unlock();
4014 	}
4015 	i_mac_perim_exit(mip);
4016 }
4017 
4018 /*
4019  * Given a MAC, change the polling state for all its MAC clients.  'enable' is
4020  * B_TRUE to enable polling or B_FALSE to disable.  Polling is enabled by
4021  * default.
4022  */
4023 void
mac_poll_state_change(mac_handle_t mh,boolean_t enable)4024 mac_poll_state_change(mac_handle_t mh, boolean_t enable)
4025 {
4026 	mac_impl_t *mip = (mac_impl_t *)mh;
4027 	mac_client_impl_t *mcip;
4028 
4029 	i_mac_perim_enter(mip);
4030 	if (enable)
4031 		mip->mi_state_flags &= ~MIS_POLL_DISABLE;
4032 	else
4033 		mip->mi_state_flags |= MIS_POLL_DISABLE;
4034 	for (mcip = mip->mi_clients_list; mcip != NULL;
4035 	    mcip = mcip->mci_client_next)
4036 		mac_client_update_classifier(mcip, B_TRUE);
4037 	i_mac_perim_exit(mip);
4038 }
4039