xref: /titanic_50/usr/src/uts/common/io/mac/mac_client.c (revision aab83bb83be7342f6cfccaed8d5fe0b2f404855d)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright (c) 2014, Joyent, Inc.  All rights reserved.
25  */
26 
27 /*
28  * - General Introduction:
29  *
30  * This file contains the implementation of the MAC client kernel
31  * API and related code. The MAC client API allows a kernel module
32  * to gain access to a MAC instance (physical NIC, link aggregation, etc).
33  * It allows a MAC client to associate itself with a MAC address,
34  * VLANs, callback functions for data traffic and for promiscuous mode.
35  * The MAC client API is also used to specify the properties associated
36  * with a MAC client, such as bandwidth limits, priority, CPUS, etc.
37  * These properties are further used to determine the hardware resources
38  * to allocate to the various MAC clients.
39  *
40  * - Primary MAC clients:
41  *
42  * The MAC client API refers to "primary MAC clients". A primary MAC
43  * client is a client which "owns" the primary MAC address of
44  * the underlying MAC instance. The primary MAC address is called out
45  * since it is associated with specific semantics: the primary MAC
46  * address is the MAC address which is assigned to the IP interface
47  * when it is plumbed, and the primary MAC address is assigned
48  * to VLAN data-links. The primary address of a MAC instance can
49  * also change dynamically from under the MAC client, for example
50  * as a result of a change of state of a link aggregation. In that
51  * case the MAC layer automatically updates all data-structures which
52  * refer to the current value of the primary MAC address. Typical
53  * primary MAC clients are dls, aggr, and xnb. A typical non-primary
54  * MAC client is the vnic driver.
55  *
56  * - Virtual Switching:
57  *
58  * The MAC layer implements a virtual switch between the MAC clients
59  * (primary and non-primary) defined on top of the same underlying
60  * NIC (physical, link aggregation, etc). The virtual switch is
61  * VLAN-aware, i.e. it allows multiple MAC clients to be member
62  * of one or more VLANs, and the virtual switch will distribute
63  * multicast tagged packets only to the member of the corresponding
64  * VLANs.
65  *
66  * - Upper vs Lower MAC:
67  *
68  * Creating a VNIC on top of a MAC instance effectively causes
69  * two MAC instances to be layered on top of each other, one for
70  * the VNIC(s), one for the underlying MAC instance (physical NIC,
71  * link aggregation, etc). In the code below we refer to the
72  * underlying NIC as the "lower MAC", and we refer to VNICs as
73  * the "upper MAC".
74  *
75  * - Pass-through for VNICs:
76  *
77  * When VNICs are created on top of an underlying MAC, this causes
78  * a layering of two MAC instances. Since the lower MAC already
79  * does the switching and demultiplexing to its MAC clients, the
80  * upper MAC would simply have to pass packets to the layer below
81  * or above it, which would introduce overhead. In order to avoid
82  * this overhead, the MAC layer implements a pass-through mechanism
83  * for VNICs. When a VNIC opens the lower MAC instance, it saves
84  * the MAC client handle it optains from the MAC layer. When a MAC
85  * client opens a VNIC (upper MAC), the MAC layer detects that
86  * the MAC being opened is a VNIC, and gets the MAC client handle
87  * that the VNIC driver obtained from the lower MAC. This exchange
88  * is done through a private capability between the MAC layer
89  * and the VNIC driver. The upper MAC then returns that handle
90  * directly to its MAC client. Any operation done by the upper
91  * MAC client is now done on the lower MAC client handle, which
92  * allows the VNIC driver to be completely bypassed for the
93  * performance sensitive data-path.
94  *
95  * - Secondary MACs for VNICs:
96  *
97  * VNICs support multiple upper mac clients to enable support for
98  * multiple MAC addresses on the VNIC. When the VNIC is created the
99  * initial mac client is the primary upper mac. Any additional mac
100  * clients are secondary macs. These are kept in sync with the primary
101  * (for things such as the rx function and resource control settings)
102  * using the same private capability interface between the MAC layer
103  * and the VNIC layer.
104  *
105  */
106 
107 #include <sys/types.h>
108 #include <sys/conf.h>
109 #include <sys/id_space.h>
110 #include <sys/esunddi.h>
111 #include <sys/stat.h>
112 #include <sys/mkdev.h>
113 #include <sys/stream.h>
114 #include <sys/strsun.h>
115 #include <sys/strsubr.h>
116 #include <sys/dlpi.h>
117 #include <sys/modhash.h>
118 #include <sys/mac_impl.h>
119 #include <sys/mac_client_impl.h>
120 #include <sys/mac_soft_ring.h>
121 #include <sys/mac_stat.h>
122 #include <sys/dls.h>
123 #include <sys/dld.h>
124 #include <sys/modctl.h>
125 #include <sys/fs/dv_node.h>
126 #include <sys/thread.h>
127 #include <sys/proc.h>
128 #include <sys/callb.h>
129 #include <sys/cpuvar.h>
130 #include <sys/atomic.h>
131 #include <sys/sdt.h>
132 #include <sys/mac_flow.h>
133 #include <sys/ddi_intr_impl.h>
134 #include <sys/disp.h>
135 #include <sys/sdt.h>
136 #include <sys/vnic.h>
137 #include <sys/vnic_impl.h>
138 #include <sys/vlan.h>
139 #include <inet/ip.h>
140 #include <inet/ip6.h>
141 #include <sys/exacct.h>
142 #include <sys/exacct_impl.h>
143 #include <inet/nd.h>
144 #include <sys/ethernet.h>
145 
146 kmem_cache_t	*mac_client_impl_cache;
147 kmem_cache_t	*mac_promisc_impl_cache;
148 
149 static boolean_t mac_client_single_rcvr(mac_client_impl_t *);
150 static flow_entry_t *mac_client_swap_mciflent(mac_client_impl_t *);
151 static flow_entry_t *mac_client_get_flow(mac_client_impl_t *,
152     mac_unicast_impl_t *);
153 static void mac_client_remove_flow_from_list(mac_client_impl_t *,
154     flow_entry_t *);
155 static void mac_client_add_to_flow_list(mac_client_impl_t *, flow_entry_t *);
156 static void mac_rename_flow_names(mac_client_impl_t *, const char *);
157 static void mac_virtual_link_update(mac_impl_t *);
158 static int mac_client_datapath_setup(mac_client_impl_t *, uint16_t,
159     uint8_t *, mac_resource_props_t *, boolean_t, mac_unicast_impl_t *);
160 static void mac_client_datapath_teardown(mac_client_handle_t,
161     mac_unicast_impl_t *, flow_entry_t *);
162 static int mac_resource_ctl_set(mac_client_handle_t, mac_resource_props_t *);
163 
164 /* ARGSUSED */
165 static int
166 i_mac_client_impl_ctor(void *buf, void *arg, int kmflag)
167 {
168 	int	i;
169 	mac_client_impl_t	*mcip = buf;
170 
171 	bzero(buf, MAC_CLIENT_IMPL_SIZE);
172 	mutex_init(&mcip->mci_tx_cb_lock, NULL, MUTEX_DRIVER, NULL);
173 	mcip->mci_tx_notify_cb_info.mcbi_lockp = &mcip->mci_tx_cb_lock;
174 
175 	ASSERT(mac_tx_percpu_cnt >= 0);
176 	for (i = 0; i <= mac_tx_percpu_cnt; i++) {
177 		mutex_init(&mcip->mci_tx_pcpu[i].pcpu_tx_lock, NULL,
178 		    MUTEX_DRIVER, NULL);
179 	}
180 	cv_init(&mcip->mci_tx_cv, NULL, CV_DRIVER, NULL);
181 
182 	return (0);
183 }
184 
185 /* ARGSUSED */
186 static void
187 i_mac_client_impl_dtor(void *buf, void *arg)
188 {
189 	int	i;
190 	mac_client_impl_t *mcip = buf;
191 
192 	ASSERT(mcip->mci_promisc_list == NULL);
193 	ASSERT(mcip->mci_unicast_list == NULL);
194 	ASSERT(mcip->mci_state_flags == 0);
195 	ASSERT(mcip->mci_tx_flag == 0);
196 
197 	mutex_destroy(&mcip->mci_tx_cb_lock);
198 
199 	ASSERT(mac_tx_percpu_cnt >= 0);
200 	for (i = 0; i <= mac_tx_percpu_cnt; i++) {
201 		ASSERT(mcip->mci_tx_pcpu[i].pcpu_tx_refcnt == 0);
202 		mutex_destroy(&mcip->mci_tx_pcpu[i].pcpu_tx_lock);
203 	}
204 	cv_destroy(&mcip->mci_tx_cv);
205 }
206 
207 /* ARGSUSED */
208 static int
209 i_mac_promisc_impl_ctor(void *buf, void *arg, int kmflag)
210 {
211 	mac_promisc_impl_t	*mpip = buf;
212 
213 	bzero(buf, sizeof (mac_promisc_impl_t));
214 	mpip->mpi_mci_link.mcb_objp = buf;
215 	mpip->mpi_mci_link.mcb_objsize = sizeof (mac_promisc_impl_t);
216 	mpip->mpi_mi_link.mcb_objp = buf;
217 	mpip->mpi_mi_link.mcb_objsize = sizeof (mac_promisc_impl_t);
218 	return (0);
219 }
220 
221 /* ARGSUSED */
222 static void
223 i_mac_promisc_impl_dtor(void *buf, void *arg)
224 {
225 	mac_promisc_impl_t	*mpip = buf;
226 
227 	ASSERT(mpip->mpi_mci_link.mcb_objp != NULL);
228 	ASSERT(mpip->mpi_mci_link.mcb_objsize == sizeof (mac_promisc_impl_t));
229 	ASSERT(mpip->mpi_mi_link.mcb_objp == mpip->mpi_mci_link.mcb_objp);
230 	ASSERT(mpip->mpi_mi_link.mcb_objsize == sizeof (mac_promisc_impl_t));
231 
232 	mpip->mpi_mci_link.mcb_objp = NULL;
233 	mpip->mpi_mci_link.mcb_objsize = 0;
234 	mpip->mpi_mi_link.mcb_objp = NULL;
235 	mpip->mpi_mi_link.mcb_objsize = 0;
236 
237 	ASSERT(mpip->mpi_mci_link.mcb_flags == 0);
238 	mpip->mpi_mci_link.mcb_objsize = 0;
239 }
240 
241 void
242 mac_client_init(void)
243 {
244 	ASSERT(mac_tx_percpu_cnt >= 0);
245 
246 	mac_client_impl_cache = kmem_cache_create("mac_client_impl_cache",
247 	    MAC_CLIENT_IMPL_SIZE, 0, i_mac_client_impl_ctor,
248 	    i_mac_client_impl_dtor, NULL, NULL, NULL, 0);
249 	ASSERT(mac_client_impl_cache != NULL);
250 
251 	mac_promisc_impl_cache = kmem_cache_create("mac_promisc_impl_cache",
252 	    sizeof (mac_promisc_impl_t), 0, i_mac_promisc_impl_ctor,
253 	    i_mac_promisc_impl_dtor, NULL, NULL, NULL, 0);
254 	ASSERT(mac_promisc_impl_cache != NULL);
255 }
256 
257 void
258 mac_client_fini(void)
259 {
260 	kmem_cache_destroy(mac_client_impl_cache);
261 	kmem_cache_destroy(mac_promisc_impl_cache);
262 }
263 
264 /*
265  * Return the lower MAC client handle from the VNIC driver for the
266  * specified VNIC MAC instance.
267  */
268 mac_client_impl_t *
269 mac_vnic_lower(mac_impl_t *mip)
270 {
271 	mac_capab_vnic_t cap;
272 	mac_client_impl_t *mcip;
273 
274 	VERIFY(i_mac_capab_get((mac_handle_t)mip, MAC_CAPAB_VNIC, &cap));
275 	mcip = cap.mcv_mac_client_handle(cap.mcv_arg);
276 
277 	return (mcip);
278 }
279 
280 /*
281  * Update the secondary macs
282  */
283 void
284 mac_vnic_secondary_update(mac_impl_t *mip)
285 {
286 	mac_capab_vnic_t cap;
287 
288 	VERIFY(i_mac_capab_get((mac_handle_t)mip, MAC_CAPAB_VNIC, &cap));
289 	cap.mcv_mac_secondary_update(cap.mcv_arg);
290 }
291 
292 /*
293  * Return the MAC client handle of the primary MAC client for the
294  * specified MAC instance, or NULL otherwise.
295  */
296 mac_client_impl_t *
297 mac_primary_client_handle(mac_impl_t *mip)
298 {
299 	mac_client_impl_t *mcip;
300 
301 	if (mip->mi_state_flags & MIS_IS_VNIC)
302 		return (mac_vnic_lower(mip));
303 
304 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mip));
305 
306 	for (mcip = mip->mi_clients_list; mcip != NULL;
307 	    mcip = mcip->mci_client_next) {
308 		if (MCIP_DATAPATH_SETUP(mcip) && mac_is_primary_client(mcip))
309 			return (mcip);
310 	}
311 	return (NULL);
312 }
313 
314 /*
315  * Open a MAC specified by its MAC name.
316  */
317 int
318 mac_open(const char *macname, mac_handle_t *mhp)
319 {
320 	mac_impl_t	*mip;
321 	int		err;
322 
323 	/*
324 	 * Look up its entry in the global hash table.
325 	 */
326 	if ((err = mac_hold(macname, &mip)) != 0)
327 		return (err);
328 
329 	/*
330 	 * Hold the dip associated to the MAC to prevent it from being
331 	 * detached. For a softmac, its underlying dip is held by the
332 	 * mi_open() callback.
333 	 *
334 	 * This is done to be more tolerant with some defective drivers,
335 	 * which incorrectly handle mac_unregister() failure in their
336 	 * xxx_detach() routine. For example, some drivers ignore the
337 	 * failure of mac_unregister() and free all resources that
338 	 * that are needed for data transmition.
339 	 */
340 	e_ddi_hold_devi(mip->mi_dip);
341 
342 	if (!(mip->mi_callbacks->mc_callbacks & MC_OPEN)) {
343 		*mhp = (mac_handle_t)mip;
344 		return (0);
345 	}
346 
347 	/*
348 	 * The mac perimeter is used in both mac_open and mac_close by the
349 	 * framework to single thread the MC_OPEN/MC_CLOSE of drivers.
350 	 */
351 	i_mac_perim_enter(mip);
352 	mip->mi_oref++;
353 	if (mip->mi_oref != 1 || ((err = mip->mi_open(mip->mi_driver)) == 0)) {
354 		*mhp = (mac_handle_t)mip;
355 		i_mac_perim_exit(mip);
356 		return (0);
357 	}
358 	mip->mi_oref--;
359 	ddi_release_devi(mip->mi_dip);
360 	mac_rele(mip);
361 	i_mac_perim_exit(mip);
362 	return (err);
363 }
364 
365 /*
366  * Open a MAC specified by its linkid.
367  */
368 int
369 mac_open_by_linkid(datalink_id_t linkid, mac_handle_t *mhp)
370 {
371 	dls_dl_handle_t	dlh;
372 	int		err;
373 
374 	if ((err = dls_devnet_hold_tmp(linkid, &dlh)) != 0)
375 		return (err);
376 
377 	dls_devnet_prop_task_wait(dlh);
378 
379 	err = mac_open(dls_devnet_mac(dlh), mhp);
380 
381 	dls_devnet_rele_tmp(dlh);
382 	return (err);
383 }
384 
385 /*
386  * Open a MAC specified by its link name.
387  */
388 int
389 mac_open_by_linkname(const char *link, mac_handle_t *mhp)
390 {
391 	datalink_id_t	linkid;
392 	int		err;
393 
394 	if ((err = dls_mgmt_get_linkid(link, &linkid)) != 0)
395 		return (err);
396 	return (mac_open_by_linkid(linkid, mhp));
397 }
398 
399 /*
400  * Close the specified MAC.
401  */
402 void
403 mac_close(mac_handle_t mh)
404 {
405 	mac_impl_t	*mip = (mac_impl_t *)mh;
406 
407 	i_mac_perim_enter(mip);
408 	/*
409 	 * The mac perimeter is used in both mac_open and mac_close by the
410 	 * framework to single thread the MC_OPEN/MC_CLOSE of drivers.
411 	 */
412 	if (mip->mi_callbacks->mc_callbacks & MC_OPEN) {
413 		ASSERT(mip->mi_oref != 0);
414 		if (--mip->mi_oref == 0) {
415 			if ((mip->mi_callbacks->mc_callbacks & MC_CLOSE))
416 				mip->mi_close(mip->mi_driver);
417 		}
418 	}
419 	i_mac_perim_exit(mip);
420 	ddi_release_devi(mip->mi_dip);
421 	mac_rele(mip);
422 }
423 
424 /*
425  * Misc utility functions to retrieve various information about a MAC
426  * instance or a MAC client.
427  */
428 
429 const mac_info_t *
430 mac_info(mac_handle_t mh)
431 {
432 	return (&((mac_impl_t *)mh)->mi_info);
433 }
434 
435 dev_info_t *
436 mac_devinfo_get(mac_handle_t mh)
437 {
438 	return (((mac_impl_t *)mh)->mi_dip);
439 }
440 
441 void *
442 mac_driver(mac_handle_t mh)
443 {
444 	return (((mac_impl_t *)mh)->mi_driver);
445 }
446 
447 const char *
448 mac_name(mac_handle_t mh)
449 {
450 	return (((mac_impl_t *)mh)->mi_name);
451 }
452 
453 int
454 mac_type(mac_handle_t mh)
455 {
456 	return (((mac_impl_t *)mh)->mi_type->mt_type);
457 }
458 
459 int
460 mac_nativetype(mac_handle_t mh)
461 {
462 	return (((mac_impl_t *)mh)->mi_type->mt_nativetype);
463 }
464 
465 char *
466 mac_client_name(mac_client_handle_t mch)
467 {
468 	return (((mac_client_impl_t *)mch)->mci_name);
469 }
470 
471 minor_t
472 mac_minor(mac_handle_t mh)
473 {
474 	return (((mac_impl_t *)mh)->mi_minor);
475 }
476 
477 /*
478  * Return the VID associated with a MAC client. This function should
479  * be called for clients which are associated with only one VID.
480  */
481 uint16_t
482 mac_client_vid(mac_client_handle_t mch)
483 {
484 	uint16_t		vid = VLAN_ID_NONE;
485 	mac_client_impl_t	*mcip = (mac_client_impl_t *)mch;
486 	flow_desc_t		flow_desc;
487 
488 	if (mcip->mci_nflents == 0)
489 		return (vid);
490 
491 	ASSERT(MCIP_DATAPATH_SETUP(mcip) && mac_client_single_rcvr(mcip));
492 
493 	mac_flow_get_desc(mcip->mci_flent, &flow_desc);
494 	if ((flow_desc.fd_mask & FLOW_LINK_VID) != 0)
495 		vid = flow_desc.fd_vid;
496 
497 	return (vid);
498 }
499 
500 /*
501  * Return whether the specified MAC client corresponds to a VLAN VNIC.
502  */
503 boolean_t
504 mac_client_is_vlan_vnic(mac_client_handle_t mch)
505 {
506 	mac_client_impl_t *mcip = (mac_client_impl_t *)mch;
507 
508 	return (((mcip->mci_state_flags & MCIS_IS_VNIC) != 0) &&
509 	    ((mcip->mci_flent->fe_type & FLOW_PRIMARY_MAC) != 0));
510 }
511 
512 /*
513  * Return the link speed associated with the specified MAC client.
514  *
515  * The link speed of a MAC client is equal to the smallest value of
516  * 1) the current link speed of the underlying NIC, or
517  * 2) the bandwidth limit set for the MAC client.
518  *
519  * Note that the bandwidth limit can be higher than the speed
520  * of the underlying NIC. This is allowed to avoid spurious
521  * administration action failures or artifically lowering the
522  * bandwidth limit of a link that may  have temporarily lowered
523  * its link speed due to hardware problem or administrator action.
524  */
525 static uint64_t
526 mac_client_ifspeed(mac_client_impl_t *mcip)
527 {
528 	mac_impl_t *mip = mcip->mci_mip;
529 	uint64_t nic_speed;
530 
531 	nic_speed = mac_stat_get((mac_handle_t)mip, MAC_STAT_IFSPEED);
532 
533 	if (nic_speed == 0) {
534 		return (0);
535 	} else {
536 		uint64_t policy_limit = (uint64_t)-1;
537 
538 		if (MCIP_RESOURCE_PROPS_MASK(mcip) & MRP_MAXBW)
539 			policy_limit = MCIP_RESOURCE_PROPS_MAXBW(mcip);
540 
541 		return (MIN(policy_limit, nic_speed));
542 	}
543 }
544 
545 /*
546  * Return the link state of the specified client. If here are more
547  * than one clients of the underying mac_impl_t, the link state
548  * will always be UP regardless of the link state of the underlying
549  * mac_impl_t. This is needed to allow the MAC clients to continue
550  * to communicate with each other even when the physical link of
551  * their mac_impl_t is down.
552  */
553 static uint64_t
554 mac_client_link_state(mac_client_impl_t *mcip)
555 {
556 	mac_impl_t *mip = mcip->mci_mip;
557 	uint16_t vid;
558 	mac_client_impl_t *mci_list;
559 	mac_unicast_impl_t *mui_list, *oth_mui_list;
560 
561 	/*
562 	 * Returns LINK_STATE_UP if there are other MAC clients defined on
563 	 * mac_impl_t which share same VLAN ID as that of mcip. Note that
564 	 * if 'mcip' has more than one VID's then we match ANY one of the
565 	 * VID's with other MAC client's VID's and return LINK_STATE_UP.
566 	 */
567 	rw_enter(&mcip->mci_rw_lock, RW_READER);
568 	for (mui_list = mcip->mci_unicast_list; mui_list != NULL;
569 	    mui_list = mui_list->mui_next) {
570 		vid = mui_list->mui_vid;
571 		for (mci_list = mip->mi_clients_list; mci_list != NULL;
572 		    mci_list = mci_list->mci_client_next) {
573 			if (mci_list == mcip)
574 				continue;
575 			for (oth_mui_list = mci_list->mci_unicast_list;
576 			    oth_mui_list != NULL; oth_mui_list = oth_mui_list->
577 			    mui_next) {
578 				if (vid == oth_mui_list->mui_vid) {
579 					rw_exit(&mcip->mci_rw_lock);
580 					return (LINK_STATE_UP);
581 				}
582 			}
583 		}
584 	}
585 	rw_exit(&mcip->mci_rw_lock);
586 
587 	return (mac_stat_get((mac_handle_t)mip, MAC_STAT_LINK_STATE));
588 }
589 
590 /*
591  * These statistics are consumed by dladm show-link -s <vnic>,
592  * dladm show-vnic -s and netstat. With the introduction of dlstat,
593  * dladm show-link -s and dladm show-vnic -s witll be EOL'ed while
594  * netstat will consume from kstats introduced for dlstat. This code
595  * will be removed at that time.
596  */
597 
598 /*
599  * Return the statistics of a MAC client. These statistics are different
600  * then the statistics of the underlying MAC which are returned by
601  * mac_stat_get().
602  *
603  * Note that for things based on the tx and rx stats, mac will end up clobbering
604  * those stats when the underlying set of rings in the srs changes. As such, we
605  * need to source not only the current set, but also the historical set when
606  * returning to the client, lest our counters appear to go backwards.
607  */
608 uint64_t
609 mac_client_stat_get(mac_client_handle_t mch, uint_t stat)
610 {
611 	mac_client_impl_t 	*mcip = (mac_client_impl_t *)mch;
612 	mac_impl_t 		*mip = mcip->mci_mip;
613 	flow_entry_t 		*flent = mcip->mci_flent;
614 	mac_soft_ring_set_t 	*mac_srs;
615 	mac_rx_stats_t		*mac_rx_stat, *old_rx_stat;
616 	mac_tx_stats_t		*mac_tx_stat, *old_tx_stat;
617 	int i;
618 	uint64_t val = 0;
619 
620 	mac_srs = (mac_soft_ring_set_t *)(flent->fe_tx_srs);
621 	mac_tx_stat = &mac_srs->srs_tx.st_stat;
622 	old_rx_stat = &mcip->mci_misc_stat.mms_defunctrxlanestats;
623 	old_tx_stat = &mcip->mci_misc_stat.mms_defuncttxlanestats;
624 
625 	switch (stat) {
626 	case MAC_STAT_LINK_STATE:
627 		val = mac_client_link_state(mcip);
628 		break;
629 	case MAC_STAT_LINK_UP:
630 		val = (mac_client_link_state(mcip) == LINK_STATE_UP);
631 		break;
632 	case MAC_STAT_PROMISC:
633 		val = mac_stat_get((mac_handle_t)mip, MAC_STAT_PROMISC);
634 		break;
635 	case MAC_STAT_LOWLINK_STATE:
636 		val = mac_stat_get((mac_handle_t)mip, MAC_STAT_LOWLINK_STATE);
637 		break;
638 	case MAC_STAT_IFSPEED:
639 		val = mac_client_ifspeed(mcip);
640 		break;
641 	case MAC_STAT_MULTIRCV:
642 		val = mcip->mci_misc_stat.mms_multircv;
643 		break;
644 	case MAC_STAT_BRDCSTRCV:
645 		val = mcip->mci_misc_stat.mms_brdcstrcv;
646 		break;
647 	case MAC_STAT_MULTIXMT:
648 		val = mcip->mci_misc_stat.mms_multixmt;
649 		break;
650 	case MAC_STAT_BRDCSTXMT:
651 		val = mcip->mci_misc_stat.mms_brdcstxmt;
652 		break;
653 	case MAC_STAT_OBYTES:
654 		val = mac_tx_stat->mts_obytes;
655 		val += old_tx_stat->mts_obytes;
656 		break;
657 	case MAC_STAT_OPACKETS:
658 		val = mac_tx_stat->mts_opackets;
659 		val += old_tx_stat->mts_opackets;
660 		break;
661 	case MAC_STAT_OERRORS:
662 		val = mac_tx_stat->mts_oerrors;
663 		val += old_tx_stat->mts_oerrors;
664 		break;
665 	case MAC_STAT_IPACKETS:
666 		for (i = 0; i < flent->fe_rx_srs_cnt; i++) {
667 			mac_srs = (mac_soft_ring_set_t *)flent->fe_rx_srs[i];
668 			mac_rx_stat = &mac_srs->srs_rx.sr_stat;
669 			val += mac_rx_stat->mrs_intrcnt +
670 			    mac_rx_stat->mrs_pollcnt + mac_rx_stat->mrs_lclcnt;
671 		}
672 		val += old_rx_stat->mrs_intrcnt + old_rx_stat->mrs_pollcnt +
673 		    old_rx_stat->mrs_lclcnt;
674 		break;
675 	case MAC_STAT_RBYTES:
676 		for (i = 0; i < flent->fe_rx_srs_cnt; i++) {
677 			mac_srs = (mac_soft_ring_set_t *)flent->fe_rx_srs[i];
678 			mac_rx_stat = &mac_srs->srs_rx.sr_stat;
679 			val += mac_rx_stat->mrs_intrbytes +
680 			    mac_rx_stat->mrs_pollbytes +
681 			    mac_rx_stat->mrs_lclbytes;
682 		}
683 		val += old_rx_stat->mrs_intrbytes + old_rx_stat->mrs_pollbytes +
684 		    old_rx_stat->mrs_lclbytes;
685 		break;
686 	case MAC_STAT_IERRORS:
687 		for (i = 0; i < flent->fe_rx_srs_cnt; i++) {
688 			mac_srs = (mac_soft_ring_set_t *)flent->fe_rx_srs[i];
689 			mac_rx_stat = &mac_srs->srs_rx.sr_stat;
690 			val += mac_rx_stat->mrs_ierrors;
691 		}
692 		val += old_rx_stat->mrs_ierrors;
693 		break;
694 	default:
695 		val = mac_driver_stat_default(mip, stat);
696 		break;
697 	}
698 
699 	return (val);
700 }
701 
702 /*
703  * Return the statistics of the specified MAC instance.
704  */
705 uint64_t
706 mac_stat_get(mac_handle_t mh, uint_t stat)
707 {
708 	mac_impl_t	*mip = (mac_impl_t *)mh;
709 	uint64_t	val;
710 	int		ret;
711 
712 	/*
713 	 * The range of stat determines where it is maintained.  Stat
714 	 * values from 0 up to (but not including) MAC_STAT_MIN are
715 	 * mainteined by the mac module itself.  Everything else is
716 	 * maintained by the driver.
717 	 *
718 	 * If the mac_impl_t being queried corresponds to a VNIC,
719 	 * the stats need to be queried from the lower MAC client
720 	 * corresponding to the VNIC. (The mac_link_update()
721 	 * invoked by the driver to the lower MAC causes the *lower
722 	 * MAC* to update its mi_linkstate, and send a notification
723 	 * to its MAC clients. Due to the VNIC passthrough,
724 	 * these notifications are sent to the upper MAC clients
725 	 * of the VNIC directly, and the upper mac_impl_t of the VNIC
726 	 * does not have a valid mi_linkstate.
727 	 */
728 	if (stat < MAC_STAT_MIN && !(mip->mi_state_flags & MIS_IS_VNIC)) {
729 		/* these stats are maintained by the mac module itself */
730 		switch (stat) {
731 		case MAC_STAT_LINK_STATE:
732 			return (mip->mi_linkstate);
733 		case MAC_STAT_LINK_UP:
734 			return (mip->mi_linkstate == LINK_STATE_UP);
735 		case MAC_STAT_PROMISC:
736 			return (mip->mi_devpromisc != 0);
737 		case MAC_STAT_LOWLINK_STATE:
738 			return (mip->mi_lowlinkstate);
739 		default:
740 			ASSERT(B_FALSE);
741 		}
742 	}
743 
744 	/*
745 	 * Call the driver to get the given statistic.
746 	 */
747 	ret = mip->mi_getstat(mip->mi_driver, stat, &val);
748 	if (ret != 0) {
749 		/*
750 		 * The driver doesn't support this statistic.  Get the
751 		 * statistic's default value.
752 		 */
753 		val = mac_driver_stat_default(mip, stat);
754 	}
755 	return (val);
756 }
757 
758 /*
759  * Query hardware rx ring corresponding to the pseudo ring.
760  */
761 uint64_t
762 mac_pseudo_rx_ring_stat_get(mac_ring_handle_t handle, uint_t stat)
763 {
764 	return (mac_rx_ring_stat_get(handle, stat));
765 }
766 
767 /*
768  * Query hardware tx ring corresponding to the pseudo ring.
769  */
770 uint64_t
771 mac_pseudo_tx_ring_stat_get(mac_ring_handle_t handle, uint_t stat)
772 {
773 	return (mac_tx_ring_stat_get(handle, stat));
774 }
775 
776 /*
777  * Utility function which returns the VID associated with a flow entry.
778  */
779 uint16_t
780 i_mac_flow_vid(flow_entry_t *flent)
781 {
782 	flow_desc_t	flow_desc;
783 
784 	mac_flow_get_desc(flent, &flow_desc);
785 
786 	if ((flow_desc.fd_mask & FLOW_LINK_VID) != 0)
787 		return (flow_desc.fd_vid);
788 	return (VLAN_ID_NONE);
789 }
790 
791 /*
792  * Verify the validity of the specified unicast MAC address. Returns B_TRUE
793  * if the address is valid, B_FALSE otherwise (multicast address, or incorrect
794  * length.
795  */
796 boolean_t
797 mac_unicst_verify(mac_handle_t mh, const uint8_t *addr, uint_t len)
798 {
799 	mac_impl_t	*mip = (mac_impl_t *)mh;
800 
801 	/*
802 	 * Verify the address. No lock is needed since mi_type and plugin
803 	 * details don't change after mac_register().
804 	 */
805 	if ((len != mip->mi_type->mt_addr_length) ||
806 	    (mip->mi_type->mt_ops.mtops_unicst_verify(addr,
807 	    mip->mi_pdata)) != 0) {
808 		return (B_FALSE);
809 	} else {
810 		return (B_TRUE);
811 	}
812 }
813 
814 void
815 mac_sdu_get(mac_handle_t mh, uint_t *min_sdu, uint_t *max_sdu)
816 {
817 	mac_impl_t	*mip = (mac_impl_t *)mh;
818 
819 	if (min_sdu != NULL)
820 		*min_sdu = mip->mi_sdu_min;
821 	if (max_sdu != NULL)
822 		*max_sdu = mip->mi_sdu_max;
823 }
824 
825 void
826 mac_sdu_get2(mac_handle_t mh, uint_t *min_sdu, uint_t *max_sdu,
827     uint_t *multicast_sdu)
828 {
829 	mac_impl_t	*mip = (mac_impl_t *)mh;
830 
831 	if (min_sdu != NULL)
832 		*min_sdu = mip->mi_sdu_min;
833 	if (max_sdu != NULL)
834 		*max_sdu = mip->mi_sdu_max;
835 	if (multicast_sdu != NULL)
836 		*multicast_sdu = mip->mi_sdu_multicast;
837 }
838 
839 /*
840  * Update the MAC unicast address of the specified client's flows. Currently
841  * only one unicast MAC unicast address is allowed per client.
842  */
843 static void
844 mac_unicast_update_client_flow(mac_client_impl_t *mcip)
845 {
846 	mac_impl_t *mip = mcip->mci_mip;
847 	flow_entry_t *flent = mcip->mci_flent;
848 	mac_address_t *map = mcip->mci_unicast;
849 	flow_desc_t flow_desc;
850 
851 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mip));
852 	ASSERT(flent != NULL);
853 
854 	mac_flow_get_desc(flent, &flow_desc);
855 	ASSERT(flow_desc.fd_mask & FLOW_LINK_DST);
856 
857 	bcopy(map->ma_addr, flow_desc.fd_dst_mac, map->ma_len);
858 	mac_flow_set_desc(flent, &flow_desc);
859 
860 	/*
861 	 * The v6 local addr (used by mac protection) needs to be
862 	 * regenerated because our mac address has changed.
863 	 */
864 	mac_protect_update_v6_local_addr(mcip);
865 
866 	/*
867 	 * A MAC client could have one MAC address but multiple
868 	 * VLANs. In that case update the flow entries corresponding
869 	 * to all VLANs of the MAC client.
870 	 */
871 	for (flent = mcip->mci_flent_list; flent != NULL;
872 	    flent = flent->fe_client_next) {
873 		mac_flow_get_desc(flent, &flow_desc);
874 		if (!(flent->fe_type & FLOW_PRIMARY_MAC ||
875 		    flent->fe_type & FLOW_VNIC_MAC))
876 			continue;
877 
878 		bcopy(map->ma_addr, flow_desc.fd_dst_mac, map->ma_len);
879 		mac_flow_set_desc(flent, &flow_desc);
880 	}
881 }
882 
883 /*
884  * Update all clients that share the same unicast address.
885  */
886 void
887 mac_unicast_update_clients(mac_impl_t *mip, mac_address_t *map)
888 {
889 	mac_client_impl_t *mcip;
890 
891 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mip));
892 
893 	/*
894 	 * Find all clients that share the same unicast MAC address and update
895 	 * them appropriately.
896 	 */
897 	for (mcip = mip->mi_clients_list; mcip != NULL;
898 	    mcip = mcip->mci_client_next) {
899 		/*
900 		 * Ignore clients that don't share this MAC address.
901 		 */
902 		if (map != mcip->mci_unicast)
903 			continue;
904 
905 		/*
906 		 * Update those clients with same old unicast MAC address.
907 		 */
908 		mac_unicast_update_client_flow(mcip);
909 	}
910 }
911 
912 /*
913  * Update the unicast MAC address of the specified VNIC MAC client.
914  *
915  * Check whether the operation is valid. Any of following cases should fail:
916  *
917  * 1. It's a VLAN type of VNIC.
918  * 2. The new value is current "primary" MAC address.
919  * 3. The current MAC address is shared with other clients.
920  * 4. The new MAC address has been used. This case will be valid when
921  *    client migration is fully supported.
922  */
923 int
924 mac_vnic_unicast_set(mac_client_handle_t mch, const uint8_t *addr)
925 {
926 	mac_client_impl_t *mcip = (mac_client_impl_t *)mch;
927 	mac_impl_t *mip = mcip->mci_mip;
928 	mac_address_t *map = mcip->mci_unicast;
929 	int err;
930 
931 	ASSERT(!(mip->mi_state_flags & MIS_IS_VNIC));
932 	ASSERT(mcip->mci_state_flags & MCIS_IS_VNIC);
933 	ASSERT(mcip->mci_flags != MAC_CLIENT_FLAGS_PRIMARY);
934 
935 	i_mac_perim_enter(mip);
936 
937 	/*
938 	 * If this is a VLAN type of VNIC, it's using "primary" MAC address
939 	 * of the underlying interface. Must fail here. Refer to case 1 above.
940 	 */
941 	if (bcmp(map->ma_addr, mip->mi_addr, map->ma_len) == 0) {
942 		i_mac_perim_exit(mip);
943 		return (ENOTSUP);
944 	}
945 
946 	/*
947 	 * If the new address is the "primary" one, must fail. Refer to
948 	 * case 2 above.
949 	 */
950 	if (bcmp(addr, mip->mi_addr, map->ma_len) == 0) {
951 		i_mac_perim_exit(mip);
952 		return (EACCES);
953 	}
954 
955 	/*
956 	 * If the address is shared by multiple clients, must fail. Refer
957 	 * to case 3 above.
958 	 */
959 	if (mac_check_macaddr_shared(map)) {
960 		i_mac_perim_exit(mip);
961 		return (EBUSY);
962 	}
963 
964 	/*
965 	 * If the new address has been used, must fail for now. Refer to
966 	 * case 4 above.
967 	 */
968 	if (mac_find_macaddr(mip, (uint8_t *)addr) != NULL) {
969 		i_mac_perim_exit(mip);
970 		return (ENOTSUP);
971 	}
972 
973 	/*
974 	 * Update the MAC address.
975 	 */
976 	err = mac_update_macaddr(map, (uint8_t *)addr);
977 
978 	if (err != 0) {
979 		i_mac_perim_exit(mip);
980 		return (err);
981 	}
982 
983 	/*
984 	 * Update all flows of this MAC client.
985 	 */
986 	mac_unicast_update_client_flow(mcip);
987 
988 	i_mac_perim_exit(mip);
989 	return (0);
990 }
991 
992 /*
993  * Program the new primary unicast address of the specified MAC.
994  *
995  * Function mac_update_macaddr() takes care different types of underlying
996  * MAC. If the underlying MAC is VNIC, the VNIC driver must have registerd
997  * mi_unicst() entry point, that indirectly calls mac_vnic_unicast_set()
998  * which will take care of updating the MAC address of the corresponding
999  * MAC client.
1000  *
1001  * This is the only interface that allow the client to update the "primary"
1002  * MAC address of the underlying MAC. The new value must have not been
1003  * used by other clients.
1004  */
1005 int
1006 mac_unicast_primary_set(mac_handle_t mh, const uint8_t *addr)
1007 {
1008 	mac_impl_t *mip = (mac_impl_t *)mh;
1009 	mac_address_t *map;
1010 	int err;
1011 
1012 	/* verify the address validity */
1013 	if (!mac_unicst_verify(mh, addr, mip->mi_type->mt_addr_length))
1014 		return (EINVAL);
1015 
1016 	i_mac_perim_enter(mip);
1017 
1018 	/*
1019 	 * If the new value is the same as the current primary address value,
1020 	 * there's nothing to do.
1021 	 */
1022 	if (bcmp(addr, mip->mi_addr, mip->mi_type->mt_addr_length) == 0) {
1023 		i_mac_perim_exit(mip);
1024 		return (0);
1025 	}
1026 
1027 	if (mac_find_macaddr(mip, (uint8_t *)addr) != 0) {
1028 		i_mac_perim_exit(mip);
1029 		return (EBUSY);
1030 	}
1031 
1032 	map = mac_find_macaddr(mip, mip->mi_addr);
1033 	ASSERT(map != NULL);
1034 
1035 	/*
1036 	 * Update the MAC address.
1037 	 */
1038 	if (mip->mi_state_flags & MIS_IS_AGGR) {
1039 		mac_capab_aggr_t aggr_cap;
1040 
1041 		/*
1042 		 * If the mac is an aggregation, other than the unicast
1043 		 * addresses programming, aggr must be informed about this
1044 		 * primary unicst address change to change its mac address
1045 		 * policy to be user-specified.
1046 		 */
1047 		ASSERT(map->ma_type == MAC_ADDRESS_TYPE_UNICAST_CLASSIFIED);
1048 		VERIFY(i_mac_capab_get(mh, MAC_CAPAB_AGGR, &aggr_cap));
1049 		err = aggr_cap.mca_unicst(mip->mi_driver, addr);
1050 		if (err == 0)
1051 			bcopy(addr, map->ma_addr, map->ma_len);
1052 	} else {
1053 		err = mac_update_macaddr(map, (uint8_t *)addr);
1054 	}
1055 
1056 	if (err != 0) {
1057 		i_mac_perim_exit(mip);
1058 		return (err);
1059 	}
1060 
1061 	mac_unicast_update_clients(mip, map);
1062 
1063 	/*
1064 	 * Save the new primary MAC address in mac_impl_t.
1065 	 */
1066 	bcopy(addr, mip->mi_addr, mip->mi_type->mt_addr_length);
1067 
1068 	i_mac_perim_exit(mip);
1069 
1070 	if (err == 0)
1071 		i_mac_notify(mip, MAC_NOTE_UNICST);
1072 
1073 	return (err);
1074 }
1075 
1076 /*
1077  * Return the current primary MAC address of the specified MAC.
1078  */
1079 void
1080 mac_unicast_primary_get(mac_handle_t mh, uint8_t *addr)
1081 {
1082 	mac_impl_t *mip = (mac_impl_t *)mh;
1083 
1084 	rw_enter(&mip->mi_rw_lock, RW_READER);
1085 	bcopy(mip->mi_addr, addr, mip->mi_type->mt_addr_length);
1086 	rw_exit(&mip->mi_rw_lock);
1087 }
1088 
1089 /*
1090  * Return the secondary MAC address for the specified handle
1091  */
1092 void
1093 mac_unicast_secondary_get(mac_client_handle_t mh, uint8_t *addr)
1094 {
1095 	mac_client_impl_t *mcip = (mac_client_impl_t *)mh;
1096 
1097 	ASSERT(mcip->mci_unicast != NULL);
1098 	bcopy(mcip->mci_unicast->ma_addr, addr, mcip->mci_unicast->ma_len);
1099 }
1100 
1101 /*
1102  * Return information about the use of the primary MAC address of the
1103  * specified MAC instance:
1104  *
1105  * - if client_name is non-NULL, it must point to a string of at
1106  *   least MAXNAMELEN bytes, and will be set to the name of the MAC
1107  *   client which uses the primary MAC address.
1108  *
1109  * - if in_use is non-NULL, used to return whether the primary MAC
1110  *   address is currently in use.
1111  */
1112 void
1113 mac_unicast_primary_info(mac_handle_t mh, char *client_name, boolean_t *in_use)
1114 {
1115 	mac_impl_t *mip = (mac_impl_t *)mh;
1116 	mac_client_impl_t *cur_client;
1117 
1118 	if (in_use != NULL)
1119 		*in_use = B_FALSE;
1120 	if (client_name != NULL)
1121 		bzero(client_name, MAXNAMELEN);
1122 
1123 	/*
1124 	 * The mi_rw_lock is used to protect threads that don't hold the
1125 	 * mac perimeter to get a consistent view of the mi_clients_list.
1126 	 * Threads that modify the list must hold both the mac perimeter and
1127 	 * mi_rw_lock(RW_WRITER)
1128 	 */
1129 	rw_enter(&mip->mi_rw_lock, RW_READER);
1130 	for (cur_client = mip->mi_clients_list; cur_client != NULL;
1131 	    cur_client = cur_client->mci_client_next) {
1132 		if (mac_is_primary_client(cur_client) ||
1133 		    (mip->mi_state_flags & MIS_IS_VNIC)) {
1134 			rw_exit(&mip->mi_rw_lock);
1135 			if (in_use != NULL)
1136 				*in_use = B_TRUE;
1137 			if (client_name != NULL) {
1138 				bcopy(cur_client->mci_name, client_name,
1139 				    MAXNAMELEN);
1140 			}
1141 			return;
1142 		}
1143 	}
1144 	rw_exit(&mip->mi_rw_lock);
1145 }
1146 
1147 /*
1148  * Return the current destination MAC address of the specified MAC.
1149  */
1150 boolean_t
1151 mac_dst_get(mac_handle_t mh, uint8_t *addr)
1152 {
1153 	mac_impl_t *mip = (mac_impl_t *)mh;
1154 
1155 	rw_enter(&mip->mi_rw_lock, RW_READER);
1156 	if (mip->mi_dstaddr_set)
1157 		bcopy(mip->mi_dstaddr, addr, mip->mi_type->mt_addr_length);
1158 	rw_exit(&mip->mi_rw_lock);
1159 	return (mip->mi_dstaddr_set);
1160 }
1161 
1162 /*
1163  * Add the specified MAC client to the list of clients which opened
1164  * the specified MAC.
1165  */
1166 static void
1167 mac_client_add(mac_client_impl_t *mcip)
1168 {
1169 	mac_impl_t *mip = mcip->mci_mip;
1170 
1171 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mip));
1172 
1173 	/* add VNIC to the front of the list */
1174 	rw_enter(&mip->mi_rw_lock, RW_WRITER);
1175 	mcip->mci_client_next = mip->mi_clients_list;
1176 	mip->mi_clients_list = mcip;
1177 	mip->mi_nclients++;
1178 	rw_exit(&mip->mi_rw_lock);
1179 }
1180 
1181 /*
1182  * Remove the specified MAC client from the list of clients which opened
1183  * the specified MAC.
1184  */
1185 static void
1186 mac_client_remove(mac_client_impl_t *mcip)
1187 {
1188 	mac_impl_t *mip = mcip->mci_mip;
1189 	mac_client_impl_t **prev, *cclient;
1190 
1191 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mip));
1192 
1193 	rw_enter(&mip->mi_rw_lock, RW_WRITER);
1194 	prev = &mip->mi_clients_list;
1195 	cclient = *prev;
1196 	while (cclient != NULL && cclient != mcip) {
1197 		prev = &cclient->mci_client_next;
1198 		cclient = *prev;
1199 	}
1200 	ASSERT(cclient != NULL);
1201 	*prev = cclient->mci_client_next;
1202 	mip->mi_nclients--;
1203 	rw_exit(&mip->mi_rw_lock);
1204 }
1205 
1206 static mac_unicast_impl_t *
1207 mac_client_find_vid(mac_client_impl_t *mcip, uint16_t vid)
1208 {
1209 	mac_unicast_impl_t *muip = mcip->mci_unicast_list;
1210 
1211 	while ((muip != NULL) && (muip->mui_vid != vid))
1212 		muip = muip->mui_next;
1213 
1214 	return (muip);
1215 }
1216 
1217 /*
1218  * Return whether the specified (MAC address, VID) tuple is already used by
1219  * one of the MAC clients associated with the specified MAC.
1220  */
1221 static boolean_t
1222 mac_addr_in_use(mac_impl_t *mip, uint8_t *mac_addr, uint16_t vid)
1223 {
1224 	mac_client_impl_t *client;
1225 	mac_address_t *map;
1226 
1227 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mip));
1228 
1229 	for (client = mip->mi_clients_list; client != NULL;
1230 	    client = client->mci_client_next) {
1231 
1232 		/*
1233 		 * Ignore clients that don't have unicast address.
1234 		 */
1235 		if (client->mci_unicast_list == NULL)
1236 			continue;
1237 
1238 		map = client->mci_unicast;
1239 
1240 		if ((bcmp(mac_addr, map->ma_addr, map->ma_len) == 0) &&
1241 		    (mac_client_find_vid(client, vid) != NULL)) {
1242 			return (B_TRUE);
1243 		}
1244 	}
1245 
1246 	return (B_FALSE);
1247 }
1248 
1249 /*
1250  * Generate a random MAC address. The MAC address prefix is
1251  * stored in the array pointed to by mac_addr, and its length, in bytes,
1252  * is specified by prefix_len. The least significant bits
1253  * after prefix_len bytes are generated, and stored after the prefix
1254  * in the mac_addr array.
1255  */
1256 int
1257 mac_addr_random(mac_client_handle_t mch, uint_t prefix_len,
1258     uint8_t *mac_addr, mac_diag_t *diag)
1259 {
1260 	mac_client_impl_t *mcip = (mac_client_impl_t *)mch;
1261 	mac_impl_t *mip = mcip->mci_mip;
1262 	size_t addr_len = mip->mi_type->mt_addr_length;
1263 
1264 	if (prefix_len >= addr_len) {
1265 		*diag = MAC_DIAG_MACPREFIXLEN_INVALID;
1266 		return (EINVAL);
1267 	}
1268 
1269 	/* check the prefix value */
1270 	if (prefix_len > 0) {
1271 		bzero(mac_addr + prefix_len, addr_len - prefix_len);
1272 		if (!mac_unicst_verify((mac_handle_t)mip, mac_addr,
1273 		    addr_len)) {
1274 			*diag = MAC_DIAG_MACPREFIX_INVALID;
1275 			return (EINVAL);
1276 		}
1277 	}
1278 
1279 	/* generate the MAC address */
1280 	if (prefix_len < addr_len) {
1281 		(void) random_get_pseudo_bytes(mac_addr +
1282 		    prefix_len, addr_len - prefix_len);
1283 	}
1284 
1285 	*diag = 0;
1286 	return (0);
1287 }
1288 
1289 /*
1290  * Set the priority range for this MAC client. This will be used to
1291  * determine the absolute priority for the threads created for this
1292  * MAC client using the specified "low", "medium" and "high" level.
1293  * This will also be used for any subflows on this MAC client.
1294  */
1295 #define	MAC_CLIENT_SET_PRIORITY_RANGE(mcip, pri) {			\
1296 	(mcip)->mci_min_pri = FLOW_MIN_PRIORITY(MINCLSYSPRI,	\
1297 	    MAXCLSYSPRI, (pri));					\
1298 	(mcip)->mci_max_pri = FLOW_MAX_PRIORITY(MINCLSYSPRI,	\
1299 	    MAXCLSYSPRI, (mcip)->mci_min_pri);				\
1300 	}
1301 
1302 /*
1303  * MAC client open entry point. Return a new MAC client handle. Each
1304  * MAC client is associated with a name, specified through the 'name'
1305  * argument.
1306  */
1307 int
1308 mac_client_open(mac_handle_t mh, mac_client_handle_t *mchp, char *name,
1309     uint16_t flags)
1310 {
1311 	mac_impl_t		*mip = (mac_impl_t *)mh;
1312 	mac_client_impl_t	*mcip;
1313 	int			err = 0;
1314 	boolean_t		share_desired;
1315 	flow_entry_t		*flent = NULL;
1316 
1317 	share_desired = (flags & MAC_OPEN_FLAGS_SHARES_DESIRED) != 0;
1318 	*mchp = NULL;
1319 
1320 	i_mac_perim_enter(mip);
1321 
1322 	if (mip->mi_state_flags & MIS_IS_VNIC) {
1323 		/*
1324 		 * The underlying MAC is a VNIC. Return the MAC client
1325 		 * handle of the lower MAC which was obtained by
1326 		 * the VNIC driver when it did its mac_client_open().
1327 		 */
1328 
1329 		mcip = mac_vnic_lower(mip);
1330 
1331 		/*
1332 		 * Note that multiple mac clients share the same mcip in
1333 		 * this case.
1334 		 */
1335 		if (flags & MAC_OPEN_FLAGS_EXCLUSIVE)
1336 			mcip->mci_state_flags |= MCIS_EXCLUSIVE;
1337 
1338 		if (flags & MAC_OPEN_FLAGS_MULTI_PRIMARY)
1339 			mcip->mci_flags |= MAC_CLIENT_FLAGS_MULTI_PRIMARY;
1340 
1341 		mip->mi_clients_list = mcip;
1342 		i_mac_perim_exit(mip);
1343 		*mchp = (mac_client_handle_t)mcip;
1344 
1345 		DTRACE_PROBE2(mac__client__open__nonallocated, mac_impl_t *,
1346 		    mcip->mci_mip, mac_client_impl_t *, mcip);
1347 
1348 		return (err);
1349 	}
1350 
1351 	mcip = kmem_cache_alloc(mac_client_impl_cache, KM_SLEEP);
1352 
1353 	mcip->mci_mip = mip;
1354 	mcip->mci_upper_mip = NULL;
1355 	mcip->mci_rx_fn = mac_pkt_drop;
1356 	mcip->mci_rx_arg = NULL;
1357 	mcip->mci_rx_p_fn = NULL;
1358 	mcip->mci_rx_p_arg = NULL;
1359 	mcip->mci_p_unicast_list = NULL;
1360 	mcip->mci_direct_rx_fn = NULL;
1361 	mcip->mci_direct_rx_arg = NULL;
1362 	mcip->mci_vidcache = MCIP_VIDCACHE_INVALID;
1363 
1364 	mcip->mci_unicast_list = NULL;
1365 
1366 	if ((flags & MAC_OPEN_FLAGS_IS_VNIC) != 0)
1367 		mcip->mci_state_flags |= MCIS_IS_VNIC;
1368 
1369 	if ((flags & MAC_OPEN_FLAGS_EXCLUSIVE) != 0)
1370 		mcip->mci_state_flags |= MCIS_EXCLUSIVE;
1371 
1372 	if ((flags & MAC_OPEN_FLAGS_IS_AGGR_PORT) != 0)
1373 		mcip->mci_state_flags |= MCIS_IS_AGGR_PORT;
1374 
1375 	if (mip->mi_state_flags & MIS_IS_AGGR)
1376 		mcip->mci_state_flags |= MCIS_IS_AGGR;
1377 
1378 	if ((flags & MAC_OPEN_FLAGS_USE_DATALINK_NAME) != 0) {
1379 		datalink_id_t	linkid;
1380 
1381 		ASSERT(name == NULL);
1382 		if ((err = dls_devnet_macname2linkid(mip->mi_name,
1383 		    &linkid)) != 0) {
1384 			goto done;
1385 		}
1386 		if ((err = dls_mgmt_get_linkinfo(linkid, mcip->mci_name, NULL,
1387 		    NULL, NULL)) != 0) {
1388 			/*
1389 			 * Use mac name if dlmgmtd is not available.
1390 			 */
1391 			if (err == EBADF) {
1392 				(void) strlcpy(mcip->mci_name, mip->mi_name,
1393 				    sizeof (mcip->mci_name));
1394 				err = 0;
1395 			} else {
1396 				goto done;
1397 			}
1398 		}
1399 		mcip->mci_state_flags |= MCIS_USE_DATALINK_NAME;
1400 	} else {
1401 		ASSERT(name != NULL);
1402 		if (strlen(name) > MAXNAMELEN) {
1403 			err = EINVAL;
1404 			goto done;
1405 		}
1406 		(void) strlcpy(mcip->mci_name, name, sizeof (mcip->mci_name));
1407 	}
1408 
1409 	if (flags & MAC_OPEN_FLAGS_MULTI_PRIMARY)
1410 		mcip->mci_flags |= MAC_CLIENT_FLAGS_MULTI_PRIMARY;
1411 
1412 	if (flags & MAC_OPEN_FLAGS_NO_UNICAST_ADDR)
1413 		mcip->mci_state_flags |= MCIS_NO_UNICAST_ADDR;
1414 
1415 	mac_protect_init(mcip);
1416 
1417 	/* the subflow table will be created dynamically */
1418 	mcip->mci_subflow_tab = NULL;
1419 
1420 	mcip->mci_misc_stat.mms_multircv = 0;
1421 	mcip->mci_misc_stat.mms_brdcstrcv = 0;
1422 	mcip->mci_misc_stat.mms_multixmt = 0;
1423 	mcip->mci_misc_stat.mms_brdcstxmt = 0;
1424 
1425 	/* Create an initial flow */
1426 
1427 	err = mac_flow_create(NULL, NULL, mcip->mci_name, NULL,
1428 	    mcip->mci_state_flags & MCIS_IS_VNIC ? FLOW_VNIC_MAC :
1429 	    FLOW_PRIMARY_MAC, &flent);
1430 	if (err != 0)
1431 		goto done;
1432 	mcip->mci_flent = flent;
1433 	FLOW_MARK(flent, FE_MC_NO_DATAPATH);
1434 	flent->fe_mcip = mcip;
1435 	/*
1436 	 * Place initial creation reference on the flow. This reference
1437 	 * is released in the corresponding delete action viz.
1438 	 * mac_unicast_remove after waiting for all transient refs to
1439 	 * to go away. The wait happens in mac_flow_wait.
1440 	 */
1441 	FLOW_REFHOLD(flent);
1442 
1443 	/*
1444 	 * Do this ahead of the mac_bcast_add() below so that the mi_nclients
1445 	 * will have the right value for mac_rx_srs_setup().
1446 	 */
1447 	mac_client_add(mcip);
1448 
1449 	mcip->mci_share = NULL;
1450 	if (share_desired)
1451 		i_mac_share_alloc(mcip);
1452 
1453 	/*
1454 	 * We will do mimimal datapath setup to allow a MAC client to
1455 	 * transmit or receive non-unicast packets without waiting
1456 	 * for mac_unicast_add.
1457 	 */
1458 	if (mcip->mci_state_flags & MCIS_NO_UNICAST_ADDR) {
1459 		if ((err = mac_client_datapath_setup(mcip, VLAN_ID_NONE,
1460 		    NULL, NULL, B_TRUE, NULL)) != 0) {
1461 			goto done;
1462 		}
1463 	}
1464 
1465 	DTRACE_PROBE2(mac__client__open__allocated, mac_impl_t *,
1466 	    mcip->mci_mip, mac_client_impl_t *, mcip);
1467 
1468 	*mchp = (mac_client_handle_t)mcip;
1469 	i_mac_perim_exit(mip);
1470 	return (0);
1471 
1472 done:
1473 	i_mac_perim_exit(mip);
1474 	mcip->mci_state_flags = 0;
1475 	mcip->mci_tx_flag = 0;
1476 	kmem_cache_free(mac_client_impl_cache, mcip);
1477 	return (err);
1478 }
1479 
1480 /*
1481  * Close the specified MAC client handle.
1482  */
1483 void
1484 mac_client_close(mac_client_handle_t mch, uint16_t flags)
1485 {
1486 	mac_client_impl_t	*mcip = (mac_client_impl_t *)mch;
1487 	mac_impl_t		*mip = mcip->mci_mip;
1488 	flow_entry_t		*flent;
1489 
1490 	i_mac_perim_enter(mip);
1491 
1492 	if (flags & MAC_CLOSE_FLAGS_EXCLUSIVE)
1493 		mcip->mci_state_flags &= ~MCIS_EXCLUSIVE;
1494 
1495 	if ((mcip->mci_state_flags & MCIS_IS_VNIC) &&
1496 	    !(flags & MAC_CLOSE_FLAGS_IS_VNIC)) {
1497 		/*
1498 		 * This is an upper VNIC client initiated operation.
1499 		 * The lower MAC client will be closed by the VNIC driver
1500 		 * when the VNIC is deleted.
1501 		 */
1502 
1503 		i_mac_perim_exit(mip);
1504 		return;
1505 	}
1506 
1507 	/* If we have only setup up minimal datapth setup, tear it down */
1508 	if (mcip->mci_state_flags & MCIS_NO_UNICAST_ADDR) {
1509 		mac_client_datapath_teardown((mac_client_handle_t)mcip, NULL,
1510 		    mcip->mci_flent);
1511 		mcip->mci_state_flags &= ~MCIS_NO_UNICAST_ADDR;
1512 	}
1513 
1514 	/*
1515 	 * Remove the flent associated with the MAC client
1516 	 */
1517 	flent = mcip->mci_flent;
1518 	mcip->mci_flent = NULL;
1519 	FLOW_FINAL_REFRELE(flent);
1520 
1521 	/*
1522 	 * MAC clients must remove the unicast addresses and promisc callbacks
1523 	 * they added before issuing a mac_client_close().
1524 	 */
1525 	ASSERT(mcip->mci_unicast_list == NULL);
1526 	ASSERT(mcip->mci_promisc_list == NULL);
1527 	ASSERT(mcip->mci_tx_notify_cb_list == NULL);
1528 
1529 	i_mac_share_free(mcip);
1530 	mac_protect_fini(mcip);
1531 	mac_client_remove(mcip);
1532 
1533 	i_mac_perim_exit(mip);
1534 	mcip->mci_subflow_tab = NULL;
1535 	mcip->mci_state_flags = 0;
1536 	mcip->mci_tx_flag = 0;
1537 	kmem_cache_free(mac_client_impl_cache, mch);
1538 }
1539 
1540 /*
1541  * Set the rx bypass receive callback.
1542  */
1543 boolean_t
1544 mac_rx_bypass_set(mac_client_handle_t mch, mac_direct_rx_t rx_fn, void *arg1)
1545 {
1546 	mac_client_impl_t	*mcip = (mac_client_impl_t *)mch;
1547 	mac_impl_t		*mip = mcip->mci_mip;
1548 
1549 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mip));
1550 
1551 	/*
1552 	 * If the mac_client is a VLAN, we should not do DLS bypass and
1553 	 * instead let the packets come up via mac_rx_deliver so the vlan
1554 	 * header can be stripped.
1555 	 */
1556 	if (mcip->mci_nvids > 0)
1557 		return (B_FALSE);
1558 
1559 	/*
1560 	 * These are not accessed directly in the data path, and hence
1561 	 * don't need any protection
1562 	 */
1563 	mcip->mci_direct_rx_fn = rx_fn;
1564 	mcip->mci_direct_rx_arg = arg1;
1565 	return (B_TRUE);
1566 }
1567 
1568 /*
1569  * Enable/Disable rx bypass. By default, bypass is assumed to be enabled.
1570  */
1571 void
1572 mac_rx_bypass_enable(mac_client_handle_t mch)
1573 {
1574 	((mac_client_impl_t *)mch)->mci_state_flags &= ~MCIS_RX_BYPASS_DISABLE;
1575 }
1576 
1577 void
1578 mac_rx_bypass_disable(mac_client_handle_t mch)
1579 {
1580 	((mac_client_impl_t *)mch)->mci_state_flags |= MCIS_RX_BYPASS_DISABLE;
1581 }
1582 
1583 /*
1584  * Set the receive callback for the specified MAC client. There can be
1585  * at most one such callback per MAC client.
1586  */
1587 void
1588 mac_rx_set(mac_client_handle_t mch, mac_rx_t rx_fn, void *arg)
1589 {
1590 	mac_client_impl_t *mcip = (mac_client_impl_t *)mch;
1591 	mac_impl_t	*mip = mcip->mci_mip;
1592 	mac_impl_t	*umip = mcip->mci_upper_mip;
1593 
1594 	/*
1595 	 * Instead of adding an extra set of locks and refcnts in
1596 	 * the datapath at the mac client boundary, we temporarily quiesce
1597 	 * the SRS and related entities. We then change the receive function
1598 	 * without interference from any receive data thread and then reenable
1599 	 * the data flow subsequently.
1600 	 */
1601 	i_mac_perim_enter(mip);
1602 	mac_rx_client_quiesce(mch);
1603 
1604 	mcip->mci_rx_fn = rx_fn;
1605 	mcip->mci_rx_arg = arg;
1606 	mac_rx_client_restart(mch);
1607 	i_mac_perim_exit(mip);
1608 
1609 	/*
1610 	 * If we're changing the rx function on the primary mac of a vnic,
1611 	 * make sure any secondary macs on the vnic are updated as well.
1612 	 */
1613 	if (umip != NULL) {
1614 		ASSERT((umip->mi_state_flags & MIS_IS_VNIC) != 0);
1615 		mac_vnic_secondary_update(umip);
1616 	}
1617 }
1618 
1619 /*
1620  * Reset the receive callback for the specified MAC client.
1621  */
1622 void
1623 mac_rx_clear(mac_client_handle_t mch)
1624 {
1625 	mac_rx_set(mch, mac_pkt_drop, NULL);
1626 }
1627 
1628 void
1629 mac_secondary_dup(mac_client_handle_t smch, mac_client_handle_t dmch)
1630 {
1631 	mac_client_impl_t *smcip = (mac_client_impl_t *)smch;
1632 	mac_client_impl_t *dmcip = (mac_client_impl_t *)dmch;
1633 	flow_entry_t *flent = dmcip->mci_flent;
1634 
1635 	/* This should only be called to setup secondary macs */
1636 	ASSERT((flent->fe_type & FLOW_PRIMARY_MAC) == 0);
1637 
1638 	mac_rx_set(dmch, smcip->mci_rx_fn, smcip->mci_rx_arg);
1639 	dmcip->mci_promisc_list = smcip->mci_promisc_list;
1640 
1641 	/*
1642 	 * Duplicate the primary mac resources to the secondary.
1643 	 * Since we already validated the resource controls when setting
1644 	 * them on the primary, we can ignore errors here.
1645 	 */
1646 	(void) mac_resource_ctl_set(dmch, MCIP_RESOURCE_PROPS(smcip));
1647 }
1648 
1649 /*
1650  * Called when removing a secondary MAC. Currently only clears the promisc_list
1651  * since we share the primary mac's promisc_list.
1652  */
1653 void
1654 mac_secondary_cleanup(mac_client_handle_t mch)
1655 {
1656 	mac_client_impl_t *mcip = (mac_client_impl_t *)mch;
1657 	flow_entry_t *flent = mcip->mci_flent;
1658 
1659 	/* This should only be called for secondary macs */
1660 	ASSERT((flent->fe_type & FLOW_PRIMARY_MAC) == 0);
1661 	mcip->mci_promisc_list = NULL;
1662 }
1663 
1664 /*
1665  * Walk the MAC client subflow table and updates their priority values.
1666  */
1667 static int
1668 mac_update_subflow_priority_cb(flow_entry_t *flent, void *arg)
1669 {
1670 	mac_flow_update_priority(arg, flent);
1671 	return (0);
1672 }
1673 
1674 void
1675 mac_update_subflow_priority(mac_client_impl_t *mcip)
1676 {
1677 	(void) mac_flow_walk(mcip->mci_subflow_tab,
1678 	    mac_update_subflow_priority_cb, mcip);
1679 }
1680 
1681 /*
1682  * Modify the TX or RX ring properties. We could either just move around
1683  * rings, i.e add/remove rings given to a client. Or this might cause the
1684  * client to move from hardware based to software or the other way around.
1685  * If we want to reset this property, then we clear the mask, additionally
1686  * if the client was given a non-default group we remove all rings except
1687  * for 1 and give it back to the default group.
1688  */
1689 int
1690 mac_client_set_rings_prop(mac_client_impl_t *mcip, mac_resource_props_t *mrp,
1691     mac_resource_props_t *tmrp)
1692 {
1693 	mac_impl_t		*mip = mcip->mci_mip;
1694 	flow_entry_t		*flent = mcip->mci_flent;
1695 	uint8_t			*mac_addr;
1696 	int			err = 0;
1697 	mac_group_t		*defgrp;
1698 	mac_group_t		*group;
1699 	mac_group_t		*ngrp;
1700 	mac_resource_props_t	*cmrp = MCIP_RESOURCE_PROPS(mcip);
1701 	uint_t			ringcnt;
1702 	boolean_t		unspec;
1703 
1704 	if (mcip->mci_share != NULL)
1705 		return (EINVAL);
1706 
1707 	if (mrp->mrp_mask & MRP_RX_RINGS) {
1708 		unspec = mrp->mrp_mask & MRP_RXRINGS_UNSPEC;
1709 		group = flent->fe_rx_ring_group;
1710 		defgrp = MAC_DEFAULT_RX_GROUP(mip);
1711 		mac_addr = flent->fe_flow_desc.fd_dst_mac;
1712 
1713 		/*
1714 		 * No resulting change. If we are resetting on a client on
1715 		 * which there was no rx rings property. For dynamic group
1716 		 * if we are setting the same number of rings already set.
1717 		 * For static group if we are requesting a group again.
1718 		 */
1719 		if (mrp->mrp_mask & MRP_RINGS_RESET) {
1720 			if (!(tmrp->mrp_mask & MRP_RX_RINGS))
1721 				return (0);
1722 		} else {
1723 			if (unspec) {
1724 				if (tmrp->mrp_mask & MRP_RXRINGS_UNSPEC)
1725 					return (0);
1726 			} else if (mip->mi_rx_group_type ==
1727 			    MAC_GROUP_TYPE_DYNAMIC) {
1728 				if ((tmrp->mrp_mask & MRP_RX_RINGS) &&
1729 				    !(tmrp->mrp_mask & MRP_RXRINGS_UNSPEC) &&
1730 				    mrp->mrp_nrxrings == tmrp->mrp_nrxrings) {
1731 					return (0);
1732 				}
1733 			}
1734 		}
1735 		/* Resetting the prop */
1736 		if (mrp->mrp_mask & MRP_RINGS_RESET) {
1737 			/*
1738 			 * We will just keep one ring and give others back if
1739 			 * we are not the primary. For the primary we give
1740 			 * all the rings in the default group except the
1741 			 * default ring. If it is a static group, then
1742 			 * we don't do anything, but clear the MRP_RX_RINGS
1743 			 * flag.
1744 			 */
1745 			if (group != defgrp) {
1746 				if (mip->mi_rx_group_type ==
1747 				    MAC_GROUP_TYPE_DYNAMIC) {
1748 					/*
1749 					 * This group has reserved rings
1750 					 * that need to be released now,
1751 					 * so does the group.
1752 					 */
1753 					MAC_RX_RING_RELEASED(mip,
1754 					    group->mrg_cur_count);
1755 					MAC_RX_GRP_RELEASED(mip);
1756 					if ((flent->fe_type &
1757 					    FLOW_PRIMARY_MAC) != 0) {
1758 						if (mip->mi_nactiveclients ==
1759 						    1) {
1760 							(void)
1761 							    mac_rx_switch_group(
1762 							    mcip, group,
1763 							    defgrp);
1764 							return (0);
1765 						} else {
1766 							cmrp->mrp_nrxrings =
1767 							    group->
1768 							    mrg_cur_count +
1769 							    defgrp->
1770 							    mrg_cur_count - 1;
1771 						}
1772 					} else {
1773 						cmrp->mrp_nrxrings = 1;
1774 					}
1775 					(void) mac_group_ring_modify(mcip,
1776 					    group, defgrp);
1777 				} else {
1778 					/*
1779 					 * If this is a static group, we
1780 					 * need to release the group. The
1781 					 * client will remain in the same
1782 					 * group till some other client
1783 					 * needs this group.
1784 					 */
1785 					MAC_RX_GRP_RELEASED(mip);
1786 				}
1787 			/* Let check if we can give this an excl group */
1788 			} else if (group == defgrp) {
1789 				ngrp = 	mac_reserve_rx_group(mcip, mac_addr,
1790 				    B_TRUE);
1791 				/* Couldn't give it a group, that's fine */
1792 				if (ngrp == NULL)
1793 					return (0);
1794 				/* Switch to H/W */
1795 				if (mac_rx_switch_group(mcip, defgrp, ngrp) !=
1796 				    0) {
1797 					mac_stop_group(ngrp);
1798 					return (0);
1799 				}
1800 			}
1801 			/*
1802 			 * If the client is in the default group, we will
1803 			 * just clear the MRP_RX_RINGS and leave it as
1804 			 * it rather than look for an exclusive group
1805 			 * for it.
1806 			 */
1807 			return (0);
1808 		}
1809 
1810 		if (group == defgrp && ((mrp->mrp_nrxrings > 0) || unspec)) {
1811 			ngrp = 	mac_reserve_rx_group(mcip, mac_addr, B_TRUE);
1812 			if (ngrp == NULL)
1813 				return (ENOSPC);
1814 
1815 			/* Switch to H/W */
1816 			if (mac_rx_switch_group(mcip, defgrp, ngrp) != 0) {
1817 				mac_release_rx_group(mcip, ngrp);
1818 				return (ENOSPC);
1819 			}
1820 			MAC_RX_GRP_RESERVED(mip);
1821 			if (mip->mi_rx_group_type == MAC_GROUP_TYPE_DYNAMIC)
1822 				MAC_RX_RING_RESERVED(mip, ngrp->mrg_cur_count);
1823 		} else if (group != defgrp && !unspec &&
1824 		    mrp->mrp_nrxrings == 0) {
1825 			/* Switch to S/W */
1826 			ringcnt = group->mrg_cur_count;
1827 			if (mac_rx_switch_group(mcip, group, defgrp) != 0)
1828 				return (ENOSPC);
1829 			if (tmrp->mrp_mask & MRP_RX_RINGS) {
1830 				MAC_RX_GRP_RELEASED(mip);
1831 				if (mip->mi_rx_group_type ==
1832 				    MAC_GROUP_TYPE_DYNAMIC) {
1833 					MAC_RX_RING_RELEASED(mip, ringcnt);
1834 				}
1835 			}
1836 		} else if (group != defgrp && mip->mi_rx_group_type ==
1837 		    MAC_GROUP_TYPE_DYNAMIC) {
1838 			ringcnt = group->mrg_cur_count;
1839 			err = mac_group_ring_modify(mcip, group, defgrp);
1840 			if (err != 0)
1841 				return (err);
1842 			/*
1843 			 * Update the accounting. If this group
1844 			 * already had explicitly reserved rings,
1845 			 * we need to update the rings based on
1846 			 * the new ring count. If this group
1847 			 * had not explicitly reserved rings,
1848 			 * then we just reserve the rings asked for
1849 			 * and reserve the group.
1850 			 */
1851 			if (tmrp->mrp_mask & MRP_RX_RINGS) {
1852 				if (ringcnt > group->mrg_cur_count) {
1853 					MAC_RX_RING_RELEASED(mip,
1854 					    ringcnt - group->mrg_cur_count);
1855 				} else {
1856 					MAC_RX_RING_RESERVED(mip,
1857 					    group->mrg_cur_count - ringcnt);
1858 				}
1859 			} else {
1860 				MAC_RX_RING_RESERVED(mip, group->mrg_cur_count);
1861 				MAC_RX_GRP_RESERVED(mip);
1862 			}
1863 		}
1864 	}
1865 	if (mrp->mrp_mask & MRP_TX_RINGS) {
1866 		unspec = mrp->mrp_mask & MRP_TXRINGS_UNSPEC;
1867 		group = flent->fe_tx_ring_group;
1868 		defgrp = MAC_DEFAULT_TX_GROUP(mip);
1869 
1870 		/*
1871 		 * For static groups we only allow rings=0 or resetting the
1872 		 * rings property.
1873 		 */
1874 		if (mrp->mrp_ntxrings > 0 &&
1875 		    mip->mi_tx_group_type != MAC_GROUP_TYPE_DYNAMIC) {
1876 			return (ENOTSUP);
1877 		}
1878 		if (mrp->mrp_mask & MRP_RINGS_RESET) {
1879 			if (!(tmrp->mrp_mask & MRP_TX_RINGS))
1880 				return (0);
1881 		} else {
1882 			if (unspec) {
1883 				if (tmrp->mrp_mask & MRP_TXRINGS_UNSPEC)
1884 					return (0);
1885 			} else if (mip->mi_tx_group_type ==
1886 			    MAC_GROUP_TYPE_DYNAMIC) {
1887 				if ((tmrp->mrp_mask & MRP_TX_RINGS) &&
1888 				    !(tmrp->mrp_mask & MRP_TXRINGS_UNSPEC) &&
1889 				    mrp->mrp_ntxrings == tmrp->mrp_ntxrings) {
1890 					return (0);
1891 				}
1892 			}
1893 		}
1894 		/* Resetting the prop */
1895 		if (mrp->mrp_mask & MRP_RINGS_RESET) {
1896 			if (group != defgrp) {
1897 				if (mip->mi_tx_group_type ==
1898 				    MAC_GROUP_TYPE_DYNAMIC) {
1899 					ringcnt = group->mrg_cur_count;
1900 					if ((flent->fe_type &
1901 					    FLOW_PRIMARY_MAC) != 0) {
1902 						mac_tx_client_quiesce(
1903 						    (mac_client_handle_t)
1904 						    mcip);
1905 						mac_tx_switch_group(mcip,
1906 						    group, defgrp);
1907 						mac_tx_client_restart(
1908 						    (mac_client_handle_t)
1909 						    mcip);
1910 						MAC_TX_GRP_RELEASED(mip);
1911 						MAC_TX_RING_RELEASED(mip,
1912 						    ringcnt);
1913 						return (0);
1914 					}
1915 					cmrp->mrp_ntxrings = 1;
1916 					(void) mac_group_ring_modify(mcip,
1917 					    group, defgrp);
1918 					/*
1919 					 * This group has reserved rings
1920 					 * that need to be released now.
1921 					 */
1922 					MAC_TX_RING_RELEASED(mip, ringcnt);
1923 				}
1924 				/*
1925 				 * If this is a static group, we
1926 				 * need to release the group. The
1927 				 * client will remain in the same
1928 				 * group till some other client
1929 				 * needs this group.
1930 				 */
1931 				MAC_TX_GRP_RELEASED(mip);
1932 			} else if (group == defgrp &&
1933 			    (flent->fe_type & FLOW_PRIMARY_MAC) == 0) {
1934 				ngrp = mac_reserve_tx_group(mcip, B_TRUE);
1935 				if (ngrp == NULL)
1936 					return (0);
1937 				mac_tx_client_quiesce(
1938 				    (mac_client_handle_t)mcip);
1939 				mac_tx_switch_group(mcip, defgrp, ngrp);
1940 				mac_tx_client_restart(
1941 				    (mac_client_handle_t)mcip);
1942 			}
1943 			/*
1944 			 * If the client is in the default group, we will
1945 			 * just clear the MRP_TX_RINGS and leave it as
1946 			 * it rather than look for an exclusive group
1947 			 * for it.
1948 			 */
1949 			return (0);
1950 		}
1951 
1952 		/* Switch to H/W */
1953 		if (group == defgrp && ((mrp->mrp_ntxrings > 0) || unspec)) {
1954 			ngrp = 	mac_reserve_tx_group(mcip, B_TRUE);
1955 			if (ngrp == NULL)
1956 				return (ENOSPC);
1957 			mac_tx_client_quiesce((mac_client_handle_t)mcip);
1958 			mac_tx_switch_group(mcip, defgrp, ngrp);
1959 			mac_tx_client_restart((mac_client_handle_t)mcip);
1960 			MAC_TX_GRP_RESERVED(mip);
1961 			if (mip->mi_tx_group_type == MAC_GROUP_TYPE_DYNAMIC)
1962 				MAC_TX_RING_RESERVED(mip, ngrp->mrg_cur_count);
1963 		/* Switch to S/W */
1964 		} else if (group != defgrp && !unspec &&
1965 		    mrp->mrp_ntxrings == 0) {
1966 			/* Switch to S/W */
1967 			ringcnt = group->mrg_cur_count;
1968 			mac_tx_client_quiesce((mac_client_handle_t)mcip);
1969 			mac_tx_switch_group(mcip, group, defgrp);
1970 			mac_tx_client_restart((mac_client_handle_t)mcip);
1971 			if (tmrp->mrp_mask & MRP_TX_RINGS) {
1972 				MAC_TX_GRP_RELEASED(mip);
1973 				if (mip->mi_tx_group_type ==
1974 				    MAC_GROUP_TYPE_DYNAMIC) {
1975 					MAC_TX_RING_RELEASED(mip, ringcnt);
1976 				}
1977 			}
1978 		} else if (group != defgrp && mip->mi_tx_group_type ==
1979 		    MAC_GROUP_TYPE_DYNAMIC) {
1980 			ringcnt = group->mrg_cur_count;
1981 			err = mac_group_ring_modify(mcip, group, defgrp);
1982 			if (err != 0)
1983 				return (err);
1984 			/*
1985 			 * Update the accounting. If this group
1986 			 * already had explicitly reserved rings,
1987 			 * we need to update the rings based on
1988 			 * the new ring count. If this group
1989 			 * had not explicitly reserved rings,
1990 			 * then we just reserve the rings asked for
1991 			 * and reserve the group.
1992 			 */
1993 			if (tmrp->mrp_mask & MRP_TX_RINGS) {
1994 				if (ringcnt > group->mrg_cur_count) {
1995 					MAC_TX_RING_RELEASED(mip,
1996 					    ringcnt - group->mrg_cur_count);
1997 				} else {
1998 					MAC_TX_RING_RESERVED(mip,
1999 					    group->mrg_cur_count - ringcnt);
2000 				}
2001 			} else {
2002 				MAC_TX_RING_RESERVED(mip, group->mrg_cur_count);
2003 				MAC_TX_GRP_RESERVED(mip);
2004 			}
2005 		}
2006 	}
2007 	return (0);
2008 }
2009 
2010 /*
2011  * When the MAC client is being brought up (i.e. we do a unicast_add) we need
2012  * to initialize the cpu and resource control structure in the
2013  * mac_client_impl_t from the mac_impl_t (i.e if there are any cached
2014  * properties before the flow entry for the unicast address was created).
2015  */
2016 static int
2017 mac_resource_ctl_set(mac_client_handle_t mch, mac_resource_props_t *mrp)
2018 {
2019 	mac_client_impl_t 	*mcip = (mac_client_impl_t *)mch;
2020 	mac_impl_t		*mip = (mac_impl_t *)mcip->mci_mip;
2021 	mac_impl_t		*umip = mcip->mci_upper_mip;
2022 	int			err = 0;
2023 	flow_entry_t		*flent = mcip->mci_flent;
2024 	mac_resource_props_t	*omrp, *nmrp = MCIP_RESOURCE_PROPS(mcip);
2025 
2026 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mip));
2027 
2028 	err = mac_validate_props(mcip->mci_state_flags & MCIS_IS_VNIC ?
2029 	    mcip->mci_upper_mip : mip, mrp);
2030 	if (err != 0)
2031 		return (err);
2032 
2033 	/*
2034 	 * Copy over the existing properties since mac_update_resources
2035 	 * will modify the client's mrp. Currently, the saved property
2036 	 * is used to determine the difference between existing and
2037 	 * modified rings property.
2038 	 */
2039 	omrp = kmem_zalloc(sizeof (*omrp), KM_SLEEP);
2040 	bcopy(nmrp, omrp, sizeof (*omrp));
2041 	mac_update_resources(mrp, MCIP_RESOURCE_PROPS(mcip), B_FALSE);
2042 	if (MCIP_DATAPATH_SETUP(mcip)) {
2043 		/*
2044 		 * We support rings only for primary client when there are
2045 		 * multiple clients sharing the same MAC address (e.g. VLAN).
2046 		 */
2047 		if (mrp->mrp_mask & MRP_RX_RINGS ||
2048 		    mrp->mrp_mask & MRP_TX_RINGS) {
2049 
2050 			if ((err = mac_client_set_rings_prop(mcip, mrp,
2051 			    omrp)) != 0) {
2052 				if (omrp->mrp_mask & MRP_RX_RINGS) {
2053 					nmrp->mrp_mask |= MRP_RX_RINGS;
2054 					nmrp->mrp_nrxrings = omrp->mrp_nrxrings;
2055 				} else {
2056 					nmrp->mrp_mask &= ~MRP_RX_RINGS;
2057 					nmrp->mrp_nrxrings = 0;
2058 				}
2059 				if (omrp->mrp_mask & MRP_TX_RINGS) {
2060 					nmrp->mrp_mask |= MRP_TX_RINGS;
2061 					nmrp->mrp_ntxrings = omrp->mrp_ntxrings;
2062 				} else {
2063 					nmrp->mrp_mask &= ~MRP_TX_RINGS;
2064 					nmrp->mrp_ntxrings = 0;
2065 				}
2066 				if (omrp->mrp_mask & MRP_RXRINGS_UNSPEC)
2067 					omrp->mrp_mask |= MRP_RXRINGS_UNSPEC;
2068 				else
2069 					omrp->mrp_mask &= ~MRP_RXRINGS_UNSPEC;
2070 
2071 				if (omrp->mrp_mask & MRP_TXRINGS_UNSPEC)
2072 					omrp->mrp_mask |= MRP_TXRINGS_UNSPEC;
2073 				else
2074 					omrp->mrp_mask &= ~MRP_TXRINGS_UNSPEC;
2075 				kmem_free(omrp, sizeof (*omrp));
2076 				return (err);
2077 			}
2078 
2079 			/*
2080 			 * If we modified the rings property of the primary
2081 			 * we need to update the property fields of its
2082 			 * VLANs as they inherit the primary's properites.
2083 			 */
2084 			if (mac_is_primary_client(mcip)) {
2085 				mac_set_prim_vlan_rings(mip,
2086 				    MCIP_RESOURCE_PROPS(mcip));
2087 			}
2088 		}
2089 		/*
2090 		 * We have to set this prior to calling mac_flow_modify.
2091 		 */
2092 		if (mrp->mrp_mask & MRP_PRIORITY) {
2093 			if (mrp->mrp_priority == MPL_RESET) {
2094 				MAC_CLIENT_SET_PRIORITY_RANGE(mcip,
2095 				    MPL_LINK_DEFAULT);
2096 			} else {
2097 				MAC_CLIENT_SET_PRIORITY_RANGE(mcip,
2098 				    mrp->mrp_priority);
2099 			}
2100 		}
2101 
2102 		mac_flow_modify(mip->mi_flow_tab, flent, mrp);
2103 		if (mrp->mrp_mask & MRP_PRIORITY)
2104 			mac_update_subflow_priority(mcip);
2105 
2106 		/* Apply these resource settings to any secondary macs */
2107 		if (umip != NULL) {
2108 			ASSERT((umip->mi_state_flags & MIS_IS_VNIC) != 0);
2109 			mac_vnic_secondary_update(umip);
2110 		}
2111 	}
2112 	kmem_free(omrp, sizeof (*omrp));
2113 	return (0);
2114 }
2115 
2116 static int
2117 mac_unicast_flow_create(mac_client_impl_t *mcip, uint8_t *mac_addr,
2118     uint16_t vid, boolean_t is_primary, boolean_t first_flow,
2119     flow_entry_t **flent, mac_resource_props_t *mrp)
2120 {
2121 	mac_impl_t	*mip = (mac_impl_t *)mcip->mci_mip;
2122 	flow_desc_t	flow_desc;
2123 	char		flowname[MAXFLOWNAMELEN];
2124 	int		err;
2125 	uint_t		flent_flags;
2126 
2127 	/*
2128 	 * First unicast address being added, create a new flow
2129 	 * for that MAC client.
2130 	 */
2131 	bzero(&flow_desc, sizeof (flow_desc));
2132 
2133 	ASSERT(mac_addr != NULL ||
2134 	    (mcip->mci_state_flags & MCIS_NO_UNICAST_ADDR));
2135 	if (mac_addr != NULL) {
2136 		flow_desc.fd_mac_len = mip->mi_type->mt_addr_length;
2137 		bcopy(mac_addr, flow_desc.fd_dst_mac, flow_desc.fd_mac_len);
2138 	}
2139 	flow_desc.fd_mask = FLOW_LINK_DST;
2140 	if (vid != 0) {
2141 		flow_desc.fd_vid = vid;
2142 		flow_desc.fd_mask |= FLOW_LINK_VID;
2143 	}
2144 
2145 	/*
2146 	 * XXX-nicolas. For now I'm keeping the FLOW_PRIMARY_MAC
2147 	 * and FLOW_VNIC. Even though they're a hack inherited
2148 	 * from the SRS code, we'll keep them for now. They're currently
2149 	 * consumed by mac_datapath_setup() to create the SRS.
2150 	 * That code should be eventually moved out of
2151 	 * mac_datapath_setup() and moved to a mac_srs_create()
2152 	 * function of some sort to keep things clean.
2153 	 *
2154 	 * Also, there's no reason why the SRS for the primary MAC
2155 	 * client should be different than any other MAC client. Until
2156 	 * this is cleaned-up, we support only one MAC unicast address
2157 	 * per client.
2158 	 *
2159 	 * We set FLOW_PRIMARY_MAC for the primary MAC address,
2160 	 * FLOW_VNIC for everything else.
2161 	 */
2162 	if (is_primary)
2163 		flent_flags = FLOW_PRIMARY_MAC;
2164 	else
2165 		flent_flags = FLOW_VNIC_MAC;
2166 
2167 	/*
2168 	 * For the first flow we use the mac client's name - mci_name, for
2169 	 * subsequent ones we just create a name with the vid. This is
2170 	 * so that we can add these flows to the same flow table. This is
2171 	 * fine as the flow name (except for the one with the mac client's
2172 	 * name) is not visible. When the first flow is removed, we just replace
2173 	 * its fdesc with another from the list, so we will still retain the
2174 	 * flent with the MAC client's flow name.
2175 	 */
2176 	if (first_flow) {
2177 		bcopy(mcip->mci_name, flowname, MAXFLOWNAMELEN);
2178 	} else {
2179 		(void) sprintf(flowname, "%s%u", mcip->mci_name, vid);
2180 		flent_flags = FLOW_NO_STATS;
2181 	}
2182 
2183 	if ((err = mac_flow_create(&flow_desc, mrp, flowname, NULL,
2184 	    flent_flags, flent)) != 0)
2185 		return (err);
2186 
2187 	mac_misc_stat_create(*flent);
2188 	FLOW_MARK(*flent, FE_INCIPIENT);
2189 	(*flent)->fe_mcip = mcip;
2190 
2191 	/*
2192 	 * Place initial creation reference on the flow. This reference
2193 	 * is released in the corresponding delete action viz.
2194 	 * mac_unicast_remove after waiting for all transient refs to
2195 	 * to go away. The wait happens in mac_flow_wait.
2196 	 * We have already held the reference in mac_client_open().
2197 	 */
2198 	if (!first_flow)
2199 		FLOW_REFHOLD(*flent);
2200 	return (0);
2201 }
2202 
2203 /* Refresh the multicast grouping for this VID. */
2204 int
2205 mac_client_update_mcast(void *arg, boolean_t add, const uint8_t *addrp)
2206 {
2207 	flow_entry_t		*flent = arg;
2208 	mac_client_impl_t	*mcip = flent->fe_mcip;
2209 	uint16_t		vid;
2210 	flow_desc_t		flow_desc;
2211 
2212 	mac_flow_get_desc(flent, &flow_desc);
2213 	vid = (flow_desc.fd_mask & FLOW_LINK_VID) != 0 ?
2214 	    flow_desc.fd_vid : VLAN_ID_NONE;
2215 
2216 	/*
2217 	 * We don't call mac_multicast_add()/mac_multicast_remove() as
2218 	 * we want to add/remove for this specific vid.
2219 	 */
2220 	if (add) {
2221 		return (mac_bcast_add(mcip, addrp, vid,
2222 		    MAC_ADDRTYPE_MULTICAST));
2223 	} else {
2224 		mac_bcast_delete(mcip, addrp, vid);
2225 		return (0);
2226 	}
2227 }
2228 
2229 static void
2230 mac_update_single_active_client(mac_impl_t *mip)
2231 {
2232 	mac_client_impl_t *client = NULL;
2233 
2234 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mip));
2235 
2236 	rw_enter(&mip->mi_rw_lock, RW_WRITER);
2237 	if (mip->mi_nactiveclients == 1) {
2238 		/*
2239 		 * Find the one active MAC client from the list of MAC
2240 		 * clients. The active MAC client has at least one
2241 		 * unicast address.
2242 		 */
2243 		for (client = mip->mi_clients_list; client != NULL;
2244 		    client = client->mci_client_next) {
2245 			if (client->mci_unicast_list != NULL)
2246 				break;
2247 		}
2248 		ASSERT(client != NULL);
2249 	}
2250 
2251 	/*
2252 	 * mi_single_active_client is protected by the MAC impl's read/writer
2253 	 * lock, which allows mac_rx() to check the value of that pointer
2254 	 * as a reader.
2255 	 */
2256 	mip->mi_single_active_client = client;
2257 	rw_exit(&mip->mi_rw_lock);
2258 }
2259 
2260 /*
2261  * Set up the data path. Called from i_mac_unicast_add after having
2262  * done all the validations including making sure this is an active
2263  * client (i.e that is ready to process packets.)
2264  */
2265 static int
2266 mac_client_datapath_setup(mac_client_impl_t *mcip, uint16_t vid,
2267     uint8_t *mac_addr, mac_resource_props_t *mrp, boolean_t isprimary,
2268     mac_unicast_impl_t *muip)
2269 {
2270 	mac_impl_t	*mip = mcip->mci_mip;
2271 	boolean_t	mac_started = B_FALSE;
2272 	boolean_t	bcast_added = B_FALSE;
2273 	boolean_t	nactiveclients_added = B_FALSE;
2274 	flow_entry_t	*flent;
2275 	int		err = 0;
2276 	boolean_t	no_unicast;
2277 
2278 	no_unicast = mcip->mci_state_flags & MCIS_NO_UNICAST_ADDR;
2279 
2280 	if ((err = mac_start((mac_handle_t)mip)) != 0)
2281 		goto bail;
2282 
2283 	mac_started = B_TRUE;
2284 
2285 	/* add the MAC client to the broadcast address group by default */
2286 	if (mip->mi_type->mt_brdcst_addr != NULL) {
2287 		err = mac_bcast_add(mcip, mip->mi_type->mt_brdcst_addr, vid,
2288 		    MAC_ADDRTYPE_BROADCAST);
2289 		if (err != 0)
2290 			goto bail;
2291 		bcast_added = B_TRUE;
2292 	}
2293 
2294 	/*
2295 	 * If this is the first unicast address addition for this
2296 	 * client, reuse the pre-allocated larval flow entry associated with
2297 	 * the MAC client.
2298 	 */
2299 	flent = (mcip->mci_nflents == 0) ? mcip->mci_flent : NULL;
2300 
2301 	/* We are configuring the unicast flow now */
2302 	if (!MCIP_DATAPATH_SETUP(mcip)) {
2303 
2304 		if (mrp != NULL) {
2305 			MAC_CLIENT_SET_PRIORITY_RANGE(mcip,
2306 			    (mrp->mrp_mask & MRP_PRIORITY) ? mrp->mrp_priority :
2307 			    MPL_LINK_DEFAULT);
2308 		}
2309 		if ((err = mac_unicast_flow_create(mcip, mac_addr, vid,
2310 		    isprimary, B_TRUE, &flent, mrp)) != 0)
2311 			goto bail;
2312 
2313 		mip->mi_nactiveclients++;
2314 		nactiveclients_added = B_TRUE;
2315 
2316 		/*
2317 		 * This will allocate the RX ring group if possible for the
2318 		 * flow and program the software classifier as needed.
2319 		 */
2320 		if ((err = mac_datapath_setup(mcip, flent, SRST_LINK)) != 0)
2321 			goto bail;
2322 
2323 		if (no_unicast)
2324 			goto done_setup;
2325 		/*
2326 		 * The unicast MAC address must have been added successfully.
2327 		 */
2328 		ASSERT(mcip->mci_unicast != NULL);
2329 		/*
2330 		 * Push down the sub-flows that were defined on this link
2331 		 * hitherto. The flows are added to the active flow table
2332 		 * and SRS, softrings etc. are created as needed.
2333 		 */
2334 		mac_link_init_flows((mac_client_handle_t)mcip);
2335 	} else {
2336 		mac_address_t *map = mcip->mci_unicast;
2337 
2338 		ASSERT(!no_unicast);
2339 		/*
2340 		 * A unicast flow already exists for that MAC client,
2341 		 * this flow must be the same mac address but with
2342 		 * different VID. It has been checked by mac_addr_in_use().
2343 		 *
2344 		 * We will use the SRS etc. from the mci_flent. Note that
2345 		 * We don't need to create kstat for this as except for
2346 		 * the fdesc, everything will be used from in the 1st flent.
2347 		 */
2348 
2349 		if (bcmp(mac_addr, map->ma_addr, map->ma_len) != 0) {
2350 			err = EINVAL;
2351 			goto bail;
2352 		}
2353 
2354 		if ((err = mac_unicast_flow_create(mcip, mac_addr, vid,
2355 		    isprimary, B_FALSE, &flent, NULL)) != 0) {
2356 			goto bail;
2357 		}
2358 		if ((err = mac_flow_add(mip->mi_flow_tab, flent)) != 0) {
2359 			FLOW_FINAL_REFRELE(flent);
2360 			goto bail;
2361 		}
2362 
2363 		/* update the multicast group for this vid */
2364 		mac_client_bcast_refresh(mcip, mac_client_update_mcast,
2365 		    (void *)flent, B_TRUE);
2366 
2367 	}
2368 
2369 	/* populate the shared MAC address */
2370 	muip->mui_map = mcip->mci_unicast;
2371 
2372 	rw_enter(&mcip->mci_rw_lock, RW_WRITER);
2373 	muip->mui_next = mcip->mci_unicast_list;
2374 	mcip->mci_unicast_list = muip;
2375 	rw_exit(&mcip->mci_rw_lock);
2376 
2377 done_setup:
2378 	/*
2379 	 * First add the flent to the flow list of this mcip. Then set
2380 	 * the mip's mi_single_active_client if needed. The Rx path assumes
2381 	 * that mip->mi_single_active_client will always have an associated
2382 	 * flent.
2383 	 */
2384 	mac_client_add_to_flow_list(mcip, flent);
2385 	if (nactiveclients_added)
2386 		mac_update_single_active_client(mip);
2387 	/*
2388 	 * Trigger a renegotiation of the capabilities when the number of
2389 	 * active clients changes from 1 to 2, since some of the capabilities
2390 	 * might have to be disabled. Also send a MAC_NOTE_LINK notification
2391 	 * to all the MAC clients whenever physical link is DOWN.
2392 	 */
2393 	if (mip->mi_nactiveclients == 2) {
2394 		mac_capab_update((mac_handle_t)mip);
2395 		mac_virtual_link_update(mip);
2396 	}
2397 	/*
2398 	 * Now that the setup is complete, clear the INCIPIENT flag.
2399 	 * The flag was set to avoid incoming packets seeing inconsistent
2400 	 * structures while the setup was in progress. Clear the mci_tx_flag
2401 	 * by calling mac_tx_client_block. It is possible that
2402 	 * mac_unicast_remove was called prior to this mac_unicast_add which
2403 	 * could have set the MCI_TX_QUIESCE flag.
2404 	 */
2405 	if (flent->fe_rx_ring_group != NULL)
2406 		mac_rx_group_unmark(flent->fe_rx_ring_group, MR_INCIPIENT);
2407 	FLOW_UNMARK(flent, FE_INCIPIENT);
2408 	FLOW_UNMARK(flent, FE_MC_NO_DATAPATH);
2409 	mac_tx_client_unblock(mcip);
2410 	return (0);
2411 bail:
2412 	if (bcast_added)
2413 		mac_bcast_delete(mcip, mip->mi_type->mt_brdcst_addr, vid);
2414 
2415 	if (nactiveclients_added)
2416 		mip->mi_nactiveclients--;
2417 
2418 	if (mac_started)
2419 		mac_stop((mac_handle_t)mip);
2420 
2421 	return (err);
2422 }
2423 
2424 /*
2425  * Return the passive primary MAC client, if present. The passive client is
2426  * a stand-by client that has the same unicast address as another that is
2427  * currenly active. Once the active client goes away, the passive client
2428  * becomes active.
2429  */
2430 static mac_client_impl_t *
2431 mac_get_passive_primary_client(mac_impl_t *mip)
2432 {
2433 	mac_client_impl_t	*mcip;
2434 
2435 	for (mcip = mip->mi_clients_list; mcip != NULL;
2436 	    mcip = mcip->mci_client_next) {
2437 		if (mac_is_primary_client(mcip) &&
2438 		    (mcip->mci_flags & MAC_CLIENT_FLAGS_PASSIVE_PRIMARY) != 0) {
2439 			return (mcip);
2440 		}
2441 	}
2442 	return (NULL);
2443 }
2444 
2445 /*
2446  * Add a new unicast address to the MAC client.
2447  *
2448  * The MAC address can be specified either by value, or the MAC client
2449  * can specify that it wants to use the primary MAC address of the
2450  * underlying MAC. See the introductory comments at the beginning
2451  * of this file for more more information on primary MAC addresses.
2452  *
2453  * Note also the tuple (MAC address, VID) must be unique
2454  * for the MAC clients defined on top of the same underlying MAC
2455  * instance, unless the MAC_UNICAST_NODUPCHECK is specified.
2456  *
2457  * In no case can a client use the PVID for the MAC, if the MAC has one set.
2458  */
2459 int
2460 i_mac_unicast_add(mac_client_handle_t mch, uint8_t *mac_addr, uint16_t flags,
2461     mac_unicast_handle_t *mah, uint16_t vid, mac_diag_t *diag)
2462 {
2463 	mac_client_impl_t	*mcip = (mac_client_impl_t *)mch;
2464 	mac_impl_t		*mip = mcip->mci_mip;
2465 	int			err;
2466 	uint_t			mac_len = mip->mi_type->mt_addr_length;
2467 	boolean_t		check_dups = !(flags & MAC_UNICAST_NODUPCHECK);
2468 	boolean_t		fastpath_disabled = B_FALSE;
2469 	boolean_t		is_primary = (flags & MAC_UNICAST_PRIMARY);
2470 	boolean_t		is_unicast_hw = (flags & MAC_UNICAST_HW);
2471 	mac_resource_props_t	*mrp;
2472 	boolean_t		passive_client = B_FALSE;
2473 	mac_unicast_impl_t	*muip;
2474 	boolean_t		is_vnic_primary =
2475 	    (flags & MAC_UNICAST_VNIC_PRIMARY);
2476 
2477 	/* when VID is non-zero, the underlying MAC can not be VNIC */
2478 	ASSERT(!((mip->mi_state_flags & MIS_IS_VNIC) && (vid != 0)));
2479 
2480 	/*
2481 	 * Can't unicast add if the client asked only for minimal datapath
2482 	 * setup.
2483 	 */
2484 	if (mcip->mci_state_flags & MCIS_NO_UNICAST_ADDR)
2485 		return (ENOTSUP);
2486 
2487 	/*
2488 	 * Check for an attempted use of the current Port VLAN ID, if enabled.
2489 	 * No client may use it.
2490 	 */
2491 	if (mip->mi_pvid != 0 && vid == mip->mi_pvid)
2492 		return (EBUSY);
2493 
2494 	/*
2495 	 * Check whether it's the primary client and flag it.
2496 	 */
2497 	if (!(mcip->mci_state_flags & MCIS_IS_VNIC) && is_primary && vid == 0)
2498 		mcip->mci_flags |= MAC_CLIENT_FLAGS_PRIMARY;
2499 
2500 	/*
2501 	 * is_vnic_primary is true when we come here as a VLAN VNIC
2502 	 * which uses the primary mac client's address but with a non-zero
2503 	 * VID. In this case the MAC address is not specified by an upper
2504 	 * MAC client.
2505 	 */
2506 	if ((mcip->mci_state_flags & MCIS_IS_VNIC) && is_primary &&
2507 	    !is_vnic_primary) {
2508 		/*
2509 		 * The address is being set by the upper MAC client
2510 		 * of a VNIC. The MAC address was already set by the
2511 		 * VNIC driver during VNIC creation.
2512 		 *
2513 		 * Note: a VNIC has only one MAC address. We return
2514 		 * the MAC unicast address handle of the lower MAC client
2515 		 * corresponding to the VNIC. We allocate a new entry
2516 		 * which is flagged appropriately, so that mac_unicast_remove()
2517 		 * doesn't attempt to free the original entry that
2518 		 * was allocated by the VNIC driver.
2519 		 */
2520 		ASSERT(mcip->mci_unicast != NULL);
2521 
2522 		/* Check for VLAN flags, if present */
2523 		if ((flags & MAC_UNICAST_TAG_DISABLE) != 0)
2524 			mcip->mci_state_flags |= MCIS_TAG_DISABLE;
2525 
2526 		if ((flags & MAC_UNICAST_STRIP_DISABLE) != 0)
2527 			mcip->mci_state_flags |= MCIS_STRIP_DISABLE;
2528 
2529 		if ((flags & MAC_UNICAST_DISABLE_TX_VID_CHECK) != 0)
2530 			mcip->mci_state_flags |= MCIS_DISABLE_TX_VID_CHECK;
2531 
2532 		/*
2533 		 * Ensure that the primary unicast address of the VNIC
2534 		 * is added only once unless we have the
2535 		 * MAC_CLIENT_FLAGS_MULTI_PRIMARY set (and this is not
2536 		 * a passive MAC client).
2537 		 */
2538 		if ((mcip->mci_flags & MAC_CLIENT_FLAGS_VNIC_PRIMARY) != 0) {
2539 			if ((mcip->mci_flags &
2540 			    MAC_CLIENT_FLAGS_MULTI_PRIMARY) == 0 ||
2541 			    (mcip->mci_flags &
2542 			    MAC_CLIENT_FLAGS_PASSIVE_PRIMARY) != 0) {
2543 				return (EBUSY);
2544 			}
2545 			mcip->mci_flags |= MAC_CLIENT_FLAGS_PASSIVE_PRIMARY;
2546 			passive_client = B_TRUE;
2547 		}
2548 
2549 		mcip->mci_flags |= MAC_CLIENT_FLAGS_VNIC_PRIMARY;
2550 
2551 		/*
2552 		 * Create a handle for vid 0.
2553 		 */
2554 		ASSERT(vid == 0);
2555 		muip = kmem_zalloc(sizeof (mac_unicast_impl_t), KM_SLEEP);
2556 		muip->mui_vid = vid;
2557 		*mah = (mac_unicast_handle_t)muip;
2558 		/*
2559 		 * This will be used by the caller to defer setting the
2560 		 * rx functions.
2561 		 */
2562 		if (passive_client)
2563 			return (EAGAIN);
2564 		return (0);
2565 	}
2566 
2567 	/* primary MAC clients cannot be opened on top of anchor VNICs */
2568 	if ((is_vnic_primary || is_primary) &&
2569 	    i_mac_capab_get((mac_handle_t)mip, MAC_CAPAB_ANCHOR_VNIC, NULL)) {
2570 		return (ENXIO);
2571 	}
2572 
2573 	/*
2574 	 * If this is a VNIC/VLAN, disable softmac fast-path.
2575 	 */
2576 	if (mcip->mci_state_flags & MCIS_IS_VNIC) {
2577 		err = mac_fastpath_disable((mac_handle_t)mip);
2578 		if (err != 0)
2579 			return (err);
2580 		fastpath_disabled = B_TRUE;
2581 	}
2582 
2583 	/*
2584 	 * Return EBUSY if:
2585 	 *  - there is an exclusively active mac client exists.
2586 	 *  - this is an exclusive active mac client but
2587 	 *	a. there is already active mac clients exist, or
2588 	 *	b. fastpath streams are already plumbed on this legacy device
2589 	 *  - the mac creator has disallowed active mac clients.
2590 	 */
2591 	if (mip->mi_state_flags & (MIS_EXCLUSIVE|MIS_NO_ACTIVE)) {
2592 		if (fastpath_disabled)
2593 			mac_fastpath_enable((mac_handle_t)mip);
2594 		return (EBUSY);
2595 	}
2596 
2597 	if (mcip->mci_state_flags & MCIS_EXCLUSIVE) {
2598 		ASSERT(!fastpath_disabled);
2599 		if (mip->mi_nactiveclients != 0)
2600 			return (EBUSY);
2601 
2602 		if ((mip->mi_state_flags & MIS_LEGACY) &&
2603 		    !(mip->mi_capab_legacy.ml_active_set(mip->mi_driver))) {
2604 			return (EBUSY);
2605 		}
2606 		mip->mi_state_flags |= MIS_EXCLUSIVE;
2607 	}
2608 
2609 	mrp = kmem_zalloc(sizeof (*mrp), KM_SLEEP);
2610 	if (is_primary && !(mcip->mci_state_flags & (MCIS_IS_VNIC |
2611 	    MCIS_IS_AGGR_PORT))) {
2612 		/*
2613 		 * Apply the property cached in the mac_impl_t to the primary
2614 		 * mac client. If the mac client is a VNIC or an aggregation
2615 		 * port, its property should be set in the mcip when the
2616 		 * VNIC/aggr was created.
2617 		 */
2618 		mac_get_resources((mac_handle_t)mip, mrp);
2619 		(void) mac_client_set_resources(mch, mrp);
2620 	} else if (mcip->mci_state_flags & MCIS_IS_VNIC) {
2621 		/*
2622 		 * This is a primary VLAN client, we don't support
2623 		 * specifying rings property for this as it inherits the
2624 		 * rings property from its MAC.
2625 		 */
2626 		if (is_vnic_primary) {
2627 			mac_resource_props_t	*vmrp;
2628 
2629 			vmrp = MCIP_RESOURCE_PROPS(mcip);
2630 			if (vmrp->mrp_mask & MRP_RX_RINGS ||
2631 			    vmrp->mrp_mask & MRP_TX_RINGS) {
2632 				if (fastpath_disabled)
2633 					mac_fastpath_enable((mac_handle_t)mip);
2634 				kmem_free(mrp, sizeof (*mrp));
2635 				return (ENOTSUP);
2636 			}
2637 			/*
2638 			 * Additionally we also need to inherit any
2639 			 * rings property from the MAC.
2640 			 */
2641 			mac_get_resources((mac_handle_t)mip, mrp);
2642 			if (mrp->mrp_mask & MRP_RX_RINGS) {
2643 				vmrp->mrp_mask |= MRP_RX_RINGS;
2644 				vmrp->mrp_nrxrings = mrp->mrp_nrxrings;
2645 			}
2646 			if (mrp->mrp_mask & MRP_TX_RINGS) {
2647 				vmrp->mrp_mask |= MRP_TX_RINGS;
2648 				vmrp->mrp_ntxrings = mrp->mrp_ntxrings;
2649 			}
2650 		}
2651 		bcopy(MCIP_RESOURCE_PROPS(mcip), mrp, sizeof (*mrp));
2652 	}
2653 
2654 	muip = kmem_zalloc(sizeof (mac_unicast_impl_t), KM_SLEEP);
2655 	muip->mui_vid = vid;
2656 
2657 	if (is_primary || is_vnic_primary) {
2658 		mac_addr = mip->mi_addr;
2659 	} else {
2660 
2661 		/*
2662 		 * Verify the validity of the specified MAC addresses value.
2663 		 */
2664 		if (!mac_unicst_verify((mac_handle_t)mip, mac_addr, mac_len)) {
2665 			*diag = MAC_DIAG_MACADDR_INVALID;
2666 			err = EINVAL;
2667 			goto bail_out;
2668 		}
2669 
2670 		/*
2671 		 * Make sure that the specified MAC address is different
2672 		 * than the unicast MAC address of the underlying NIC.
2673 		 */
2674 		if (check_dups && bcmp(mip->mi_addr, mac_addr, mac_len) == 0) {
2675 			*diag = MAC_DIAG_MACADDR_NIC;
2676 			err = EINVAL;
2677 			goto bail_out;
2678 		}
2679 	}
2680 
2681 	/*
2682 	 * Set the flags here so that if this is a passive client, we
2683 	 * can return  and set it when we call mac_client_datapath_setup
2684 	 * when this becomes the active client. If we defer to using these
2685 	 * flags to mac_client_datapath_setup, then for a passive client,
2686 	 * we'd have to store the flags somewhere (probably fe_flags)
2687 	 * and then use it.
2688 	 */
2689 	if (!MCIP_DATAPATH_SETUP(mcip)) {
2690 		if (is_unicast_hw) {
2691 			/*
2692 			 * The client requires a hardware MAC address slot
2693 			 * for that unicast address. Since we support only
2694 			 * one unicast MAC address per client, flag the
2695 			 * MAC client itself.
2696 			 */
2697 			mcip->mci_state_flags |= MCIS_UNICAST_HW;
2698 		}
2699 
2700 		/* Check for VLAN flags, if present */
2701 		if ((flags & MAC_UNICAST_TAG_DISABLE) != 0)
2702 			mcip->mci_state_flags |= MCIS_TAG_DISABLE;
2703 
2704 		if ((flags & MAC_UNICAST_STRIP_DISABLE) != 0)
2705 			mcip->mci_state_flags |= MCIS_STRIP_DISABLE;
2706 
2707 		if ((flags & MAC_UNICAST_DISABLE_TX_VID_CHECK) != 0)
2708 			mcip->mci_state_flags |= MCIS_DISABLE_TX_VID_CHECK;
2709 	} else {
2710 		/*
2711 		 * Assert that the specified flags are consistent with the
2712 		 * flags specified by previous calls to mac_unicast_add().
2713 		 */
2714 		ASSERT(((flags & MAC_UNICAST_TAG_DISABLE) != 0 &&
2715 		    (mcip->mci_state_flags & MCIS_TAG_DISABLE) != 0) ||
2716 		    ((flags & MAC_UNICAST_TAG_DISABLE) == 0 &&
2717 		    (mcip->mci_state_flags & MCIS_TAG_DISABLE) == 0));
2718 
2719 		ASSERT(((flags & MAC_UNICAST_STRIP_DISABLE) != 0 &&
2720 		    (mcip->mci_state_flags & MCIS_STRIP_DISABLE) != 0) ||
2721 		    ((flags & MAC_UNICAST_STRIP_DISABLE) == 0 &&
2722 		    (mcip->mci_state_flags & MCIS_STRIP_DISABLE) == 0));
2723 
2724 		ASSERT(((flags & MAC_UNICAST_DISABLE_TX_VID_CHECK) != 0 &&
2725 		    (mcip->mci_state_flags & MCIS_DISABLE_TX_VID_CHECK) != 0) ||
2726 		    ((flags & MAC_UNICAST_DISABLE_TX_VID_CHECK) == 0 &&
2727 		    (mcip->mci_state_flags & MCIS_DISABLE_TX_VID_CHECK) == 0));
2728 
2729 		/*
2730 		 * Make sure the client is consistent about its requests
2731 		 * for MAC addresses. I.e. all requests from the clients
2732 		 * must have the MAC_UNICAST_HW flag set or clear.
2733 		 */
2734 		if ((mcip->mci_state_flags & MCIS_UNICAST_HW) != 0 &&
2735 		    !is_unicast_hw ||
2736 		    (mcip->mci_state_flags & MCIS_UNICAST_HW) == 0 &&
2737 		    is_unicast_hw) {
2738 			err = EINVAL;
2739 			goto bail_out;
2740 		}
2741 	}
2742 	/*
2743 	 * Make sure the MAC address is not already used by
2744 	 * another MAC client defined on top of the same
2745 	 * underlying NIC. Unless we have MAC_CLIENT_FLAGS_MULTI_PRIMARY
2746 	 * set when we allow a passive client to be present which will
2747 	 * be activated when the currently active client goes away - this
2748 	 * works only with primary addresses.
2749 	 */
2750 	if ((check_dups || is_primary || is_vnic_primary) &&
2751 	    mac_addr_in_use(mip, mac_addr, vid)) {
2752 		/*
2753 		 * Must have set the multiple primary address flag when
2754 		 * we did a mac_client_open AND this should be a primary
2755 		 * MAC client AND there should not already be a passive
2756 		 * primary. If all is true then we let this succeed
2757 		 * even if the address is a dup.
2758 		 */
2759 		if ((mcip->mci_flags & MAC_CLIENT_FLAGS_MULTI_PRIMARY) == 0 ||
2760 		    (mcip->mci_flags & MAC_CLIENT_FLAGS_PRIMARY) == 0 ||
2761 		    mac_get_passive_primary_client(mip) != NULL) {
2762 			*diag = MAC_DIAG_MACADDR_INUSE;
2763 			err = EEXIST;
2764 			goto bail_out;
2765 		}
2766 		ASSERT((mcip->mci_flags &
2767 		    MAC_CLIENT_FLAGS_PASSIVE_PRIMARY) == 0);
2768 		mcip->mci_flags |= MAC_CLIENT_FLAGS_PASSIVE_PRIMARY;
2769 		kmem_free(mrp, sizeof (*mrp));
2770 
2771 		/*
2772 		 * Stash the unicast address handle, we will use it when
2773 		 * we set up the passive client.
2774 		 */
2775 		mcip->mci_p_unicast_list = muip;
2776 		*mah = (mac_unicast_handle_t)muip;
2777 		return (0);
2778 	}
2779 
2780 	err = mac_client_datapath_setup(mcip, vid, mac_addr, mrp,
2781 	    is_primary || is_vnic_primary, muip);
2782 	if (err != 0)
2783 		goto bail_out;
2784 
2785 	kmem_free(mrp, sizeof (*mrp));
2786 	*mah = (mac_unicast_handle_t)muip;
2787 	return (0);
2788 
2789 bail_out:
2790 	if (fastpath_disabled)
2791 		mac_fastpath_enable((mac_handle_t)mip);
2792 	if (mcip->mci_state_flags & MCIS_EXCLUSIVE) {
2793 		mip->mi_state_flags &= ~MIS_EXCLUSIVE;
2794 		if (mip->mi_state_flags & MIS_LEGACY) {
2795 			mip->mi_capab_legacy.ml_active_clear(
2796 			    mip->mi_driver);
2797 		}
2798 	}
2799 	kmem_free(mrp, sizeof (*mrp));
2800 	kmem_free(muip, sizeof (mac_unicast_impl_t));
2801 	return (err);
2802 }
2803 
2804 /*
2805  * Wrapper function to mac_unicast_add when we want to have the same mac
2806  * client open for two instances, one that is currently active and another
2807  * that will become active when the current one is removed. In this case
2808  * mac_unicast_add will return EGAIN and we will save the rx function and
2809  * arg which will be used when we activate the passive client in
2810  * mac_unicast_remove.
2811  */
2812 int
2813 mac_unicast_add_set_rx(mac_client_handle_t mch, uint8_t *mac_addr,
2814     uint16_t flags, mac_unicast_handle_t *mah,  uint16_t vid, mac_diag_t *diag,
2815     mac_rx_t rx_fn, void *arg)
2816 {
2817 	mac_client_impl_t	*mcip = (mac_client_impl_t *)mch;
2818 	uint_t			err;
2819 
2820 	err = mac_unicast_add(mch, mac_addr, flags, mah, vid, diag);
2821 	if (err != 0 && err != EAGAIN)
2822 		return (err);
2823 	if (err == EAGAIN) {
2824 		if (rx_fn != NULL) {
2825 			mcip->mci_rx_p_fn = rx_fn;
2826 			mcip->mci_rx_p_arg = arg;
2827 		}
2828 		return (0);
2829 	}
2830 	if (rx_fn != NULL)
2831 		mac_rx_set(mch, rx_fn, arg);
2832 	return (err);
2833 }
2834 
2835 int
2836 mac_unicast_add(mac_client_handle_t mch, uint8_t *mac_addr, uint16_t flags,
2837     mac_unicast_handle_t *mah, uint16_t vid, mac_diag_t *diag)
2838 {
2839 	mac_impl_t *mip = ((mac_client_impl_t *)mch)->mci_mip;
2840 	uint_t err;
2841 
2842 	i_mac_perim_enter(mip);
2843 	err = i_mac_unicast_add(mch, mac_addr, flags, mah, vid, diag);
2844 	i_mac_perim_exit(mip);
2845 
2846 	return (err);
2847 }
2848 
2849 static void
2850 mac_client_datapath_teardown(mac_client_handle_t mch, mac_unicast_impl_t *muip,
2851     flow_entry_t *flent)
2852 {
2853 	mac_client_impl_t	*mcip = (mac_client_impl_t *)mch;
2854 	mac_impl_t		*mip = mcip->mci_mip;
2855 	boolean_t		no_unicast;
2856 
2857 	/*
2858 	 * If we have not added a unicast address for this MAC client, just
2859 	 * teardown the datapath.
2860 	 */
2861 	no_unicast = mcip->mci_state_flags & MCIS_NO_UNICAST_ADDR;
2862 
2863 	if (!no_unicast) {
2864 		/*
2865 		 * We would have initialized subflows etc. only if we brought
2866 		 * up the primary client and set the unicast unicast address
2867 		 * etc. Deactivate the flows. The flow entry will be removed
2868 		 * from the active flow tables, and the associated SRS,
2869 		 * softrings etc will be deleted. But the flow entry itself
2870 		 * won't be destroyed, instead it will continue to be archived
2871 		 * off the  the global flow hash list, for a possible future
2872 		 * activation when say IP is plumbed again.
2873 		 */
2874 		mac_link_release_flows(mch);
2875 	}
2876 	mip->mi_nactiveclients--;
2877 	mac_update_single_active_client(mip);
2878 
2879 	/* Tear down the data path */
2880 	mac_datapath_teardown(mcip, mcip->mci_flent, SRST_LINK);
2881 
2882 	/*
2883 	 * Prevent any future access to the flow entry through the mci_flent
2884 	 * pointer by setting the mci_flent to NULL. Access to mci_flent in
2885 	 * mac_bcast_send is also under mi_rw_lock.
2886 	 */
2887 	rw_enter(&mip->mi_rw_lock, RW_WRITER);
2888 	flent = mcip->mci_flent;
2889 	mac_client_remove_flow_from_list(mcip, flent);
2890 
2891 	if (mcip->mci_state_flags & MCIS_DESC_LOGGED)
2892 		mcip->mci_state_flags &= ~MCIS_DESC_LOGGED;
2893 
2894 	/*
2895 	 * This is the last unicast address being removed and there shouldn't
2896 	 * be any outbound data threads at this point coming down from mac
2897 	 * clients. We have waited for the data threads to finish before
2898 	 * starting dld_str_detach. Non-data threads must access TX SRS
2899 	 * under mi_rw_lock.
2900 	 */
2901 	rw_exit(&mip->mi_rw_lock);
2902 
2903 	/*
2904 	 * Don't use FLOW_MARK with FE_MC_NO_DATAPATH, as the flow might
2905 	 * contain other flags, such as FE_CONDEMNED, which we need to
2906 	 * cleared. We don't call mac_flow_cleanup() for this unicast
2907 	 * flow as we have a already cleaned up SRSs etc. (via the teadown
2908 	 * path). We just clear the stats and reset the initial callback
2909 	 * function, the rest will be set when we call mac_flow_create,
2910 	 * if at all.
2911 	 */
2912 	mutex_enter(&flent->fe_lock);
2913 	ASSERT(flent->fe_refcnt == 1 && flent->fe_mbg == NULL &&
2914 	    flent->fe_tx_srs == NULL && flent->fe_rx_srs_cnt == 0);
2915 	flent->fe_flags = FE_MC_NO_DATAPATH;
2916 	flow_stat_destroy(flent);
2917 	mac_misc_stat_delete(flent);
2918 
2919 	/* Initialize the receiver function to a safe routine */
2920 	flent->fe_cb_fn = (flow_fn_t)mac_pkt_drop;
2921 	flent->fe_cb_arg1 = NULL;
2922 	flent->fe_cb_arg2 = NULL;
2923 
2924 	flent->fe_index = -1;
2925 	mutex_exit(&flent->fe_lock);
2926 
2927 	if (mip->mi_type->mt_brdcst_addr != NULL) {
2928 		ASSERT(muip != NULL || no_unicast);
2929 		mac_bcast_delete(mcip, mip->mi_type->mt_brdcst_addr,
2930 		    muip != NULL ? muip->mui_vid : VLAN_ID_NONE);
2931 	}
2932 
2933 	if (mip->mi_nactiveclients == 1) {
2934 		mac_capab_update((mac_handle_t)mip);
2935 		mac_virtual_link_update(mip);
2936 	}
2937 
2938 	if (mcip->mci_state_flags & MCIS_EXCLUSIVE) {
2939 		mip->mi_state_flags &= ~MIS_EXCLUSIVE;
2940 
2941 		if (mip->mi_state_flags & MIS_LEGACY)
2942 			mip->mi_capab_legacy.ml_active_clear(mip->mi_driver);
2943 	}
2944 
2945 	mcip->mci_state_flags &= ~MCIS_UNICAST_HW;
2946 
2947 	if (mcip->mci_state_flags & MCIS_TAG_DISABLE)
2948 		mcip->mci_state_flags &= ~MCIS_TAG_DISABLE;
2949 
2950 	if (mcip->mci_state_flags & MCIS_STRIP_DISABLE)
2951 		mcip->mci_state_flags &= ~MCIS_STRIP_DISABLE;
2952 
2953 	if (mcip->mci_state_flags & MCIS_DISABLE_TX_VID_CHECK)
2954 		mcip->mci_state_flags &= ~MCIS_DISABLE_TX_VID_CHECK;
2955 
2956 	if (muip != NULL)
2957 		kmem_free(muip, sizeof (mac_unicast_impl_t));
2958 	mac_protect_cancel_timer(mcip);
2959 	mac_protect_flush_dhcp(mcip);
2960 
2961 	bzero(&mcip->mci_misc_stat, sizeof (mcip->mci_misc_stat));
2962 	/*
2963 	 * Disable fastpath if this is a VNIC or a VLAN.
2964 	 */
2965 	if (mcip->mci_state_flags & MCIS_IS_VNIC)
2966 		mac_fastpath_enable((mac_handle_t)mip);
2967 	mac_stop((mac_handle_t)mip);
2968 }
2969 
2970 /*
2971  * Remove a MAC address which was previously added by mac_unicast_add().
2972  */
2973 int
2974 mac_unicast_remove(mac_client_handle_t mch, mac_unicast_handle_t mah)
2975 {
2976 	mac_client_impl_t *mcip = (mac_client_impl_t *)mch;
2977 	mac_unicast_impl_t *muip = (mac_unicast_impl_t *)mah;
2978 	mac_unicast_impl_t *pre;
2979 	mac_impl_t *mip = mcip->mci_mip;
2980 	flow_entry_t		*flent;
2981 	uint16_t mui_vid;
2982 
2983 	i_mac_perim_enter(mip);
2984 	if (mcip->mci_flags & MAC_CLIENT_FLAGS_VNIC_PRIMARY) {
2985 		/*
2986 		 * Called made by the upper MAC client of a VNIC.
2987 		 * There's nothing much to do, the unicast address will
2988 		 * be removed by the VNIC driver when the VNIC is deleted,
2989 		 * but let's ensure that all our transmit is done before
2990 		 * the client does a mac_client_stop lest it trigger an
2991 		 * assert in the driver.
2992 		 */
2993 		ASSERT(muip->mui_vid == 0);
2994 
2995 		mac_tx_client_flush(mcip);
2996 
2997 		if ((mcip->mci_flags & MAC_CLIENT_FLAGS_PASSIVE_PRIMARY) != 0) {
2998 			mcip->mci_flags &= ~MAC_CLIENT_FLAGS_PASSIVE_PRIMARY;
2999 			if (mcip->mci_rx_p_fn != NULL) {
3000 				mac_rx_set(mch, mcip->mci_rx_p_fn,
3001 				    mcip->mci_rx_p_arg);
3002 				mcip->mci_rx_p_fn = NULL;
3003 				mcip->mci_rx_p_arg = NULL;
3004 			}
3005 			kmem_free(muip, sizeof (mac_unicast_impl_t));
3006 			i_mac_perim_exit(mip);
3007 			return (0);
3008 		}
3009 		mcip->mci_flags &= ~MAC_CLIENT_FLAGS_VNIC_PRIMARY;
3010 
3011 		if (mcip->mci_state_flags & MCIS_TAG_DISABLE)
3012 			mcip->mci_state_flags &= ~MCIS_TAG_DISABLE;
3013 
3014 		if (mcip->mci_state_flags & MCIS_STRIP_DISABLE)
3015 			mcip->mci_state_flags &= ~MCIS_STRIP_DISABLE;
3016 
3017 		if (mcip->mci_state_flags & MCIS_DISABLE_TX_VID_CHECK)
3018 			mcip->mci_state_flags &= ~MCIS_DISABLE_TX_VID_CHECK;
3019 
3020 		kmem_free(muip, sizeof (mac_unicast_impl_t));
3021 		i_mac_perim_exit(mip);
3022 		return (0);
3023 	}
3024 
3025 	ASSERT(muip != NULL);
3026 
3027 	/*
3028 	 * We are removing a passive client, we haven't setup the datapath
3029 	 * for this yet, so nothing much to do.
3030 	 */
3031 	if ((mcip->mci_flags & MAC_CLIENT_FLAGS_PASSIVE_PRIMARY) != 0) {
3032 
3033 		ASSERT((mcip->mci_flent->fe_flags & FE_MC_NO_DATAPATH) != 0);
3034 		ASSERT(mcip->mci_p_unicast_list == muip);
3035 
3036 		mcip->mci_flags &= ~MAC_CLIENT_FLAGS_PASSIVE_PRIMARY;
3037 
3038 		mcip->mci_p_unicast_list = NULL;
3039 		mcip->mci_rx_p_fn = NULL;
3040 		mcip->mci_rx_p_arg = NULL;
3041 
3042 		mcip->mci_state_flags &= ~MCIS_UNICAST_HW;
3043 
3044 		if (mcip->mci_state_flags & MCIS_TAG_DISABLE)
3045 			mcip->mci_state_flags &= ~MCIS_TAG_DISABLE;
3046 
3047 		if (mcip->mci_state_flags & MCIS_STRIP_DISABLE)
3048 			mcip->mci_state_flags &= ~MCIS_STRIP_DISABLE;
3049 
3050 		if (mcip->mci_state_flags & MCIS_DISABLE_TX_VID_CHECK)
3051 			mcip->mci_state_flags &= ~MCIS_DISABLE_TX_VID_CHECK;
3052 
3053 		kmem_free(muip, sizeof (mac_unicast_impl_t));
3054 		i_mac_perim_exit(mip);
3055 		return (0);
3056 	}
3057 	/*
3058 	 * Remove the VID from the list of client's VIDs.
3059 	 */
3060 	pre = mcip->mci_unicast_list;
3061 	if (muip == pre) {
3062 		mcip->mci_unicast_list = muip->mui_next;
3063 	} else {
3064 		while ((pre->mui_next != NULL) && (pre->mui_next != muip))
3065 			pre = pre->mui_next;
3066 		ASSERT(pre->mui_next == muip);
3067 		rw_enter(&mcip->mci_rw_lock, RW_WRITER);
3068 		pre->mui_next = muip->mui_next;
3069 		rw_exit(&mcip->mci_rw_lock);
3070 	}
3071 
3072 	if (!mac_client_single_rcvr(mcip)) {
3073 		/*
3074 		 * This MAC client is shared by more than one unicast
3075 		 * addresses, so we will just remove the flent
3076 		 * corresponding to the address being removed. We don't invoke
3077 		 * mac_rx_classify_flow_rem() since the additional flow is
3078 		 * not associated with its own separate set of SRS and rings,
3079 		 * and these constructs are still needed for the remaining
3080 		 * flows.
3081 		 */
3082 		flent = mac_client_get_flow(mcip, muip);
3083 		ASSERT(flent != NULL);
3084 
3085 		/*
3086 		 * The first one is disappearing, need to make sure
3087 		 * we replace it with another from the list of
3088 		 * shared clients.
3089 		 */
3090 		if (flent == mcip->mci_flent)
3091 			flent = mac_client_swap_mciflent(mcip);
3092 		mac_client_remove_flow_from_list(mcip, flent);
3093 		mac_flow_remove(mip->mi_flow_tab, flent, B_FALSE);
3094 		mac_flow_wait(flent, FLOW_DRIVER_UPCALL);
3095 
3096 		/*
3097 		 * The multicast groups that were added by the client so
3098 		 * far must be removed from the brodcast domain corresponding
3099 		 * to the VID being removed.
3100 		 */
3101 		mac_client_bcast_refresh(mcip, mac_client_update_mcast,
3102 		    (void *)flent, B_FALSE);
3103 
3104 		if (mip->mi_type->mt_brdcst_addr != NULL) {
3105 			mac_bcast_delete(mcip, mip->mi_type->mt_brdcst_addr,
3106 			    muip->mui_vid);
3107 		}
3108 
3109 		FLOW_FINAL_REFRELE(flent);
3110 		ASSERT(!(mcip->mci_state_flags & MCIS_EXCLUSIVE));
3111 		/*
3112 		 * Enable fastpath if this is a VNIC or a VLAN.
3113 		 */
3114 		if (mcip->mci_state_flags & MCIS_IS_VNIC)
3115 			mac_fastpath_enable((mac_handle_t)mip);
3116 		mac_stop((mac_handle_t)mip);
3117 		i_mac_perim_exit(mip);
3118 		return (0);
3119 	}
3120 
3121 	mui_vid = muip->mui_vid;
3122 	mac_client_datapath_teardown(mch, muip, flent);
3123 
3124 	if ((mcip->mci_flags & MAC_CLIENT_FLAGS_PRIMARY) && mui_vid == 0) {
3125 		mcip->mci_flags &= ~MAC_CLIENT_FLAGS_PRIMARY;
3126 	} else {
3127 		i_mac_perim_exit(mip);
3128 		return (0);
3129 	}
3130 
3131 	/*
3132 	 * If we are removing the primary, check if we have a passive primary
3133 	 * client that we need to activate now.
3134 	 */
3135 	mcip = mac_get_passive_primary_client(mip);
3136 	if (mcip != NULL) {
3137 		mac_resource_props_t	*mrp;
3138 		mac_unicast_impl_t	*muip;
3139 
3140 		mcip->mci_flags &= ~MAC_CLIENT_FLAGS_PASSIVE_PRIMARY;
3141 		mrp = kmem_zalloc(sizeof (*mrp), KM_SLEEP);
3142 
3143 		/*
3144 		 * Apply the property cached in the mac_impl_t to the
3145 		 * primary mac client.
3146 		 */
3147 		mac_get_resources((mac_handle_t)mip, mrp);
3148 		(void) mac_client_set_resources(mch, mrp);
3149 		ASSERT(mcip->mci_p_unicast_list != NULL);
3150 		muip = mcip->mci_p_unicast_list;
3151 		mcip->mci_p_unicast_list = NULL;
3152 		if (mac_client_datapath_setup(mcip, VLAN_ID_NONE,
3153 		    mip->mi_addr, mrp, B_TRUE, muip) == 0) {
3154 			if (mcip->mci_rx_p_fn != NULL) {
3155 				mac_rx_set(mch, mcip->mci_rx_p_fn,
3156 				    mcip->mci_rx_p_arg);
3157 				mcip->mci_rx_p_fn = NULL;
3158 				mcip->mci_rx_p_arg = NULL;
3159 			}
3160 		} else {
3161 			kmem_free(muip, sizeof (mac_unicast_impl_t));
3162 		}
3163 		kmem_free(mrp, sizeof (*mrp));
3164 	}
3165 	i_mac_perim_exit(mip);
3166 	return (0);
3167 }
3168 
3169 /*
3170  * Multicast add function invoked by MAC clients.
3171  */
3172 int
3173 mac_multicast_add(mac_client_handle_t mch, const uint8_t *addr)
3174 {
3175 	mac_client_impl_t	*mcip = (mac_client_impl_t *)mch;
3176 	mac_impl_t		*mip = mcip->mci_mip;
3177 	flow_entry_t		*flent = mcip->mci_flent_list;
3178 	flow_entry_t		*prev_fe = NULL;
3179 	uint16_t		vid;
3180 	int			err = 0;
3181 
3182 	/* Verify the address is a valid multicast address */
3183 	if ((err = mip->mi_type->mt_ops.mtops_multicst_verify(addr,
3184 	    mip->mi_pdata)) != 0)
3185 		return (err);
3186 
3187 	i_mac_perim_enter(mip);
3188 	while (flent != NULL) {
3189 		vid = i_mac_flow_vid(flent);
3190 
3191 		err = mac_bcast_add((mac_client_impl_t *)mch, addr, vid,
3192 		    MAC_ADDRTYPE_MULTICAST);
3193 		if (err != 0)
3194 			break;
3195 		prev_fe = flent;
3196 		flent = flent->fe_client_next;
3197 	}
3198 
3199 	/*
3200 	 * If we failed adding, then undo all, rather than partial
3201 	 * success.
3202 	 */
3203 	if (flent != NULL && prev_fe != NULL) {
3204 		flent = mcip->mci_flent_list;
3205 		while (flent != prev_fe->fe_client_next) {
3206 			vid = i_mac_flow_vid(flent);
3207 			mac_bcast_delete((mac_client_impl_t *)mch, addr, vid);
3208 			flent = flent->fe_client_next;
3209 		}
3210 	}
3211 	i_mac_perim_exit(mip);
3212 	return (err);
3213 }
3214 
3215 /*
3216  * Multicast delete function invoked by MAC clients.
3217  */
3218 void
3219 mac_multicast_remove(mac_client_handle_t mch, const uint8_t *addr)
3220 {
3221 	mac_client_impl_t	*mcip = (mac_client_impl_t *)mch;
3222 	mac_impl_t		*mip = mcip->mci_mip;
3223 	flow_entry_t		*flent;
3224 	uint16_t		vid;
3225 
3226 	i_mac_perim_enter(mip);
3227 	for (flent = mcip->mci_flent_list; flent != NULL;
3228 	    flent = flent->fe_client_next) {
3229 		vid = i_mac_flow_vid(flent);
3230 		mac_bcast_delete((mac_client_impl_t *)mch, addr, vid);
3231 	}
3232 	i_mac_perim_exit(mip);
3233 }
3234 
3235 /*
3236  * When a MAC client desires to capture packets on an interface,
3237  * it registers a promiscuous call back with mac_promisc_add().
3238  * There are three types of promiscuous callbacks:
3239  *
3240  * * MAC_CLIENT_PROMISC_ALL
3241  *   Captures all packets sent and received by the MAC client,
3242  *   the physical interface, as well as all other MAC clients
3243  *   defined on top of the same MAC.
3244  *
3245  * * MAC_CLIENT_PROMISC_FILTERED
3246  *   Captures all packets sent and received by the MAC client,
3247  *   plus all multicast traffic sent and received by the phyisical
3248  *   interface and the other MAC clients.
3249  *
3250  * * MAC_CLIENT_PROMISC_MULTI
3251  *   Captures all broadcast and multicast packets sent and
3252  *   received by the MAC clients as well as the physical interface.
3253  *
3254  * In all cases, the underlying MAC is put in promiscuous mode.
3255  */
3256 int
3257 mac_promisc_add(mac_client_handle_t mch, mac_client_promisc_type_t type,
3258     mac_rx_t fn, void *arg, mac_promisc_handle_t *mphp, uint16_t flags)
3259 {
3260 	mac_client_impl_t *mcip = (mac_client_impl_t *)mch;
3261 	mac_impl_t *mip = mcip->mci_mip;
3262 	mac_promisc_impl_t *mpip;
3263 	mac_cb_info_t	*mcbi;
3264 	int rc;
3265 
3266 	i_mac_perim_enter(mip);
3267 
3268 	if ((rc = mac_start((mac_handle_t)mip)) != 0) {
3269 		i_mac_perim_exit(mip);
3270 		return (rc);
3271 	}
3272 
3273 	if ((mcip->mci_state_flags & MCIS_IS_VNIC) &&
3274 	    type == MAC_CLIENT_PROMISC_ALL) {
3275 		/*
3276 		 * The function is being invoked by the upper MAC client
3277 		 * of a VNIC. The VNIC should only see the traffic
3278 		 * it is entitled to.
3279 		 */
3280 		type = MAC_CLIENT_PROMISC_FILTERED;
3281 	}
3282 
3283 
3284 	/*
3285 	 * Turn on promiscuous mode for the underlying NIC.
3286 	 * This is needed even for filtered callbacks which
3287 	 * expect to receive all multicast traffic on the wire.
3288 	 *
3289 	 * Physical promiscuous mode should not be turned on if
3290 	 * MAC_PROMISC_FLAGS_NO_PHYS is set.
3291 	 */
3292 	if ((flags & MAC_PROMISC_FLAGS_NO_PHYS) == 0) {
3293 		if ((rc = i_mac_promisc_set(mip, B_TRUE)) != 0) {
3294 			mac_stop((mac_handle_t)mip);
3295 			i_mac_perim_exit(mip);
3296 			return (rc);
3297 		}
3298 	}
3299 
3300 	mpip = kmem_cache_alloc(mac_promisc_impl_cache, KM_SLEEP);
3301 
3302 	mpip->mpi_type = type;
3303 	mpip->mpi_fn = fn;
3304 	mpip->mpi_arg = arg;
3305 	mpip->mpi_mcip = mcip;
3306 	mpip->mpi_no_tx_loop = ((flags & MAC_PROMISC_FLAGS_NO_TX_LOOP) != 0);
3307 	mpip->mpi_no_phys = ((flags & MAC_PROMISC_FLAGS_NO_PHYS) != 0);
3308 	mpip->mpi_strip_vlan_tag =
3309 	    ((flags & MAC_PROMISC_FLAGS_VLAN_TAG_STRIP) != 0);
3310 	mpip->mpi_no_copy = ((flags & MAC_PROMISC_FLAGS_NO_COPY) != 0);
3311 
3312 	mcbi = &mip->mi_promisc_cb_info;
3313 	mutex_enter(mcbi->mcbi_lockp);
3314 
3315 	mac_callback_add(&mip->mi_promisc_cb_info, &mcip->mci_promisc_list,
3316 	    &mpip->mpi_mci_link);
3317 	mac_callback_add(&mip->mi_promisc_cb_info, &mip->mi_promisc_list,
3318 	    &mpip->mpi_mi_link);
3319 
3320 	mutex_exit(mcbi->mcbi_lockp);
3321 
3322 	*mphp = (mac_promisc_handle_t)mpip;
3323 
3324 	if (mcip->mci_state_flags & MCIS_IS_VNIC) {
3325 		mac_impl_t *umip = mcip->mci_upper_mip;
3326 
3327 		ASSERT(umip != NULL);
3328 		mac_vnic_secondary_update(umip);
3329 	}
3330 
3331 	i_mac_perim_exit(mip);
3332 
3333 	return (0);
3334 }
3335 
3336 /*
3337  * Remove a multicast address previously aded through mac_promisc_add().
3338  */
3339 void
3340 mac_promisc_remove(mac_promisc_handle_t mph)
3341 {
3342 	mac_promisc_impl_t *mpip = (mac_promisc_impl_t *)mph;
3343 	mac_client_impl_t *mcip = mpip->mpi_mcip;
3344 	mac_impl_t *mip = mcip->mci_mip;
3345 	mac_cb_info_t *mcbi;
3346 	int rv;
3347 
3348 	i_mac_perim_enter(mip);
3349 
3350 	/*
3351 	 * Even if the device can't be reset into normal mode, we still
3352 	 * need to clear the client promisc callbacks. The client may want
3353 	 * to close the mac end point and we can't have stale callbacks.
3354 	 */
3355 	if (!(mpip->mpi_no_phys)) {
3356 		if ((rv = i_mac_promisc_set(mip, B_FALSE)) != 0) {
3357 			cmn_err(CE_WARN, "%s: failed to switch OFF promiscuous"
3358 			    " mode because of error 0x%x", mip->mi_name, rv);
3359 		}
3360 	}
3361 	mcbi = &mip->mi_promisc_cb_info;
3362 	mutex_enter(mcbi->mcbi_lockp);
3363 	if (mac_callback_remove(mcbi, &mip->mi_promisc_list,
3364 	    &mpip->mpi_mi_link)) {
3365 		VERIFY(mac_callback_remove(&mip->mi_promisc_cb_info,
3366 		    &mcip->mci_promisc_list, &mpip->mpi_mci_link));
3367 		kmem_cache_free(mac_promisc_impl_cache, mpip);
3368 	} else {
3369 		mac_callback_remove_wait(&mip->mi_promisc_cb_info);
3370 	}
3371 
3372 	if (mcip->mci_state_flags & MCIS_IS_VNIC) {
3373 		mac_impl_t *umip = mcip->mci_upper_mip;
3374 
3375 		ASSERT(umip != NULL);
3376 		mac_vnic_secondary_update(umip);
3377 	}
3378 
3379 	mutex_exit(mcbi->mcbi_lockp);
3380 	mac_stop((mac_handle_t)mip);
3381 
3382 	i_mac_perim_exit(mip);
3383 }
3384 
3385 /*
3386  * Reference count the number of active Tx threads. MCI_TX_QUIESCE indicates
3387  * that a control operation wants to quiesce the Tx data flow in which case
3388  * we return an error. Holding any of the per cpu locks ensures that the
3389  * mci_tx_flag won't change.
3390  *
3391  * 'CPU' must be accessed just once and used to compute the index into the
3392  * percpu array, and that index must be used for the entire duration of the
3393  * packet send operation. Note that the thread may be preempted and run on
3394  * another cpu any time and so we can't use 'CPU' more than once for the
3395  * operation.
3396  */
3397 #define	MAC_TX_TRY_HOLD(mcip, mytx, error)				\
3398 {									\
3399 	(error) = 0;							\
3400 	(mytx) = &(mcip)->mci_tx_pcpu[CPU->cpu_seqid & mac_tx_percpu_cnt]; \
3401 	mutex_enter(&(mytx)->pcpu_tx_lock);				\
3402 	if (!((mcip)->mci_tx_flag & MCI_TX_QUIESCE)) {			\
3403 		(mytx)->pcpu_tx_refcnt++;				\
3404 	} else {							\
3405 		(error) = -1;						\
3406 	}								\
3407 	mutex_exit(&(mytx)->pcpu_tx_lock);				\
3408 }
3409 
3410 /*
3411  * Release the reference. If needed, signal any control operation waiting
3412  * for Tx quiescence. The wait and signal are always done using the
3413  * mci_tx_pcpu[0]'s lock
3414  */
3415 #define	MAC_TX_RELE(mcip, mytx) {					\
3416 	mutex_enter(&(mytx)->pcpu_tx_lock);				\
3417 	if (--(mytx)->pcpu_tx_refcnt == 0 &&				\
3418 	    (mcip)->mci_tx_flag & MCI_TX_QUIESCE) {			\
3419 		mutex_exit(&(mytx)->pcpu_tx_lock);			\
3420 		mutex_enter(&(mcip)->mci_tx_pcpu[0].pcpu_tx_lock);	\
3421 		cv_signal(&(mcip)->mci_tx_cv);				\
3422 		mutex_exit(&(mcip)->mci_tx_pcpu[0].pcpu_tx_lock);	\
3423 	} else {							\
3424 		mutex_exit(&(mytx)->pcpu_tx_lock);			\
3425 	}								\
3426 }
3427 
3428 /*
3429  * Send function invoked by MAC clients.
3430  */
3431 mac_tx_cookie_t
3432 mac_tx(mac_client_handle_t mch, mblk_t *mp_chain, uintptr_t hint,
3433     uint16_t flag, mblk_t **ret_mp)
3434 {
3435 	mac_tx_cookie_t		cookie = NULL;
3436 	int			error;
3437 	mac_tx_percpu_t		*mytx;
3438 	mac_soft_ring_set_t	*srs;
3439 	flow_entry_t		*flent;
3440 	boolean_t		is_subflow = B_FALSE;
3441 	mac_client_impl_t	*mcip = (mac_client_impl_t *)mch;
3442 	mac_impl_t		*mip = mcip->mci_mip;
3443 	mac_srs_tx_t		*srs_tx;
3444 
3445 	/*
3446 	 * Check whether the active Tx threads count is bumped already.
3447 	 */
3448 	if (!(flag & MAC_TX_NO_HOLD)) {
3449 		MAC_TX_TRY_HOLD(mcip, mytx, error);
3450 		if (error != 0) {
3451 			freemsgchain(mp_chain);
3452 			return (NULL);
3453 		}
3454 	}
3455 
3456 	/*
3457 	 * If mac protection is enabled, only the permissible packets will be
3458 	 * returned by mac_protect_check().
3459 	 */
3460 	if ((mcip->mci_flent->
3461 	    fe_resource_props.mrp_mask & MRP_PROTECT) != 0 &&
3462 	    (mp_chain = mac_protect_check(mch, mp_chain)) == NULL)
3463 		goto done;
3464 
3465 	if (mcip->mci_subflow_tab != NULL &&
3466 	    mcip->mci_subflow_tab->ft_flow_count > 0 &&
3467 	    mac_flow_lookup(mcip->mci_subflow_tab, mp_chain,
3468 	    FLOW_OUTBOUND, &flent) == 0) {
3469 		/*
3470 		 * The main assumption here is that if in the event
3471 		 * we get a chain, all the packets will be classified
3472 		 * to the same Flow/SRS. If this changes for any
3473 		 * reason, the following logic should change as well.
3474 		 * I suppose the fanout_hint also assumes this .
3475 		 */
3476 		ASSERT(flent != NULL);
3477 		is_subflow = B_TRUE;
3478 	} else {
3479 		flent = mcip->mci_flent;
3480 	}
3481 
3482 	srs = flent->fe_tx_srs;
3483 	/*
3484 	 * This is to avoid panics with PF_PACKET that can call mac_tx()
3485 	 * against an interface that is not capable of sending. A rewrite
3486 	 * of the mac datapath is required to remove this limitation.
3487 	 */
3488 	if (srs == NULL) {
3489 		freemsgchain(mp_chain);
3490 		goto done;
3491 	}
3492 
3493 	srs_tx = &srs->srs_tx;
3494 	if (srs_tx->st_mode == SRS_TX_DEFAULT &&
3495 	    (srs->srs_state & SRS_ENQUEUED) == 0 &&
3496 	    mip->mi_nactiveclients == 1 && mp_chain->b_next == NULL) {
3497 		uint64_t	obytes;
3498 
3499 		/*
3500 		 * Since dls always opens the underlying MAC, nclients equals
3501 		 * to 1 means that the only active client is dls itself acting
3502 		 * as a primary client of the MAC instance. Since dls will not
3503 		 * send tagged packets in that case, and dls is trusted to send
3504 		 * packets for its allowed VLAN(s), the VLAN tag insertion and
3505 		 * check is required only if nclients is greater than 1.
3506 		 */
3507 		if (mip->mi_nclients > 1) {
3508 			if (MAC_VID_CHECK_NEEDED(mcip)) {
3509 				int	err = 0;
3510 
3511 				MAC_VID_CHECK(mcip, mp_chain, err);
3512 				if (err != 0) {
3513 					freemsg(mp_chain);
3514 					mcip->mci_misc_stat.mms_txerrors++;
3515 					goto done;
3516 				}
3517 			}
3518 			if (MAC_TAG_NEEDED(mcip)) {
3519 				mp_chain = mac_add_vlan_tag(mp_chain, 0,
3520 				    mac_client_vid(mch));
3521 				if (mp_chain == NULL) {
3522 					mcip->mci_misc_stat.mms_txerrors++;
3523 					goto done;
3524 				}
3525 			}
3526 		}
3527 
3528 		obytes = (mp_chain->b_cont == NULL ? MBLKL(mp_chain) :
3529 		    msgdsize(mp_chain));
3530 
3531 		MAC_TX(mip, srs_tx->st_arg2, mp_chain, mcip);
3532 		if (mp_chain == NULL) {
3533 			cookie = NULL;
3534 			SRS_TX_STAT_UPDATE(srs, opackets, 1);
3535 			SRS_TX_STAT_UPDATE(srs, obytes, obytes);
3536 		} else {
3537 			mutex_enter(&srs->srs_lock);
3538 			cookie = mac_tx_srs_no_desc(srs, mp_chain,
3539 			    flag, ret_mp);
3540 			mutex_exit(&srs->srs_lock);
3541 		}
3542 	} else {
3543 		cookie = srs_tx->st_func(srs, mp_chain, hint, flag, ret_mp);
3544 	}
3545 
3546 done:
3547 	if (is_subflow)
3548 		FLOW_REFRELE(flent);
3549 
3550 	if (!(flag & MAC_TX_NO_HOLD))
3551 		MAC_TX_RELE(mcip, mytx);
3552 
3553 	return (cookie);
3554 }
3555 
3556 /*
3557  * mac_tx_is_blocked
3558  *
3559  * Given a cookie, it returns if the ring identified by the cookie is
3560  * flow-controlled or not. If NULL is passed in place of a cookie,
3561  * then it finds out if any of the underlying rings belonging to the
3562  * SRS is flow controlled or not and returns that status.
3563  */
3564 /* ARGSUSED */
3565 boolean_t
3566 mac_tx_is_flow_blocked(mac_client_handle_t mch, mac_tx_cookie_t cookie)
3567 {
3568 	mac_client_impl_t *mcip = (mac_client_impl_t *)mch;
3569 	mac_soft_ring_set_t *mac_srs;
3570 	mac_soft_ring_t *sringp;
3571 	boolean_t blocked = B_FALSE;
3572 	mac_tx_percpu_t *mytx;
3573 	int err;
3574 	int i;
3575 
3576 	/*
3577 	 * Bump the reference count so that mac_srs won't be deleted.
3578 	 * If the client is currently quiesced and we failed to bump
3579 	 * the reference, return B_TRUE so that flow control stays
3580 	 * as enabled.
3581 	 *
3582 	 * Flow control will then be disabled once the client is no
3583 	 * longer quiesced.
3584 	 */
3585 	MAC_TX_TRY_HOLD(mcip, mytx, err);
3586 	if (err != 0)
3587 		return (B_TRUE);
3588 
3589 	if ((mac_srs = MCIP_TX_SRS(mcip)) == NULL) {
3590 		MAC_TX_RELE(mcip, mytx);
3591 		return (B_FALSE);
3592 	}
3593 
3594 	mutex_enter(&mac_srs->srs_lock);
3595 	/*
3596 	 * Only in the case of TX_FANOUT and TX_AGGR, the underlying
3597 	 * softring (s_ring_state) will have the HIWAT set. This is
3598 	 * the multiple Tx ring flow control case. For all other
3599 	 * case, SRS (srs_state) will store the condition.
3600 	 */
3601 	if (mac_srs->srs_tx.st_mode == SRS_TX_FANOUT ||
3602 	    mac_srs->srs_tx.st_mode == SRS_TX_AGGR) {
3603 		if (cookie != NULL) {
3604 			sringp = (mac_soft_ring_t *)cookie;
3605 			mutex_enter(&sringp->s_ring_lock);
3606 			if (sringp->s_ring_state & S_RING_TX_HIWAT)
3607 				blocked = B_TRUE;
3608 			mutex_exit(&sringp->s_ring_lock);
3609 		} else {
3610 			for (i = 0; i < mac_srs->srs_tx_ring_count; i++) {
3611 				sringp = mac_srs->srs_tx_soft_rings[i];
3612 				mutex_enter(&sringp->s_ring_lock);
3613 				if (sringp->s_ring_state & S_RING_TX_HIWAT) {
3614 					blocked = B_TRUE;
3615 					mutex_exit(&sringp->s_ring_lock);
3616 					break;
3617 				}
3618 				mutex_exit(&sringp->s_ring_lock);
3619 			}
3620 		}
3621 	} else {
3622 		blocked = (mac_srs->srs_state & SRS_TX_HIWAT);
3623 	}
3624 	mutex_exit(&mac_srs->srs_lock);
3625 	MAC_TX_RELE(mcip, mytx);
3626 	return (blocked);
3627 }
3628 
3629 /*
3630  * Check if the MAC client is the primary MAC client.
3631  */
3632 boolean_t
3633 mac_is_primary_client(mac_client_impl_t *mcip)
3634 {
3635 	return (mcip->mci_flags & MAC_CLIENT_FLAGS_PRIMARY);
3636 }
3637 
3638 void
3639 mac_ioctl(mac_handle_t mh, queue_t *wq, mblk_t *bp)
3640 {
3641 	mac_impl_t	*mip = (mac_impl_t *)mh;
3642 	int cmd = ((struct iocblk *)bp->b_rptr)->ioc_cmd;
3643 
3644 	if ((cmd == ND_GET && (mip->mi_callbacks->mc_callbacks & MC_GETPROP)) ||
3645 	    (cmd == ND_SET && (mip->mi_callbacks->mc_callbacks & MC_SETPROP))) {
3646 		/*
3647 		 * If ndd props were registered, call them.
3648 		 * Note that ndd ioctls are Obsolete
3649 		 */
3650 		mac_ndd_ioctl(mip, wq, bp);
3651 		return;
3652 	}
3653 
3654 	/*
3655 	 * Call the driver to handle the ioctl.  The driver may not support
3656 	 * any ioctls, in which case we reply with a NAK on its behalf.
3657 	 */
3658 	if (mip->mi_callbacks->mc_callbacks & MC_IOCTL)
3659 		mip->mi_ioctl(mip->mi_driver, wq, bp);
3660 	else
3661 		miocnak(wq, bp, 0, EINVAL);
3662 }
3663 
3664 /*
3665  * Return the link state of the specified MAC instance.
3666  */
3667 link_state_t
3668 mac_link_get(mac_handle_t mh)
3669 {
3670 	return (((mac_impl_t *)mh)->mi_linkstate);
3671 }
3672 
3673 /*
3674  * Add a mac client specified notification callback. Please see the comments
3675  * above mac_callback_add() for general information about mac callback
3676  * addition/deletion in the presence of mac callback list walkers
3677  */
3678 mac_notify_handle_t
3679 mac_notify_add(mac_handle_t mh, mac_notify_t notify_fn, void *arg)
3680 {
3681 	mac_impl_t		*mip = (mac_impl_t *)mh;
3682 	mac_notify_cb_t		*mncb;
3683 	mac_cb_info_t		*mcbi;
3684 
3685 	/*
3686 	 * Allocate a notify callback structure, fill in the details and
3687 	 * use the mac callback list manipulation functions to chain into
3688 	 * the list of callbacks.
3689 	 */
3690 	mncb = kmem_zalloc(sizeof (mac_notify_cb_t), KM_SLEEP);
3691 	mncb->mncb_fn = notify_fn;
3692 	mncb->mncb_arg = arg;
3693 	mncb->mncb_mip = mip;
3694 	mncb->mncb_link.mcb_objp = mncb;
3695 	mncb->mncb_link.mcb_objsize = sizeof (mac_notify_cb_t);
3696 	mncb->mncb_link.mcb_flags = MCB_NOTIFY_CB_T;
3697 
3698 	mcbi = &mip->mi_notify_cb_info;
3699 
3700 	i_mac_perim_enter(mip);
3701 	mutex_enter(mcbi->mcbi_lockp);
3702 
3703 	mac_callback_add(&mip->mi_notify_cb_info, &mip->mi_notify_cb_list,
3704 	    &mncb->mncb_link);
3705 
3706 	mutex_exit(mcbi->mcbi_lockp);
3707 	i_mac_perim_exit(mip);
3708 	return ((mac_notify_handle_t)mncb);
3709 }
3710 
3711 void
3712 mac_notify_remove_wait(mac_handle_t mh)
3713 {
3714 	mac_impl_t	*mip = (mac_impl_t *)mh;
3715 	mac_cb_info_t	*mcbi = &mip->mi_notify_cb_info;
3716 
3717 	mutex_enter(mcbi->mcbi_lockp);
3718 	mac_callback_remove_wait(&mip->mi_notify_cb_info);
3719 	mutex_exit(mcbi->mcbi_lockp);
3720 }
3721 
3722 /*
3723  * Remove a mac client specified notification callback
3724  */
3725 int
3726 mac_notify_remove(mac_notify_handle_t mnh, boolean_t wait)
3727 {
3728 	mac_notify_cb_t	*mncb = (mac_notify_cb_t *)mnh;
3729 	mac_impl_t	*mip = mncb->mncb_mip;
3730 	mac_cb_info_t	*mcbi;
3731 	int		err = 0;
3732 
3733 	mcbi = &mip->mi_notify_cb_info;
3734 
3735 	i_mac_perim_enter(mip);
3736 	mutex_enter(mcbi->mcbi_lockp);
3737 
3738 	ASSERT(mncb->mncb_link.mcb_objp == mncb);
3739 	/*
3740 	 * If there aren't any list walkers, the remove would succeed
3741 	 * inline, else we wait for the deferred remove to complete
3742 	 */
3743 	if (mac_callback_remove(&mip->mi_notify_cb_info,
3744 	    &mip->mi_notify_cb_list, &mncb->mncb_link)) {
3745 		kmem_free(mncb, sizeof (mac_notify_cb_t));
3746 	} else {
3747 		err = EBUSY;
3748 	}
3749 
3750 	mutex_exit(mcbi->mcbi_lockp);
3751 	i_mac_perim_exit(mip);
3752 
3753 	/*
3754 	 * If we failed to remove the notification callback and "wait" is set
3755 	 * to be B_TRUE, wait for the callback to finish after we exit the
3756 	 * mac perimeter.
3757 	 */
3758 	if (err != 0 && wait) {
3759 		mac_notify_remove_wait((mac_handle_t)mip);
3760 		return (0);
3761 	}
3762 
3763 	return (err);
3764 }
3765 
3766 /*
3767  * Associate resource management callbacks with the specified MAC
3768  * clients.
3769  */
3770 
3771 void
3772 mac_resource_set_common(mac_client_handle_t mch, mac_resource_add_t add,
3773     mac_resource_remove_t remove, mac_resource_quiesce_t quiesce,
3774     mac_resource_restart_t restart, mac_resource_bind_t bind,
3775     void *arg)
3776 {
3777 	mac_client_impl_t *mcip = (mac_client_impl_t *)mch;
3778 
3779 	mcip->mci_resource_add = add;
3780 	mcip->mci_resource_remove = remove;
3781 	mcip->mci_resource_quiesce = quiesce;
3782 	mcip->mci_resource_restart = restart;
3783 	mcip->mci_resource_bind = bind;
3784 	mcip->mci_resource_arg = arg;
3785 }
3786 
3787 void
3788 mac_resource_set(mac_client_handle_t mch, mac_resource_add_t add, void *arg)
3789 {
3790 	/* update the 'resource_add' callback */
3791 	mac_resource_set_common(mch, add, NULL, NULL, NULL, NULL, arg);
3792 }
3793 
3794 /*
3795  * Sets up the client resources and enable the polling interface over all the
3796  * SRS's and the soft rings of the client
3797  */
3798 void
3799 mac_client_poll_enable(mac_client_handle_t mch)
3800 {
3801 	mac_client_impl_t	*mcip = (mac_client_impl_t *)mch;
3802 	mac_soft_ring_set_t	*mac_srs;
3803 	flow_entry_t		*flent;
3804 	int			i;
3805 
3806 	flent = mcip->mci_flent;
3807 	ASSERT(flent != NULL);
3808 
3809 	mcip->mci_state_flags |= MCIS_CLIENT_POLL_CAPABLE;
3810 	for (i = 0; i < flent->fe_rx_srs_cnt; i++) {
3811 		mac_srs = (mac_soft_ring_set_t *)flent->fe_rx_srs[i];
3812 		ASSERT(mac_srs->srs_mcip == mcip);
3813 		mac_srs_client_poll_enable(mcip, mac_srs);
3814 	}
3815 }
3816 
3817 /*
3818  * Tears down the client resources and disable the polling interface over all
3819  * the SRS's and the soft rings of the client
3820  */
3821 void
3822 mac_client_poll_disable(mac_client_handle_t mch)
3823 {
3824 	mac_client_impl_t	*mcip = (mac_client_impl_t *)mch;
3825 	mac_soft_ring_set_t	*mac_srs;
3826 	flow_entry_t		*flent;
3827 	int			i;
3828 
3829 	flent = mcip->mci_flent;
3830 	ASSERT(flent != NULL);
3831 
3832 	mcip->mci_state_flags &= ~MCIS_CLIENT_POLL_CAPABLE;
3833 	for (i = 0; i < flent->fe_rx_srs_cnt; i++) {
3834 		mac_srs = (mac_soft_ring_set_t *)flent->fe_rx_srs[i];
3835 		ASSERT(mac_srs->srs_mcip == mcip);
3836 		mac_srs_client_poll_disable(mcip, mac_srs);
3837 	}
3838 }
3839 
3840 /*
3841  * Associate the CPUs specified by the given property with a MAC client.
3842  */
3843 int
3844 mac_cpu_set(mac_client_handle_t mch, mac_resource_props_t *mrp)
3845 {
3846 	mac_client_impl_t *mcip = (mac_client_impl_t *)mch;
3847 	mac_impl_t *mip = mcip->mci_mip;
3848 	int err = 0;
3849 
3850 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mip));
3851 
3852 	if ((err = mac_validate_props(mcip->mci_state_flags & MCIS_IS_VNIC ?
3853 	    mcip->mci_upper_mip : mip, mrp)) != 0) {
3854 		return (err);
3855 	}
3856 	if (MCIP_DATAPATH_SETUP(mcip))
3857 		mac_flow_modify(mip->mi_flow_tab, mcip->mci_flent, mrp);
3858 
3859 	mac_update_resources(mrp, MCIP_RESOURCE_PROPS(mcip), B_FALSE);
3860 	return (0);
3861 }
3862 
3863 /*
3864  * Apply the specified properties to the specified MAC client.
3865  */
3866 int
3867 mac_client_set_resources(mac_client_handle_t mch, mac_resource_props_t *mrp)
3868 {
3869 	mac_client_impl_t *mcip = (mac_client_impl_t *)mch;
3870 	mac_impl_t *mip = mcip->mci_mip;
3871 	int err = 0;
3872 
3873 	i_mac_perim_enter(mip);
3874 
3875 	if ((mrp->mrp_mask & MRP_MAXBW) || (mrp->mrp_mask & MRP_PRIORITY)) {
3876 		err = mac_resource_ctl_set(mch, mrp);
3877 		if (err != 0)
3878 			goto done;
3879 	}
3880 
3881 	if (mrp->mrp_mask & (MRP_CPUS|MRP_POOL)) {
3882 		err = mac_cpu_set(mch, mrp);
3883 		if (err != 0)
3884 			goto done;
3885 	}
3886 
3887 	if (mrp->mrp_mask & MRP_PROTECT) {
3888 		err = mac_protect_set(mch, mrp);
3889 		if (err != 0)
3890 			goto done;
3891 	}
3892 
3893 	if ((mrp->mrp_mask & MRP_RX_RINGS) || (mrp->mrp_mask & MRP_TX_RINGS))
3894 		err = mac_resource_ctl_set(mch, mrp);
3895 
3896 done:
3897 	i_mac_perim_exit(mip);
3898 	return (err);
3899 }
3900 
3901 /*
3902  * Return the properties currently associated with the specified MAC client.
3903  */
3904 void
3905 mac_client_get_resources(mac_client_handle_t mch, mac_resource_props_t *mrp)
3906 {
3907 	mac_client_impl_t	*mcip = (mac_client_impl_t *)mch;
3908 	mac_resource_props_t	*mcip_mrp = MCIP_RESOURCE_PROPS(mcip);
3909 
3910 	bcopy(mcip_mrp, mrp, sizeof (mac_resource_props_t));
3911 }
3912 
3913 /*
3914  * Return the effective properties currently associated with the specified
3915  * MAC client.
3916  */
3917 void
3918 mac_client_get_effective_resources(mac_client_handle_t mch,
3919     mac_resource_props_t *mrp)
3920 {
3921 	mac_client_impl_t	*mcip = (mac_client_impl_t *)mch;
3922 	mac_resource_props_t	*mcip_mrp = MCIP_EFFECTIVE_PROPS(mcip);
3923 
3924 	bcopy(mcip_mrp, mrp, sizeof (mac_resource_props_t));
3925 }
3926 
3927 /*
3928  * Pass a copy of the specified packet to the promiscuous callbacks
3929  * of the specified MAC.
3930  *
3931  * If sender is NULL, the function is being invoked for a packet chain
3932  * received from the wire. If sender is non-NULL, it points to
3933  * the MAC client from which the packet is being sent.
3934  *
3935  * The packets are distributed to the promiscuous callbacks as follows:
3936  *
3937  * - all packets are sent to the MAC_CLIENT_PROMISC_ALL callbacks
3938  * - all broadcast and multicast packets are sent to the
3939  *   MAC_CLIENT_PROMISC_FILTER and MAC_CLIENT_PROMISC_MULTI.
3940  *
3941  * The unicast packets of MAC_CLIENT_PROMISC_FILTER callbacks are dispatched
3942  * after classification by mac_rx_deliver().
3943  */
3944 
3945 static void
3946 mac_promisc_dispatch_one(mac_promisc_impl_t *mpip, mblk_t *mp,
3947     boolean_t loopback)
3948 {
3949 	mblk_t *mp_copy, *mp_next;
3950 
3951 	if (!mpip->mpi_no_copy || mpip->mpi_strip_vlan_tag) {
3952 		mp_copy = copymsg(mp);
3953 		if (mp_copy == NULL)
3954 			return;
3955 
3956 		if (mpip->mpi_strip_vlan_tag) {
3957 			mp_copy = mac_strip_vlan_tag_chain(mp_copy);
3958 			if (mp_copy == NULL)
3959 				return;
3960 		}
3961 		mp_next = NULL;
3962 	} else {
3963 		mp_copy = mp;
3964 		mp_next = mp->b_next;
3965 	}
3966 	mp_copy->b_next = NULL;
3967 
3968 	mpip->mpi_fn(mpip->mpi_arg, NULL, mp_copy, loopback);
3969 	if (mp_copy == mp)
3970 		mp->b_next = mp_next;
3971 }
3972 
3973 /*
3974  * Return the VID of a packet. Zero if the packet is not tagged.
3975  */
3976 static uint16_t
3977 mac_ether_vid(mblk_t *mp)
3978 {
3979 	struct ether_header *eth = (struct ether_header *)mp->b_rptr;
3980 
3981 	if (ntohs(eth->ether_type) == ETHERTYPE_VLAN) {
3982 		struct ether_vlan_header *t_evhp =
3983 		    (struct ether_vlan_header *)mp->b_rptr;
3984 		return (VLAN_ID(ntohs(t_evhp->ether_tci)));
3985 	}
3986 
3987 	return (0);
3988 }
3989 
3990 /*
3991  * Return whether the specified packet contains a multicast or broadcast
3992  * destination MAC address.
3993  */
3994 static boolean_t
3995 mac_is_mcast(mac_impl_t *mip, mblk_t *mp)
3996 {
3997 	mac_header_info_t hdr_info;
3998 
3999 	if (mac_header_info((mac_handle_t)mip, mp, &hdr_info) != 0)
4000 		return (B_FALSE);
4001 	return ((hdr_info.mhi_dsttype == MAC_ADDRTYPE_BROADCAST) ||
4002 	    (hdr_info.mhi_dsttype == MAC_ADDRTYPE_MULTICAST));
4003 }
4004 
4005 /*
4006  * Send a copy of an mblk chain to the MAC clients of the specified MAC.
4007  * "sender" points to the sender MAC client for outbound packets, and
4008  * is set to NULL for inbound packets.
4009  */
4010 void
4011 mac_promisc_dispatch(mac_impl_t *mip, mblk_t *mp_chain,
4012     mac_client_impl_t *sender)
4013 {
4014 	mac_promisc_impl_t *mpip;
4015 	mac_cb_t *mcb;
4016 	mblk_t *mp;
4017 	boolean_t is_mcast, is_sender;
4018 
4019 	MAC_PROMISC_WALKER_INC(mip);
4020 	for (mp = mp_chain; mp != NULL; mp = mp->b_next) {
4021 		is_mcast = mac_is_mcast(mip, mp);
4022 		/* send packet to interested callbacks */
4023 		for (mcb = mip->mi_promisc_list; mcb != NULL;
4024 		    mcb = mcb->mcb_nextp) {
4025 			mpip = (mac_promisc_impl_t *)mcb->mcb_objp;
4026 			is_sender = (mpip->mpi_mcip == sender);
4027 
4028 			if (is_sender && mpip->mpi_no_tx_loop)
4029 				/*
4030 				 * The sender doesn't want to receive
4031 				 * copies of the packets it sends.
4032 				 */
4033 				continue;
4034 
4035 			/* this client doesn't need any packets (bridge) */
4036 			if (mpip->mpi_fn == NULL)
4037 				continue;
4038 
4039 			/*
4040 			 * For an ethernet MAC, don't displatch a multicast
4041 			 * packet to a non-PROMISC_ALL callbacks unless the VID
4042 			 * of the packet matches the VID of the client.
4043 			 */
4044 			if (is_mcast &&
4045 			    mpip->mpi_type != MAC_CLIENT_PROMISC_ALL &&
4046 			    !mac_client_check_flow_vid(mpip->mpi_mcip,
4047 			    mac_ether_vid(mp)))
4048 				continue;
4049 
4050 			if (is_sender ||
4051 			    mpip->mpi_type == MAC_CLIENT_PROMISC_ALL ||
4052 			    is_mcast)
4053 				mac_promisc_dispatch_one(mpip, mp, is_sender);
4054 		}
4055 	}
4056 	MAC_PROMISC_WALKER_DCR(mip);
4057 }
4058 
4059 void
4060 mac_promisc_client_dispatch(mac_client_impl_t *mcip, mblk_t *mp_chain)
4061 {
4062 	mac_impl_t		*mip = mcip->mci_mip;
4063 	mac_promisc_impl_t	*mpip;
4064 	boolean_t		is_mcast;
4065 	mblk_t			*mp;
4066 	mac_cb_t		*mcb;
4067 
4068 	/*
4069 	 * The unicast packets for the MAC client still
4070 	 * need to be delivered to the MAC_CLIENT_PROMISC_FILTERED
4071 	 * promiscuous callbacks. The broadcast and multicast
4072 	 * packets were delivered from mac_rx().
4073 	 */
4074 	MAC_PROMISC_WALKER_INC(mip);
4075 	for (mp = mp_chain; mp != NULL; mp = mp->b_next) {
4076 		is_mcast = mac_is_mcast(mip, mp);
4077 		for (mcb = mcip->mci_promisc_list; mcb != NULL;
4078 		    mcb = mcb->mcb_nextp) {
4079 			mpip = (mac_promisc_impl_t *)mcb->mcb_objp;
4080 			if (mpip->mpi_type == MAC_CLIENT_PROMISC_FILTERED &&
4081 			    !is_mcast) {
4082 				mac_promisc_dispatch_one(mpip, mp, B_FALSE);
4083 			}
4084 		}
4085 	}
4086 	MAC_PROMISC_WALKER_DCR(mip);
4087 }
4088 
4089 /*
4090  * Return the margin value currently assigned to the specified MAC instance.
4091  */
4092 void
4093 mac_margin_get(mac_handle_t mh, uint32_t *marginp)
4094 {
4095 	mac_impl_t *mip = (mac_impl_t *)mh;
4096 
4097 	rw_enter(&(mip->mi_rw_lock), RW_READER);
4098 	*marginp = mip->mi_margin;
4099 	rw_exit(&(mip->mi_rw_lock));
4100 }
4101 
4102 /*
4103  * mac_info_get() is used for retrieving the mac_info when a DL_INFO_REQ is
4104  * issued before a DL_ATTACH_REQ. we walk the i_mac_impl_hash table and find
4105  * the first mac_impl_t with a matching driver name; then we copy its mac_info_t
4106  * to the caller. we do all this with i_mac_impl_lock held so the mac_impl_t
4107  * cannot disappear while we are accessing it.
4108  */
4109 typedef struct i_mac_info_state_s {
4110 	const char	*mi_name;
4111 	mac_info_t	*mi_infop;
4112 } i_mac_info_state_t;
4113 
4114 /*ARGSUSED*/
4115 static uint_t
4116 i_mac_info_walker(mod_hash_key_t key, mod_hash_val_t *val, void *arg)
4117 {
4118 	i_mac_info_state_t *statep = arg;
4119 	mac_impl_t *mip = (mac_impl_t *)val;
4120 
4121 	if (mip->mi_state_flags & MIS_DISABLED)
4122 		return (MH_WALK_CONTINUE);
4123 
4124 	if (strcmp(statep->mi_name,
4125 	    ddi_driver_name(mip->mi_dip)) != 0)
4126 		return (MH_WALK_CONTINUE);
4127 
4128 	statep->mi_infop = &mip->mi_info;
4129 	return (MH_WALK_TERMINATE);
4130 }
4131 
4132 boolean_t
4133 mac_info_get(const char *name, mac_info_t *minfop)
4134 {
4135 	i_mac_info_state_t state;
4136 
4137 	rw_enter(&i_mac_impl_lock, RW_READER);
4138 	state.mi_name = name;
4139 	state.mi_infop = NULL;
4140 	mod_hash_walk(i_mac_impl_hash, i_mac_info_walker, &state);
4141 	if (state.mi_infop == NULL) {
4142 		rw_exit(&i_mac_impl_lock);
4143 		return (B_FALSE);
4144 	}
4145 	*minfop = *state.mi_infop;
4146 	rw_exit(&i_mac_impl_lock);
4147 	return (B_TRUE);
4148 }
4149 
4150 /*
4151  * To get the capabilities that MAC layer cares about, such as rings, factory
4152  * mac address, vnic or not, it should directly invoke this function.  If the
4153  * link is part of a bridge, then the only "capability" it has is the inability
4154  * to do zero copy.
4155  */
4156 boolean_t
4157 i_mac_capab_get(mac_handle_t mh, mac_capab_t cap, void *cap_data)
4158 {
4159 	mac_impl_t *mip = (mac_impl_t *)mh;
4160 
4161 	if (mip->mi_bridge_link != NULL)
4162 		return (cap == MAC_CAPAB_NO_ZCOPY);
4163 	else if (mip->mi_callbacks->mc_callbacks & MC_GETCAPAB)
4164 		return (mip->mi_getcapab(mip->mi_driver, cap, cap_data));
4165 	else
4166 		return (B_FALSE);
4167 }
4168 
4169 /*
4170  * Capability query function. If number of active mac clients is greater than
4171  * 1, only limited capabilities can be advertised to the caller no matter the
4172  * driver has certain capability or not. Else, we query the driver to get the
4173  * capability.
4174  */
4175 boolean_t
4176 mac_capab_get(mac_handle_t mh, mac_capab_t cap, void *cap_data)
4177 {
4178 	mac_impl_t *mip = (mac_impl_t *)mh;
4179 
4180 	/*
4181 	 * if mi_nactiveclients > 1, only MAC_CAPAB_LEGACY, MAC_CAPAB_HCKSUM,
4182 	 * MAC_CAPAB_NO_NATIVEVLAN and MAC_CAPAB_NO_ZCOPY can be advertised.
4183 	 */
4184 	if (mip->mi_nactiveclients > 1) {
4185 		switch (cap) {
4186 		case MAC_CAPAB_NO_ZCOPY:
4187 			return (B_TRUE);
4188 		case MAC_CAPAB_LEGACY:
4189 		case MAC_CAPAB_HCKSUM:
4190 		case MAC_CAPAB_NO_NATIVEVLAN:
4191 			break;
4192 		default:
4193 			return (B_FALSE);
4194 		}
4195 	}
4196 
4197 	/* else get capab from driver */
4198 	return (i_mac_capab_get(mh, cap, cap_data));
4199 }
4200 
4201 boolean_t
4202 mac_sap_verify(mac_handle_t mh, uint32_t sap, uint32_t *bind_sap)
4203 {
4204 	mac_impl_t *mip = (mac_impl_t *)mh;
4205 
4206 	return (mip->mi_type->mt_ops.mtops_sap_verify(sap, bind_sap,
4207 	    mip->mi_pdata));
4208 }
4209 
4210 mblk_t *
4211 mac_header(mac_handle_t mh, const uint8_t *daddr, uint32_t sap, mblk_t *payload,
4212     size_t extra_len)
4213 {
4214 	mac_impl_t	*mip = (mac_impl_t *)mh;
4215 	const uint8_t	*hdr_daddr;
4216 
4217 	/*
4218 	 * If the MAC is point-to-point with a fixed destination address, then
4219 	 * we must always use that destination in the MAC header.
4220 	 */
4221 	hdr_daddr = (mip->mi_dstaddr_set ? mip->mi_dstaddr : daddr);
4222 	return (mip->mi_type->mt_ops.mtops_header(mip->mi_addr, hdr_daddr, sap,
4223 	    mip->mi_pdata, payload, extra_len));
4224 }
4225 
4226 int
4227 mac_header_info(mac_handle_t mh, mblk_t *mp, mac_header_info_t *mhip)
4228 {
4229 	mac_impl_t *mip = (mac_impl_t *)mh;
4230 
4231 	return (mip->mi_type->mt_ops.mtops_header_info(mp, mip->mi_pdata,
4232 	    mhip));
4233 }
4234 
4235 int
4236 mac_vlan_header_info(mac_handle_t mh, mblk_t *mp, mac_header_info_t *mhip)
4237 {
4238 	mac_impl_t	*mip = (mac_impl_t *)mh;
4239 	boolean_t	is_ethernet = (mip->mi_info.mi_media == DL_ETHER);
4240 	int		err = 0;
4241 
4242 	/*
4243 	 * Packets should always be at least 16 bit aligned.
4244 	 */
4245 	ASSERT(IS_P2ALIGNED(mp->b_rptr, sizeof (uint16_t)));
4246 
4247 	if ((err = mac_header_info(mh, mp, mhip)) != 0)
4248 		return (err);
4249 
4250 	/*
4251 	 * If this is a VLAN-tagged Ethernet packet, then the SAP in the
4252 	 * mac_header_info_t as returned by mac_header_info() is
4253 	 * ETHERTYPE_VLAN. We need to grab the ethertype from the VLAN header.
4254 	 */
4255 	if (is_ethernet && (mhip->mhi_bindsap == ETHERTYPE_VLAN)) {
4256 		struct ether_vlan_header *evhp;
4257 		uint16_t sap;
4258 		mblk_t *tmp = NULL;
4259 		size_t size;
4260 
4261 		size = sizeof (struct ether_vlan_header);
4262 		if (MBLKL(mp) < size) {
4263 			/*
4264 			 * Pullup the message in order to get the MAC header
4265 			 * infomation. Note that this is a read-only function,
4266 			 * we keep the input packet intact.
4267 			 */
4268 			if ((tmp = msgpullup(mp, size)) == NULL)
4269 				return (EINVAL);
4270 
4271 			mp = tmp;
4272 		}
4273 		evhp = (struct ether_vlan_header *)mp->b_rptr;
4274 		sap = ntohs(evhp->ether_type);
4275 		(void) mac_sap_verify(mh, sap, &mhip->mhi_bindsap);
4276 		mhip->mhi_hdrsize = sizeof (struct ether_vlan_header);
4277 		mhip->mhi_tci = ntohs(evhp->ether_tci);
4278 		mhip->mhi_istagged = B_TRUE;
4279 		freemsg(tmp);
4280 
4281 		if (VLAN_CFI(mhip->mhi_tci) != ETHER_CFI)
4282 			return (EINVAL);
4283 	} else {
4284 		mhip->mhi_istagged = B_FALSE;
4285 		mhip->mhi_tci = 0;
4286 	}
4287 
4288 	return (0);
4289 }
4290 
4291 mblk_t *
4292 mac_header_cook(mac_handle_t mh, mblk_t *mp)
4293 {
4294 	mac_impl_t *mip = (mac_impl_t *)mh;
4295 
4296 	if (mip->mi_type->mt_ops.mtops_ops & MTOPS_HEADER_COOK) {
4297 		if (DB_REF(mp) > 1) {
4298 			mblk_t *newmp = copymsg(mp);
4299 			if (newmp == NULL)
4300 				return (NULL);
4301 			freemsg(mp);
4302 			mp = newmp;
4303 		}
4304 		return (mip->mi_type->mt_ops.mtops_header_cook(mp,
4305 		    mip->mi_pdata));
4306 	}
4307 	return (mp);
4308 }
4309 
4310 mblk_t *
4311 mac_header_uncook(mac_handle_t mh, mblk_t *mp)
4312 {
4313 	mac_impl_t *mip = (mac_impl_t *)mh;
4314 
4315 	if (mip->mi_type->mt_ops.mtops_ops & MTOPS_HEADER_UNCOOK) {
4316 		if (DB_REF(mp) > 1) {
4317 			mblk_t *newmp = copymsg(mp);
4318 			if (newmp == NULL)
4319 				return (NULL);
4320 			freemsg(mp);
4321 			mp = newmp;
4322 		}
4323 		return (mip->mi_type->mt_ops.mtops_header_uncook(mp,
4324 		    mip->mi_pdata));
4325 	}
4326 	return (mp);
4327 }
4328 
4329 uint_t
4330 mac_addr_len(mac_handle_t mh)
4331 {
4332 	mac_impl_t *mip = (mac_impl_t *)mh;
4333 
4334 	return (mip->mi_type->mt_addr_length);
4335 }
4336 
4337 /* True if a MAC is a VNIC */
4338 boolean_t
4339 mac_is_vnic(mac_handle_t mh)
4340 {
4341 	return (((mac_impl_t *)mh)->mi_state_flags & MIS_IS_VNIC);
4342 }
4343 
4344 mac_handle_t
4345 mac_get_lower_mac_handle(mac_handle_t mh)
4346 {
4347 	mac_impl_t *mip = (mac_impl_t *)mh;
4348 
4349 	ASSERT(mac_is_vnic(mh));
4350 	return (((vnic_t *)mip->mi_driver)->vn_lower_mh);
4351 }
4352 
4353 boolean_t
4354 mac_is_vnic_primary(mac_handle_t mh)
4355 {
4356 	mac_impl_t *mip = (mac_impl_t *)mh;
4357 
4358 	ASSERT(mac_is_vnic(mh));
4359 	return (((vnic_t *)mip->mi_driver)->vn_addr_type ==
4360 	    VNIC_MAC_ADDR_TYPE_PRIMARY);
4361 }
4362 
4363 void
4364 mac_update_resources(mac_resource_props_t *nmrp, mac_resource_props_t *cmrp,
4365     boolean_t is_user_flow)
4366 {
4367 	if (nmrp != NULL && cmrp != NULL) {
4368 		if (nmrp->mrp_mask & MRP_PRIORITY) {
4369 			if (nmrp->mrp_priority == MPL_RESET) {
4370 				cmrp->mrp_mask &= ~MRP_PRIORITY;
4371 				if (is_user_flow) {
4372 					cmrp->mrp_priority =
4373 					    MPL_SUBFLOW_DEFAULT;
4374 				} else {
4375 					cmrp->mrp_priority = MPL_LINK_DEFAULT;
4376 				}
4377 			} else {
4378 				cmrp->mrp_mask |= MRP_PRIORITY;
4379 				cmrp->mrp_priority = nmrp->mrp_priority;
4380 			}
4381 		}
4382 		if (nmrp->mrp_mask & MRP_MAXBW) {
4383 			if (nmrp->mrp_maxbw == MRP_MAXBW_RESETVAL) {
4384 				cmrp->mrp_mask &= ~MRP_MAXBW;
4385 				cmrp->mrp_maxbw = 0;
4386 			} else {
4387 				cmrp->mrp_mask |= MRP_MAXBW;
4388 				cmrp->mrp_maxbw = nmrp->mrp_maxbw;
4389 			}
4390 		}
4391 		if (nmrp->mrp_mask & MRP_CPUS)
4392 			MAC_COPY_CPUS(nmrp, cmrp);
4393 
4394 		if (nmrp->mrp_mask & MRP_POOL) {
4395 			if (strlen(nmrp->mrp_pool) == 0) {
4396 				cmrp->mrp_mask &= ~MRP_POOL;
4397 				bzero(cmrp->mrp_pool, sizeof (cmrp->mrp_pool));
4398 			} else {
4399 				cmrp->mrp_mask |= MRP_POOL;
4400 				(void) strncpy(cmrp->mrp_pool, nmrp->mrp_pool,
4401 				    sizeof (cmrp->mrp_pool));
4402 			}
4403 
4404 		}
4405 
4406 		if (nmrp->mrp_mask & MRP_PROTECT)
4407 			mac_protect_update(nmrp, cmrp);
4408 
4409 		/*
4410 		 * Update the rings specified.
4411 		 */
4412 		if (nmrp->mrp_mask & MRP_RX_RINGS) {
4413 			if (nmrp->mrp_mask & MRP_RINGS_RESET) {
4414 				cmrp->mrp_mask &= ~MRP_RX_RINGS;
4415 				if (cmrp->mrp_mask & MRP_RXRINGS_UNSPEC)
4416 					cmrp->mrp_mask &= ~MRP_RXRINGS_UNSPEC;
4417 				cmrp->mrp_nrxrings = 0;
4418 			} else {
4419 				cmrp->mrp_mask |= MRP_RX_RINGS;
4420 				cmrp->mrp_nrxrings = nmrp->mrp_nrxrings;
4421 			}
4422 		}
4423 		if (nmrp->mrp_mask & MRP_TX_RINGS) {
4424 			if (nmrp->mrp_mask & MRP_RINGS_RESET) {
4425 				cmrp->mrp_mask &= ~MRP_TX_RINGS;
4426 				if (cmrp->mrp_mask & MRP_TXRINGS_UNSPEC)
4427 					cmrp->mrp_mask &= ~MRP_TXRINGS_UNSPEC;
4428 				cmrp->mrp_ntxrings = 0;
4429 			} else {
4430 				cmrp->mrp_mask |= MRP_TX_RINGS;
4431 				cmrp->mrp_ntxrings = nmrp->mrp_ntxrings;
4432 			}
4433 		}
4434 		if (nmrp->mrp_mask & MRP_RXRINGS_UNSPEC)
4435 			cmrp->mrp_mask |= MRP_RXRINGS_UNSPEC;
4436 		else if (cmrp->mrp_mask & MRP_RXRINGS_UNSPEC)
4437 			cmrp->mrp_mask &= ~MRP_RXRINGS_UNSPEC;
4438 
4439 		if (nmrp->mrp_mask & MRP_TXRINGS_UNSPEC)
4440 			cmrp->mrp_mask |= MRP_TXRINGS_UNSPEC;
4441 		else if (cmrp->mrp_mask & MRP_TXRINGS_UNSPEC)
4442 			cmrp->mrp_mask &= ~MRP_TXRINGS_UNSPEC;
4443 	}
4444 }
4445 
4446 /*
4447  * i_mac_set_resources:
4448  *
4449  * This routine associates properties with the primary MAC client of
4450  * the specified MAC instance.
4451  * - Cache the properties in mac_impl_t
4452  * - Apply the properties to the primary MAC client if exists
4453  */
4454 int
4455 i_mac_set_resources(mac_handle_t mh, mac_resource_props_t *mrp)
4456 {
4457 	mac_impl_t		*mip = (mac_impl_t *)mh;
4458 	mac_client_impl_t	*mcip;
4459 	int			err = 0;
4460 	uint32_t		resmask, newresmask;
4461 	mac_resource_props_t	*tmrp, *umrp;
4462 
4463 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mip));
4464 
4465 	err = mac_validate_props(mip, mrp);
4466 	if (err != 0)
4467 		return (err);
4468 
4469 	umrp = kmem_zalloc(sizeof (*umrp), KM_SLEEP);
4470 	bcopy(&mip->mi_resource_props, umrp, sizeof (*umrp));
4471 	resmask = umrp->mrp_mask;
4472 	mac_update_resources(mrp, umrp, B_FALSE);
4473 	newresmask = umrp->mrp_mask;
4474 
4475 	if (resmask == 0 && newresmask != 0) {
4476 		/*
4477 		 * Bandwidth, priority, cpu or pool link properties configured,
4478 		 * must disable fastpath.
4479 		 */
4480 		if ((err = mac_fastpath_disable((mac_handle_t)mip)) != 0) {
4481 			kmem_free(umrp, sizeof (*umrp));
4482 			return (err);
4483 		}
4484 	}
4485 
4486 	/*
4487 	 * Since bind_cpu may be modified by mac_client_set_resources()
4488 	 * we use a copy of bind_cpu and finally cache bind_cpu in mip.
4489 	 * This allows us to cache only user edits in mip.
4490 	 */
4491 	tmrp = kmem_zalloc(sizeof (*tmrp), KM_SLEEP);
4492 	bcopy(mrp, tmrp, sizeof (*tmrp));
4493 	mcip = mac_primary_client_handle(mip);
4494 	if (mcip != NULL && (mcip->mci_state_flags & MCIS_IS_AGGR_PORT) == 0) {
4495 		err = mac_client_set_resources((mac_client_handle_t)mcip, tmrp);
4496 	} else if ((mrp->mrp_mask & MRP_RX_RINGS ||
4497 	    mrp->mrp_mask & MRP_TX_RINGS)) {
4498 		mac_client_impl_t	*vmcip;
4499 
4500 		/*
4501 		 * If the primary is not up, we need to check if there
4502 		 * are any VLANs on this primary. If there are then
4503 		 * we need to set this property on the VLANs since
4504 		 * VLANs follow the primary they are based on. Just
4505 		 * look for the first VLAN and change its properties,
4506 		 * all the other VLANs should be in the same group.
4507 		 */
4508 		for (vmcip = mip->mi_clients_list; vmcip != NULL;
4509 		    vmcip = vmcip->mci_client_next) {
4510 			if ((vmcip->mci_flent->fe_type & FLOW_PRIMARY_MAC) &&
4511 			    mac_client_vid((mac_client_handle_t)vmcip) !=
4512 			    VLAN_ID_NONE) {
4513 				break;
4514 			}
4515 		}
4516 		if (vmcip != NULL) {
4517 			mac_resource_props_t	*omrp;
4518 			mac_resource_props_t	*vmrp;
4519 
4520 			omrp = kmem_zalloc(sizeof (*omrp), KM_SLEEP);
4521 			bcopy(MCIP_RESOURCE_PROPS(vmcip), omrp, sizeof (*omrp));
4522 			/*
4523 			 * We dont' call mac_update_resources since we
4524 			 * want to take only the ring properties and
4525 			 * not all the properties that may have changed.
4526 			 */
4527 			vmrp = MCIP_RESOURCE_PROPS(vmcip);
4528 			if (mrp->mrp_mask & MRP_RX_RINGS) {
4529 				if (mrp->mrp_mask & MRP_RINGS_RESET) {
4530 					vmrp->mrp_mask &= ~MRP_RX_RINGS;
4531 					if (vmrp->mrp_mask &
4532 					    MRP_RXRINGS_UNSPEC) {
4533 						vmrp->mrp_mask &=
4534 						    ~MRP_RXRINGS_UNSPEC;
4535 					}
4536 					vmrp->mrp_nrxrings = 0;
4537 				} else {
4538 					vmrp->mrp_mask |= MRP_RX_RINGS;
4539 					vmrp->mrp_nrxrings = mrp->mrp_nrxrings;
4540 				}
4541 			}
4542 			if (mrp->mrp_mask & MRP_TX_RINGS) {
4543 				if (mrp->mrp_mask & MRP_RINGS_RESET) {
4544 					vmrp->mrp_mask &= ~MRP_TX_RINGS;
4545 					if (vmrp->mrp_mask &
4546 					    MRP_TXRINGS_UNSPEC) {
4547 						vmrp->mrp_mask &=
4548 						    ~MRP_TXRINGS_UNSPEC;
4549 					}
4550 					vmrp->mrp_ntxrings = 0;
4551 				} else {
4552 					vmrp->mrp_mask |= MRP_TX_RINGS;
4553 					vmrp->mrp_ntxrings = mrp->mrp_ntxrings;
4554 				}
4555 			}
4556 			if (mrp->mrp_mask & MRP_RXRINGS_UNSPEC)
4557 				vmrp->mrp_mask |= MRP_RXRINGS_UNSPEC;
4558 
4559 			if (mrp->mrp_mask & MRP_TXRINGS_UNSPEC)
4560 				vmrp->mrp_mask |= MRP_TXRINGS_UNSPEC;
4561 
4562 			if ((err = mac_client_set_rings_prop(vmcip, mrp,
4563 			    omrp)) != 0) {
4564 				bcopy(omrp, MCIP_RESOURCE_PROPS(vmcip),
4565 				    sizeof (*omrp));
4566 			} else {
4567 				mac_set_prim_vlan_rings(mip, vmrp);
4568 			}
4569 			kmem_free(omrp, sizeof (*omrp));
4570 		}
4571 	}
4572 
4573 	/* Only update the values if mac_client_set_resources succeeded */
4574 	if (err == 0) {
4575 		bcopy(umrp, &mip->mi_resource_props, sizeof (*umrp));
4576 		/*
4577 		 * If bandwidth, priority or cpu link properties cleared,
4578 		 * renable fastpath.
4579 		 */
4580 		if (resmask != 0 && newresmask == 0)
4581 			mac_fastpath_enable((mac_handle_t)mip);
4582 	} else if (resmask == 0 && newresmask != 0) {
4583 		mac_fastpath_enable((mac_handle_t)mip);
4584 	}
4585 	kmem_free(tmrp, sizeof (*tmrp));
4586 	kmem_free(umrp, sizeof (*umrp));
4587 	return (err);
4588 }
4589 
4590 int
4591 mac_set_resources(mac_handle_t mh, mac_resource_props_t *mrp)
4592 {
4593 	int err;
4594 
4595 	i_mac_perim_enter((mac_impl_t *)mh);
4596 	err = i_mac_set_resources(mh, mrp);
4597 	i_mac_perim_exit((mac_impl_t *)mh);
4598 	return (err);
4599 }
4600 
4601 /*
4602  * Get the properties cached for the specified MAC instance.
4603  */
4604 void
4605 mac_get_resources(mac_handle_t mh, mac_resource_props_t *mrp)
4606 {
4607 	mac_impl_t 		*mip = (mac_impl_t *)mh;
4608 	mac_client_impl_t	*mcip;
4609 
4610 	mcip = mac_primary_client_handle(mip);
4611 	if (mcip != NULL) {
4612 		mac_client_get_resources((mac_client_handle_t)mcip, mrp);
4613 		return;
4614 	}
4615 	bcopy(&mip->mi_resource_props, mrp, sizeof (mac_resource_props_t));
4616 }
4617 
4618 /*
4619  * Get the effective properties from the primary client of the
4620  * specified MAC instance.
4621  */
4622 void
4623 mac_get_effective_resources(mac_handle_t mh, mac_resource_props_t *mrp)
4624 {
4625 	mac_impl_t 		*mip = (mac_impl_t *)mh;
4626 	mac_client_impl_t	*mcip;
4627 
4628 	mcip = mac_primary_client_handle(mip);
4629 	if (mcip != NULL) {
4630 		mac_client_get_effective_resources((mac_client_handle_t)mcip,
4631 		    mrp);
4632 		return;
4633 	}
4634 	bzero(mrp, sizeof (mac_resource_props_t));
4635 }
4636 
4637 int
4638 mac_set_pvid(mac_handle_t mh, uint16_t pvid)
4639 {
4640 	mac_impl_t *mip = (mac_impl_t *)mh;
4641 	mac_client_impl_t *mcip;
4642 	mac_unicast_impl_t *muip;
4643 
4644 	i_mac_perim_enter(mip);
4645 	if (pvid != 0) {
4646 		for (mcip = mip->mi_clients_list; mcip != NULL;
4647 		    mcip = mcip->mci_client_next) {
4648 			for (muip = mcip->mci_unicast_list; muip != NULL;
4649 			    muip = muip->mui_next) {
4650 				if (muip->mui_vid == pvid) {
4651 					i_mac_perim_exit(mip);
4652 					return (EBUSY);
4653 				}
4654 			}
4655 		}
4656 	}
4657 	mip->mi_pvid = pvid;
4658 	i_mac_perim_exit(mip);
4659 	return (0);
4660 }
4661 
4662 uint16_t
4663 mac_get_pvid(mac_handle_t mh)
4664 {
4665 	mac_impl_t *mip = (mac_impl_t *)mh;
4666 
4667 	return (mip->mi_pvid);
4668 }
4669 
4670 uint32_t
4671 mac_get_llimit(mac_handle_t mh)
4672 {
4673 	mac_impl_t *mip = (mac_impl_t *)mh;
4674 
4675 	return (mip->mi_llimit);
4676 }
4677 
4678 uint32_t
4679 mac_get_ldecay(mac_handle_t mh)
4680 {
4681 	mac_impl_t *mip = (mac_impl_t *)mh;
4682 
4683 	return (mip->mi_ldecay);
4684 }
4685 
4686 /*
4687  * Rename a mac client, its flow, and the kstat.
4688  */
4689 int
4690 mac_rename_primary(mac_handle_t mh, const char *new_name)
4691 {
4692 	mac_impl_t		*mip = (mac_impl_t *)mh;
4693 	mac_client_impl_t	*cur_clnt = NULL;
4694 	flow_entry_t		*fep;
4695 
4696 	i_mac_perim_enter(mip);
4697 
4698 	/*
4699 	 * VNICs: we need to change the sys flow name and
4700 	 * the associated flow kstat.
4701 	 */
4702 	if (mip->mi_state_flags & MIS_IS_VNIC) {
4703 		mac_client_impl_t *mcip = mac_vnic_lower(mip);
4704 		ASSERT(new_name != NULL);
4705 		mac_rename_flow_names(mcip, new_name);
4706 		mac_stat_rename(mcip);
4707 		goto done;
4708 	}
4709 	/*
4710 	 * This mac may itself be an aggr link, or it may have some client
4711 	 * which is an aggr port. For both cases, we need to change the
4712 	 * aggr port's mac client name, its flow name and the associated flow
4713 	 * kstat.
4714 	 */
4715 	if (mip->mi_state_flags & MIS_IS_AGGR) {
4716 		mac_capab_aggr_t aggr_cap;
4717 		mac_rename_fn_t rename_fn;
4718 		boolean_t ret;
4719 
4720 		ASSERT(new_name != NULL);
4721 		ret = i_mac_capab_get((mac_handle_t)mip, MAC_CAPAB_AGGR,
4722 		    (void *)(&aggr_cap));
4723 		ASSERT(ret == B_TRUE);
4724 		rename_fn = aggr_cap.mca_rename_fn;
4725 		rename_fn(new_name, mip->mi_driver);
4726 		/*
4727 		 * The aggr's client name and kstat flow name will be
4728 		 * updated below, i.e. via mac_rename_flow_names.
4729 		 */
4730 	}
4731 
4732 	for (cur_clnt = mip->mi_clients_list; cur_clnt != NULL;
4733 	    cur_clnt = cur_clnt->mci_client_next) {
4734 		if (cur_clnt->mci_state_flags & MCIS_IS_AGGR_PORT) {
4735 			if (new_name != NULL) {
4736 				char *str_st = cur_clnt->mci_name;
4737 				char *str_del = strchr(str_st, '-');
4738 
4739 				ASSERT(str_del != NULL);
4740 				bzero(str_del + 1, MAXNAMELEN -
4741 				    (str_del - str_st + 1));
4742 				bcopy(new_name, str_del + 1,
4743 				    strlen(new_name));
4744 			}
4745 			fep = cur_clnt->mci_flent;
4746 			mac_rename_flow(fep, cur_clnt->mci_name);
4747 			break;
4748 		} else if (new_name != NULL &&
4749 		    cur_clnt->mci_state_flags & MCIS_USE_DATALINK_NAME) {
4750 			mac_rename_flow_names(cur_clnt, new_name);
4751 			break;
4752 		}
4753 	}
4754 
4755 	/* Recreate kstats associated with aggr pseudo rings */
4756 	if (mip->mi_state_flags & MIS_IS_AGGR)
4757 		mac_pseudo_ring_stat_rename(mip);
4758 
4759 done:
4760 	i_mac_perim_exit(mip);
4761 	return (0);
4762 }
4763 
4764 /*
4765  * Rename the MAC client's flow names
4766  */
4767 static void
4768 mac_rename_flow_names(mac_client_impl_t *mcip, const char *new_name)
4769 {
4770 	flow_entry_t	*flent;
4771 	uint16_t	vid;
4772 	char		flowname[MAXFLOWNAMELEN];
4773 	mac_impl_t	*mip = mcip->mci_mip;
4774 
4775 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mip));
4776 
4777 	/*
4778 	 * Use mi_rw_lock to ensure that threads not in the mac perimeter
4779 	 * see a self-consistent value for mci_name
4780 	 */
4781 	rw_enter(&mip->mi_rw_lock, RW_WRITER);
4782 	(void) strlcpy(mcip->mci_name, new_name, sizeof (mcip->mci_name));
4783 	rw_exit(&mip->mi_rw_lock);
4784 
4785 	mac_rename_flow(mcip->mci_flent, new_name);
4786 
4787 	if (mcip->mci_nflents == 1)
4788 		return;
4789 
4790 	/*
4791 	 * We have to rename all the others too, no stats to destroy for
4792 	 * these.
4793 	 */
4794 	for (flent = mcip->mci_flent_list; flent != NULL;
4795 	    flent = flent->fe_client_next) {
4796 		if (flent != mcip->mci_flent) {
4797 			vid = i_mac_flow_vid(flent);
4798 			(void) sprintf(flowname, "%s%u", new_name, vid);
4799 			mac_flow_set_name(flent, flowname);
4800 		}
4801 	}
4802 }
4803 
4804 
4805 /*
4806  * Add a flow to the MAC client's flow list - i.e list of MAC/VID tuples
4807  * defined for the specified MAC client.
4808  */
4809 static void
4810 mac_client_add_to_flow_list(mac_client_impl_t *mcip, flow_entry_t *flent)
4811 {
4812 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip));
4813 	/*
4814 	 * The promisc Rx data path walks the mci_flent_list. Protect by
4815 	 * using mi_rw_lock
4816 	 */
4817 	rw_enter(&mcip->mci_rw_lock, RW_WRITER);
4818 
4819 	mcip->mci_vidcache = MCIP_VIDCACHE_INVALID;
4820 
4821 	/* Add it to the head */
4822 	flent->fe_client_next = mcip->mci_flent_list;
4823 	mcip->mci_flent_list = flent;
4824 	mcip->mci_nflents++;
4825 
4826 	/*
4827 	 * Keep track of the number of non-zero VIDs addresses per MAC
4828 	 * client to avoid figuring it out in the data-path.
4829 	 */
4830 	if (i_mac_flow_vid(flent) != VLAN_ID_NONE)
4831 		mcip->mci_nvids++;
4832 
4833 	rw_exit(&mcip->mci_rw_lock);
4834 }
4835 
4836 /*
4837  * Remove a flow entry from the MAC client's list.
4838  */
4839 static void
4840 mac_client_remove_flow_from_list(mac_client_impl_t *mcip, flow_entry_t *flent)
4841 {
4842 	flow_entry_t	*fe = mcip->mci_flent_list;
4843 	flow_entry_t	*prev_fe = NULL;
4844 
4845 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip));
4846 	/*
4847 	 * The promisc Rx data path walks the mci_flent_list. Protect by
4848 	 * using mci_rw_lock
4849 	 */
4850 	rw_enter(&mcip->mci_rw_lock, RW_WRITER);
4851 	mcip->mci_vidcache = MCIP_VIDCACHE_INVALID;
4852 
4853 	while ((fe != NULL) && (fe != flent)) {
4854 		prev_fe = fe;
4855 		fe = fe->fe_client_next;
4856 	}
4857 
4858 	ASSERT(fe != NULL);
4859 	if (prev_fe == NULL) {
4860 		/* Deleting the first node */
4861 		mcip->mci_flent_list = fe->fe_client_next;
4862 	} else {
4863 		prev_fe->fe_client_next = fe->fe_client_next;
4864 	}
4865 	mcip->mci_nflents--;
4866 
4867 	if (i_mac_flow_vid(flent) != VLAN_ID_NONE)
4868 		mcip->mci_nvids--;
4869 
4870 	rw_exit(&mcip->mci_rw_lock);
4871 }
4872 
4873 /*
4874  * Check if the given VID belongs to this MAC client.
4875  */
4876 boolean_t
4877 mac_client_check_flow_vid(mac_client_impl_t *mcip, uint16_t vid)
4878 {
4879 	flow_entry_t	*flent;
4880 	uint16_t	mci_vid;
4881 	uint32_t	cache = mcip->mci_vidcache;
4882 
4883 	/*
4884 	 * In hopes of not having to touch the mci_rw_lock, check to see if
4885 	 * this vid matches our cached result.
4886 	 */
4887 	if (MCIP_VIDCACHE_ISVALID(cache) && MCIP_VIDCACHE_VID(cache) == vid)
4888 		return (MCIP_VIDCACHE_BOOL(cache) ? B_TRUE : B_FALSE);
4889 
4890 	/* The mci_flent_list is protected by mci_rw_lock */
4891 	rw_enter(&mcip->mci_rw_lock, RW_WRITER);
4892 	for (flent = mcip->mci_flent_list; flent != NULL;
4893 	    flent = flent->fe_client_next) {
4894 		mci_vid = i_mac_flow_vid(flent);
4895 		if (vid == mci_vid) {
4896 			mcip->mci_vidcache = MCIP_VIDCACHE_CACHE(vid, B_TRUE);
4897 			rw_exit(&mcip->mci_rw_lock);
4898 			return (B_TRUE);
4899 		}
4900 	}
4901 
4902 	mcip->mci_vidcache = MCIP_VIDCACHE_CACHE(vid, B_FALSE);
4903 	rw_exit(&mcip->mci_rw_lock);
4904 	return (B_FALSE);
4905 }
4906 
4907 /*
4908  * Get the flow entry for the specified <MAC addr, VID> tuple.
4909  */
4910 static flow_entry_t *
4911 mac_client_get_flow(mac_client_impl_t *mcip, mac_unicast_impl_t *muip)
4912 {
4913 	mac_address_t *map = mcip->mci_unicast;
4914 	flow_entry_t *flent;
4915 	uint16_t vid;
4916 	flow_desc_t flow_desc;
4917 
4918 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip));
4919 
4920 	mac_flow_get_desc(mcip->mci_flent, &flow_desc);
4921 	if (bcmp(flow_desc.fd_dst_mac, map->ma_addr, map->ma_len) != 0)
4922 		return (NULL);
4923 
4924 	for (flent = mcip->mci_flent_list; flent != NULL;
4925 	    flent = flent->fe_client_next) {
4926 		vid = i_mac_flow_vid(flent);
4927 		if (vid == muip->mui_vid) {
4928 			return (flent);
4929 		}
4930 	}
4931 
4932 	return (NULL);
4933 }
4934 
4935 /*
4936  * Since mci_flent has the SRSs, when we want to remove it, we replace
4937  * the flow_desc_t in mci_flent with that of an existing flent and then
4938  * remove that flent instead of mci_flent.
4939  */
4940 static flow_entry_t *
4941 mac_client_swap_mciflent(mac_client_impl_t *mcip)
4942 {
4943 	flow_entry_t	*flent = mcip->mci_flent;
4944 	flow_tab_t	*ft = flent->fe_flow_tab;
4945 	flow_entry_t	*flent1;
4946 	flow_desc_t	fl_desc;
4947 	char		fl_name[MAXFLOWNAMELEN];
4948 	int		err;
4949 
4950 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip));
4951 	ASSERT(mcip->mci_nflents > 1);
4952 
4953 	/* get the next flent following the primary flent  */
4954 	flent1 = mcip->mci_flent_list->fe_client_next;
4955 	ASSERT(flent1 != NULL && flent1->fe_flow_tab == ft);
4956 
4957 	/*
4958 	 * Remove the flent from the flow table before updating the
4959 	 * flow descriptor as the hash depends on the flow descriptor.
4960 	 * This also helps incoming packet classification avoid having
4961 	 * to grab fe_lock. Access to fe_flow_desc of a flent not in the
4962 	 * flow table is done under the fe_lock so that log or stat functions
4963 	 * see a self-consistent fe_flow_desc. The name and desc are specific
4964 	 * to a flow, the rest are shared by all the clients, including
4965 	 * resource control etc.
4966 	 */
4967 	mac_flow_remove(ft, flent, B_TRUE);
4968 	mac_flow_remove(ft, flent1, B_TRUE);
4969 
4970 	bcopy(&flent->fe_flow_desc, &fl_desc, sizeof (flow_desc_t));
4971 	bcopy(flent->fe_flow_name, fl_name, MAXFLOWNAMELEN);
4972 
4973 	/* update the primary flow entry */
4974 	mutex_enter(&flent->fe_lock);
4975 	bcopy(&flent1->fe_flow_desc, &flent->fe_flow_desc,
4976 	    sizeof (flow_desc_t));
4977 	bcopy(&flent1->fe_flow_name, &flent->fe_flow_name, MAXFLOWNAMELEN);
4978 	mutex_exit(&flent->fe_lock);
4979 
4980 	/* update the flow entry that is to be freed */
4981 	mutex_enter(&flent1->fe_lock);
4982 	bcopy(&fl_desc, &flent1->fe_flow_desc, sizeof (flow_desc_t));
4983 	bcopy(fl_name, &flent1->fe_flow_name, MAXFLOWNAMELEN);
4984 	mutex_exit(&flent1->fe_lock);
4985 
4986 	/* now reinsert the flow entries in the table */
4987 	err = mac_flow_add(ft, flent);
4988 	ASSERT(err == 0);
4989 
4990 	err = mac_flow_add(ft, flent1);
4991 	ASSERT(err == 0);
4992 
4993 	return (flent1);
4994 }
4995 
4996 /*
4997  * Return whether there is only one flow entry associated with this
4998  * MAC client.
4999  */
5000 static boolean_t
5001 mac_client_single_rcvr(mac_client_impl_t *mcip)
5002 {
5003 	return (mcip->mci_nflents == 1);
5004 }
5005 
5006 int
5007 mac_validate_props(mac_impl_t *mip, mac_resource_props_t *mrp)
5008 {
5009 	boolean_t		reset;
5010 	uint32_t		rings_needed;
5011 	uint32_t		rings_avail;
5012 	mac_group_type_t	gtype;
5013 	mac_resource_props_t	*mip_mrp;
5014 
5015 	if (mrp == NULL)
5016 		return (0);
5017 
5018 	if (mrp->mrp_mask & MRP_PRIORITY) {
5019 		mac_priority_level_t	pri = mrp->mrp_priority;
5020 
5021 		if (pri < MPL_LOW || pri > MPL_RESET)
5022 			return (EINVAL);
5023 	}
5024 
5025 	if (mrp->mrp_mask & MRP_MAXBW) {
5026 		uint64_t maxbw = mrp->mrp_maxbw;
5027 
5028 		if (maxbw < MRP_MAXBW_MINVAL && maxbw != 0)
5029 			return (EINVAL);
5030 	}
5031 	if (mrp->mrp_mask & MRP_CPUS) {
5032 		int i, j;
5033 		mac_cpu_mode_t	fanout;
5034 
5035 		if (mrp->mrp_ncpus > ncpus)
5036 			return (EINVAL);
5037 
5038 		for (i = 0; i < mrp->mrp_ncpus; i++) {
5039 			for (j = 0; j < mrp->mrp_ncpus; j++) {
5040 				if (i != j &&
5041 				    mrp->mrp_cpu[i] == mrp->mrp_cpu[j]) {
5042 					return (EINVAL);
5043 				}
5044 			}
5045 		}
5046 
5047 		for (i = 0; i < mrp->mrp_ncpus; i++) {
5048 			cpu_t *cp;
5049 			int rv;
5050 
5051 			mutex_enter(&cpu_lock);
5052 			cp = cpu_get(mrp->mrp_cpu[i]);
5053 			if (cp != NULL)
5054 				rv = cpu_is_online(cp);
5055 			else
5056 				rv = 0;
5057 			mutex_exit(&cpu_lock);
5058 			if (rv == 0)
5059 				return (EINVAL);
5060 		}
5061 
5062 		fanout = mrp->mrp_fanout_mode;
5063 		if (fanout < 0 || fanout > MCM_CPUS)
5064 			return (EINVAL);
5065 	}
5066 
5067 	if (mrp->mrp_mask & MRP_PROTECT) {
5068 		int err = mac_protect_validate(mrp);
5069 		if (err != 0)
5070 			return (err);
5071 	}
5072 
5073 	if (!(mrp->mrp_mask & MRP_RX_RINGS) &&
5074 	    !(mrp->mrp_mask & MRP_TX_RINGS)) {
5075 		return (0);
5076 	}
5077 
5078 	/*
5079 	 * mip will be null when we come from mac_flow_create or
5080 	 * mac_link_flow_modify. In the latter case it is a user flow,
5081 	 * for which we don't support rings. In the former we would
5082 	 * have validated the props beforehand (i_mac_unicast_add ->
5083 	 * mac_client_set_resources -> validate for the primary and
5084 	 * vnic_dev_create -> mac_client_set_resources -> validate for
5085 	 * a vnic.
5086 	 */
5087 	if (mip == NULL)
5088 		return (0);
5089 
5090 	/*
5091 	 * We don't support setting rings property for a VNIC that is using a
5092 	 * primary address (VLAN)
5093 	 */
5094 	if ((mip->mi_state_flags & MIS_IS_VNIC) &&
5095 	    mac_is_vnic_primary((mac_handle_t)mip)) {
5096 		return (ENOTSUP);
5097 	}
5098 
5099 	mip_mrp = &mip->mi_resource_props;
5100 	/*
5101 	 * The rings property should be validated against the NICs
5102 	 * resources
5103 	 */
5104 	if (mip->mi_state_flags & MIS_IS_VNIC)
5105 		mip = (mac_impl_t *)mac_get_lower_mac_handle((mac_handle_t)mip);
5106 
5107 	reset = mrp->mrp_mask & MRP_RINGS_RESET;
5108 	/*
5109 	 * If groups are not supported, return error.
5110 	 */
5111 	if (((mrp->mrp_mask & MRP_RX_RINGS) && mip->mi_rx_groups == NULL) ||
5112 	    ((mrp->mrp_mask & MRP_TX_RINGS) && mip->mi_tx_groups == NULL)) {
5113 		return (EINVAL);
5114 	}
5115 	/*
5116 	 * If we are just resetting, there is no validation needed.
5117 	 */
5118 	if (reset)
5119 		return (0);
5120 
5121 	if (mrp->mrp_mask & MRP_RX_RINGS) {
5122 		rings_needed = mrp->mrp_nrxrings;
5123 		/*
5124 		 * We just want to check if the number of additional
5125 		 * rings requested is available.
5126 		 */
5127 		if (mip_mrp->mrp_mask & MRP_RX_RINGS) {
5128 			if (mrp->mrp_nrxrings > mip_mrp->mrp_nrxrings)
5129 				/* Just check for the additional rings */
5130 				rings_needed -= mip_mrp->mrp_nrxrings;
5131 			else
5132 				/* We are not asking for additional rings */
5133 				rings_needed = 0;
5134 		}
5135 		rings_avail = mip->mi_rxrings_avail;
5136 		gtype = mip->mi_rx_group_type;
5137 	} else {
5138 		rings_needed = mrp->mrp_ntxrings;
5139 		/* Similarly for the TX rings */
5140 		if (mip_mrp->mrp_mask & MRP_TX_RINGS) {
5141 			if (mrp->mrp_ntxrings > mip_mrp->mrp_ntxrings)
5142 				/* Just check for the additional rings */
5143 				rings_needed -= mip_mrp->mrp_ntxrings;
5144 			else
5145 				/* We are not asking for additional rings */
5146 				rings_needed = 0;
5147 		}
5148 		rings_avail = mip->mi_txrings_avail;
5149 		gtype = mip->mi_tx_group_type;
5150 	}
5151 
5152 	/* Error if the group is dynamic .. */
5153 	if (gtype == MAC_GROUP_TYPE_DYNAMIC) {
5154 		/*
5155 		 * .. and rings specified are more than available.
5156 		 */
5157 		if (rings_needed > rings_avail)
5158 			return (EINVAL);
5159 	} else {
5160 		/*
5161 		 * OR group is static and we have specified some rings.
5162 		 */
5163 		if (rings_needed > 0)
5164 			return (EINVAL);
5165 	}
5166 	return (0);
5167 }
5168 
5169 /*
5170  * Send a MAC_NOTE_LINK notification to all the MAC clients whenever the
5171  * underlying physical link is down. This is to allow MAC clients to
5172  * communicate with other clients.
5173  */
5174 void
5175 mac_virtual_link_update(mac_impl_t *mip)
5176 {
5177 	if (mip->mi_linkstate != LINK_STATE_UP)
5178 		i_mac_notify(mip, MAC_NOTE_LINK);
5179 }
5180 
5181 /*
5182  * For clients that have a pass-thru MAC, e.g. VNIC, we set the VNIC's
5183  * mac handle in the client.
5184  */
5185 void
5186 mac_set_upper_mac(mac_client_handle_t mch, mac_handle_t mh,
5187     mac_resource_props_t *mrp)
5188 {
5189 	mac_client_impl_t	*mcip = (mac_client_impl_t *)mch;
5190 	mac_impl_t		*mip = (mac_impl_t *)mh;
5191 
5192 	mcip->mci_upper_mip = mip;
5193 	/* If there are any properties, copy it over too */
5194 	if (mrp != NULL) {
5195 		bcopy(mrp, &mip->mi_resource_props,
5196 		    sizeof (mac_resource_props_t));
5197 	}
5198 }
5199 
5200 /*
5201  * Mark the mac as being used exclusively by the single mac client that is
5202  * doing some control operation on this mac. No further opens of this mac
5203  * will be allowed until this client calls mac_unmark_exclusive. The mac
5204  * client calling this function must already be in the mac perimeter
5205  */
5206 int
5207 mac_mark_exclusive(mac_handle_t mh)
5208 {
5209 	mac_impl_t	*mip = (mac_impl_t *)mh;
5210 
5211 	ASSERT(MAC_PERIM_HELD(mh));
5212 	/*
5213 	 * Look up its entry in the global hash table.
5214 	 */
5215 	rw_enter(&i_mac_impl_lock, RW_WRITER);
5216 	if (mip->mi_state_flags & MIS_DISABLED) {
5217 		rw_exit(&i_mac_impl_lock);
5218 		return (ENOENT);
5219 	}
5220 
5221 	/*
5222 	 * A reference to mac is held even if the link is not plumbed.
5223 	 * In i_dls_link_create() we open the MAC interface and hold the
5224 	 * reference. There is an additional reference for the mac_open
5225 	 * done in acquiring the mac perimeter
5226 	 */
5227 	if (mip->mi_ref != 2) {
5228 		rw_exit(&i_mac_impl_lock);
5229 		return (EBUSY);
5230 	}
5231 
5232 	ASSERT(!(mip->mi_state_flags & MIS_EXCLUSIVE_HELD));
5233 	mip->mi_state_flags |= MIS_EXCLUSIVE_HELD;
5234 	rw_exit(&i_mac_impl_lock);
5235 	return (0);
5236 }
5237 
5238 void
5239 mac_unmark_exclusive(mac_handle_t mh)
5240 {
5241 	mac_impl_t	*mip = (mac_impl_t *)mh;
5242 
5243 	ASSERT(MAC_PERIM_HELD(mh));
5244 
5245 	rw_enter(&i_mac_impl_lock, RW_WRITER);
5246 	/* 1 for the creation and another for the perimeter */
5247 	ASSERT(mip->mi_ref == 2 && (mip->mi_state_flags & MIS_EXCLUSIVE_HELD));
5248 	mip->mi_state_flags &= ~MIS_EXCLUSIVE_HELD;
5249 	rw_exit(&i_mac_impl_lock);
5250 }
5251 
5252 /*
5253  * Set the MTU for the specified MAC.
5254  */
5255 int
5256 mac_set_mtu(mac_handle_t mh, uint_t new_mtu, uint_t *old_mtu_arg)
5257 {
5258 	mac_impl_t *mip = (mac_impl_t *)mh;
5259 	uint_t old_mtu;
5260 	int rv = 0;
5261 
5262 	i_mac_perim_enter(mip);
5263 
5264 	if (!(mip->mi_callbacks->mc_callbacks & (MC_SETPROP|MC_GETPROP))) {
5265 		rv = ENOTSUP;
5266 		goto bail;
5267 	}
5268 
5269 	old_mtu = mip->mi_sdu_max;
5270 
5271 	if (new_mtu == 0 || new_mtu < mip->mi_sdu_min) {
5272 		rv = EINVAL;
5273 		goto bail;
5274 	}
5275 
5276 	rw_enter(&mip->mi_rw_lock, RW_READER);
5277 	if (mip->mi_mtrp != NULL && new_mtu < mip->mi_mtrp->mtr_mtu) {
5278 		rv = EBUSY;
5279 		rw_exit(&mip->mi_rw_lock);
5280 		goto bail;
5281 	}
5282 	rw_exit(&mip->mi_rw_lock);
5283 
5284 	if (old_mtu != new_mtu) {
5285 		rv = mip->mi_callbacks->mc_setprop(mip->mi_driver,
5286 		    "mtu", MAC_PROP_MTU, sizeof (uint_t), &new_mtu);
5287 		if (rv != 0)
5288 			goto bail;
5289 		rv = mac_maxsdu_update(mh, new_mtu);
5290 		ASSERT(rv == 0);
5291 	}
5292 
5293 bail:
5294 	i_mac_perim_exit(mip);
5295 
5296 	if (rv == 0 && old_mtu_arg != NULL)
5297 		*old_mtu_arg = old_mtu;
5298 	return (rv);
5299 }
5300 
5301 /*
5302  * Return the RX h/w information for the group indexed by grp_num.
5303  */
5304 void
5305 mac_get_hwrxgrp_info(mac_handle_t mh, int grp_index, uint_t *grp_num,
5306     uint_t *n_rings, uint_t *rings, uint_t *type, uint_t *n_clnts,
5307     char *clnts_name)
5308 {
5309 	mac_impl_t *mip = (mac_impl_t *)mh;
5310 	mac_grp_client_t *mcip;
5311 	uint_t i = 0, index = 0;
5312 	mac_ring_t	*ring;
5313 
5314 	/* Revisit when we implement fully dynamic group allocation */
5315 	ASSERT(grp_index >= 0 && grp_index < mip->mi_rx_group_count);
5316 
5317 	rw_enter(&mip->mi_rw_lock, RW_READER);
5318 	*grp_num = mip->mi_rx_groups[grp_index].mrg_index;
5319 	*type = mip->mi_rx_groups[grp_index].mrg_type;
5320 	*n_rings = mip->mi_rx_groups[grp_index].mrg_cur_count;
5321 	ring = mip->mi_rx_groups[grp_index].mrg_rings;
5322 	for (index = 0; index < mip->mi_rx_groups[grp_index].mrg_cur_count;
5323 	    index++) {
5324 		rings[index] = ring->mr_index;
5325 		ring = ring->mr_next;
5326 	}
5327 	/* Assuming the 1st is the default group */
5328 	index = 0;
5329 	if (grp_index == 0) {
5330 		(void) strlcpy(clnts_name, "<default,mcast>,",
5331 		    MAXCLIENTNAMELEN);
5332 		index += strlen("<default,mcast>,");
5333 	}
5334 	for (mcip = mip->mi_rx_groups[grp_index].mrg_clients; mcip != NULL;
5335 	    mcip = mcip->mgc_next) {
5336 		int name_len = strlen(mcip->mgc_client->mci_name);
5337 
5338 		/*
5339 		 * MAXCLIENTNAMELEN is the buffer size reserved for client
5340 		 * names.
5341 		 * XXXX Formating the client name string needs to be moved
5342 		 * to user land when fixing the size of dhi_clnts in
5343 		 * dld_hwgrpinfo_t. We should use n_clients * client_name for
5344 		 * dhi_clntsin instead of MAXCLIENTNAMELEN
5345 		 */
5346 		if (index + name_len >= MAXCLIENTNAMELEN) {
5347 			index = MAXCLIENTNAMELEN;
5348 			break;
5349 		}
5350 		bcopy(mcip->mgc_client->mci_name, &(clnts_name[index]),
5351 		    name_len);
5352 		index += name_len;
5353 		clnts_name[index++] = ',';
5354 		i++;
5355 	}
5356 
5357 	/* Get rid of the last , */
5358 	if (index > 0)
5359 		clnts_name[index - 1] = '\0';
5360 	*n_clnts = i;
5361 	rw_exit(&mip->mi_rw_lock);
5362 }
5363 
5364 /*
5365  * Return the TX h/w information for the group indexed by grp_num.
5366  */
5367 void
5368 mac_get_hwtxgrp_info(mac_handle_t mh, int grp_index, uint_t *grp_num,
5369     uint_t *n_rings, uint_t *rings, uint_t *type, uint_t *n_clnts,
5370     char *clnts_name)
5371 {
5372 	mac_impl_t *mip = (mac_impl_t *)mh;
5373 	mac_grp_client_t *mcip;
5374 	uint_t i = 0, index = 0;
5375 	mac_ring_t	*ring;
5376 
5377 	/* Revisit when we implement fully dynamic group allocation */
5378 	ASSERT(grp_index >= 0 && grp_index <= mip->mi_tx_group_count);
5379 
5380 	rw_enter(&mip->mi_rw_lock, RW_READER);
5381 	*grp_num = mip->mi_tx_groups[grp_index].mrg_index > 0 ?
5382 	    mip->mi_tx_groups[grp_index].mrg_index : grp_index;
5383 	*type = mip->mi_tx_groups[grp_index].mrg_type;
5384 	*n_rings = mip->mi_tx_groups[grp_index].mrg_cur_count;
5385 	ring = mip->mi_tx_groups[grp_index].mrg_rings;
5386 	for (index = 0; index < mip->mi_tx_groups[grp_index].mrg_cur_count;
5387 	    index++) {
5388 		rings[index] = ring->mr_index;
5389 		ring = ring->mr_next;
5390 	}
5391 	index = 0;
5392 	/* Default group has an index of -1 */
5393 	if (mip->mi_tx_groups[grp_index].mrg_index < 0) {
5394 		(void) strlcpy(clnts_name, "<default>,",
5395 		    MAXCLIENTNAMELEN);
5396 		index += strlen("<default>,");
5397 	}
5398 	for (mcip = mip->mi_tx_groups[grp_index].mrg_clients; mcip != NULL;
5399 	    mcip = mcip->mgc_next) {
5400 		int name_len = strlen(mcip->mgc_client->mci_name);
5401 
5402 		/*
5403 		 * MAXCLIENTNAMELEN is the buffer size reserved for client
5404 		 * names.
5405 		 * XXXX Formating the client name string needs to be moved
5406 		 * to user land when fixing the size of dhi_clnts in
5407 		 * dld_hwgrpinfo_t. We should use n_clients * client_name for
5408 		 * dhi_clntsin instead of MAXCLIENTNAMELEN
5409 		 */
5410 		if (index + name_len >= MAXCLIENTNAMELEN) {
5411 			index = MAXCLIENTNAMELEN;
5412 			break;
5413 		}
5414 		bcopy(mcip->mgc_client->mci_name, &(clnts_name[index]),
5415 		    name_len);
5416 		index += name_len;
5417 		clnts_name[index++] = ',';
5418 		i++;
5419 	}
5420 
5421 	/* Get rid of the last , */
5422 	if (index > 0)
5423 		clnts_name[index - 1] = '\0';
5424 	*n_clnts = i;
5425 	rw_exit(&mip->mi_rw_lock);
5426 }
5427 
5428 /*
5429  * Return the group count for RX or TX.
5430  */
5431 uint_t
5432 mac_hwgrp_num(mac_handle_t mh, int type)
5433 {
5434 	mac_impl_t *mip = (mac_impl_t *)mh;
5435 
5436 	/*
5437 	 * Return the Rx and Tx group count; for the Tx we need to
5438 	 * include the default too.
5439 	 */
5440 	return (type == MAC_RING_TYPE_RX ? mip->mi_rx_group_count :
5441 	    mip->mi_tx_groups != NULL ? mip->mi_tx_group_count + 1 : 0);
5442 }
5443 
5444 /*
5445  * The total number of free TX rings for this MAC.
5446  */
5447 uint_t
5448 mac_txavail_get(mac_handle_t mh)
5449 {
5450 	mac_impl_t	*mip = (mac_impl_t *)mh;
5451 
5452 	return (mip->mi_txrings_avail);
5453 }
5454 
5455 /*
5456  * The total number of free RX rings for this MAC.
5457  */
5458 uint_t
5459 mac_rxavail_get(mac_handle_t mh)
5460 {
5461 	mac_impl_t	*mip = (mac_impl_t *)mh;
5462 
5463 	return (mip->mi_rxrings_avail);
5464 }
5465 
5466 /*
5467  * The total number of reserved RX rings on this MAC.
5468  */
5469 uint_t
5470 mac_rxrsvd_get(mac_handle_t mh)
5471 {
5472 	mac_impl_t	*mip = (mac_impl_t *)mh;
5473 
5474 	return (mip->mi_rxrings_rsvd);
5475 }
5476 
5477 /*
5478  * The total number of reserved TX rings on this MAC.
5479  */
5480 uint_t
5481 mac_txrsvd_get(mac_handle_t mh)
5482 {
5483 	mac_impl_t	*mip = (mac_impl_t *)mh;
5484 
5485 	return (mip->mi_txrings_rsvd);
5486 }
5487 
5488 /*
5489  * Total number of free RX groups on this MAC.
5490  */
5491 uint_t
5492 mac_rxhwlnksavail_get(mac_handle_t mh)
5493 {
5494 	mac_impl_t	*mip = (mac_impl_t *)mh;
5495 
5496 	return (mip->mi_rxhwclnt_avail);
5497 }
5498 
5499 /*
5500  * Total number of RX groups reserved on this MAC.
5501  */
5502 uint_t
5503 mac_rxhwlnksrsvd_get(mac_handle_t mh)
5504 {
5505 	mac_impl_t	*mip = (mac_impl_t *)mh;
5506 
5507 	return (mip->mi_rxhwclnt_used);
5508 }
5509 
5510 /*
5511  * Total number of free TX groups on this MAC.
5512  */
5513 uint_t
5514 mac_txhwlnksavail_get(mac_handle_t mh)
5515 {
5516 	mac_impl_t	*mip = (mac_impl_t *)mh;
5517 
5518 	return (mip->mi_txhwclnt_avail);
5519 }
5520 
5521 /*
5522  * Total number of TX groups reserved on this MAC.
5523  */
5524 uint_t
5525 mac_txhwlnksrsvd_get(mac_handle_t mh)
5526 {
5527 	mac_impl_t	*mip = (mac_impl_t *)mh;
5528 
5529 	return (mip->mi_txhwclnt_used);
5530 }
5531 
5532 /*
5533  * Initialize the rings property for a mac client. A non-0 value for
5534  * rxring or txring specifies the number of rings required, a value
5535  * of MAC_RXRINGS_NONE/MAC_TXRINGS_NONE specifies that it doesn't need
5536  * any RX/TX rings and a value of MAC_RXRINGS_DONTCARE/MAC_TXRINGS_DONTCARE
5537  * means the system can decide whether it can give any rings or not.
5538  */
5539 void
5540 mac_client_set_rings(mac_client_handle_t mch, int rxrings, int txrings)
5541 {
5542 	mac_client_impl_t	*mcip = (mac_client_impl_t *)mch;
5543 	mac_resource_props_t	*mrp = MCIP_RESOURCE_PROPS(mcip);
5544 
5545 	if (rxrings != MAC_RXRINGS_DONTCARE) {
5546 		mrp->mrp_mask |= MRP_RX_RINGS;
5547 		mrp->mrp_nrxrings = rxrings;
5548 	}
5549 
5550 	if (txrings != MAC_TXRINGS_DONTCARE) {
5551 		mrp->mrp_mask |= MRP_TX_RINGS;
5552 		mrp->mrp_ntxrings = txrings;
5553 	}
5554 }
5555