xref: /titanic_52/usr/src/uts/common/io/softmac/softmac_main.c (revision 6a1af1a67532df169a657cce07140be64bdea084)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 /*
27  * The softmac driver is used to "unify" non-GLDv3 drivers to the GLDv3
28  * framework.  It also creates the kernel datalink structure for each
29  * physical network device.
30  *
31  * Specifically, a softmac will be created for each physical network device
32  * (dip) during the device's post-attach process.  When this softmac is
33  * created, the following will also be done:
34  *   - create the device's <link name, linkid> mapping;
35  *   - register the mac if this is a non-GLDv3 device and the media type is
36  *     supported by the GLDv3 framework;
37  *   - create the kernel data-link structure for this physical device;
38  *
39  * This softmac will be destroyed during the device's pre-detach process,
40  * and all the above will be undone.
41  */
42 
43 #include <sys/types.h>
44 #include <sys/file.h>
45 #include <sys/cred.h>
46 #include <sys/dlpi.h>
47 #include <sys/mac_provider.h>
48 #include <sys/disp.h>
49 #include <sys/sunndi.h>
50 #include <sys/modhash.h>
51 #include <sys/stropts.h>
52 #include <sys/sysmacros.h>
53 #include <sys/vlan.h>
54 #include <sys/softmac_impl.h>
55 #include <sys/softmac.h>
56 #include <sys/dls.h>
57 
58 /* Used as a parameter to the mod hash walk of softmac structures */
59 typedef struct {
60 	softmac_t	*smw_softmac;
61 	boolean_t	smw_retry;
62 } softmac_walk_t;
63 
64 /*
65  * Softmac hash table including softmacs for both style-2 and style-1 devices.
66  */
67 static krwlock_t	softmac_hash_lock;
68 static mod_hash_t	*softmac_hash;
69 static kmutex_t		smac_global_lock;
70 static kcondvar_t	smac_global_cv;
71 
72 static kmem_cache_t	*softmac_cachep;
73 
74 #define	SOFTMAC_HASHSZ		64
75 
76 static void softmac_create_task(void *);
77 static void softmac_mac_register(softmac_t *);
78 static int softmac_create_datalink(softmac_t *);
79 static int softmac_m_start(void *);
80 static void softmac_m_stop(void *);
81 static int softmac_m_open(void *);
82 static void softmac_m_close(void *);
83 static boolean_t softmac_m_getcapab(void *, mac_capab_t, void *);
84 static int softmac_m_setprop(void *, const char *, mac_prop_id_t,
85     uint_t, const void *);
86 static int softmac_m_getprop(void *, const char *, mac_prop_id_t,
87     uint_t, void *);
88 static void softmac_m_propinfo(void *, const char *, mac_prop_id_t,
89     mac_prop_info_handle_t);
90 
91 #define	SOFTMAC_M_CALLBACK_FLAGS	\
92 	(MC_IOCTL | MC_GETCAPAB | MC_OPEN | MC_CLOSE | MC_SETPROP | \
93 	MC_GETPROP | MC_PROPINFO)
94 
95 static mac_callbacks_t softmac_m_callbacks = {
96 	SOFTMAC_M_CALLBACK_FLAGS,
97 	softmac_m_stat,
98 	softmac_m_start,
99 	softmac_m_stop,
100 	softmac_m_promisc,
101 	softmac_m_multicst,
102 	softmac_m_unicst,
103 	softmac_m_tx,
104 	NULL,
105 	softmac_m_ioctl,
106 	softmac_m_getcapab,
107 	softmac_m_open,
108 	softmac_m_close,
109 	softmac_m_setprop,
110 	softmac_m_getprop,
111 	softmac_m_propinfo
112 };
113 
114 /*ARGSUSED*/
115 static int
116 softmac_constructor(void *buf, void *arg, int kmflag)
117 {
118 	softmac_t	*softmac = buf;
119 
120 	bzero(buf, sizeof (softmac_t));
121 	mutex_init(&softmac->smac_mutex, NULL, MUTEX_DEFAULT, NULL);
122 	mutex_init(&softmac->smac_active_mutex, NULL, MUTEX_DEFAULT, NULL);
123 	mutex_init(&softmac->smac_fp_mutex, NULL, MUTEX_DEFAULT, NULL);
124 	cv_init(&softmac->smac_cv, NULL, CV_DEFAULT, NULL);
125 	cv_init(&softmac->smac_fp_cv, NULL, CV_DEFAULT, NULL);
126 	list_create(&softmac->smac_sup_list, sizeof (softmac_upper_t),
127 	    offsetof(softmac_upper_t, su_list_node));
128 	return (0);
129 }
130 
131 /*ARGSUSED*/
132 static void
133 softmac_destructor(void *buf, void *arg)
134 {
135 	softmac_t	*softmac = buf;
136 
137 	ASSERT(softmac->smac_fp_disable_clients == 0);
138 	ASSERT(!softmac->smac_fastpath_admin_disabled);
139 
140 	ASSERT(!(softmac->smac_flags & SOFTMAC_ATTACH_DONE));
141 	ASSERT(softmac->smac_hold_cnt == 0);
142 	ASSERT(softmac->smac_attachok_cnt == 0);
143 	ASSERT(softmac->smac_mh == NULL);
144 	ASSERT(softmac->smac_softmac[0] == NULL &&
145 	    softmac->smac_softmac[1] == NULL);
146 	ASSERT(softmac->smac_state == SOFTMAC_INITIALIZED);
147 	ASSERT(softmac->smac_lower == NULL);
148 	ASSERT(softmac->smac_active == B_FALSE);
149 	ASSERT(softmac->smac_nactive == 0);
150 	ASSERT(list_is_empty(&softmac->smac_sup_list));
151 
152 	list_destroy(&softmac->smac_sup_list);
153 	mutex_destroy(&softmac->smac_mutex);
154 	mutex_destroy(&softmac->smac_active_mutex);
155 	mutex_destroy(&softmac->smac_fp_mutex);
156 	cv_destroy(&softmac->smac_cv);
157 	cv_destroy(&softmac->smac_fp_cv);
158 }
159 
160 void
161 softmac_init()
162 {
163 	softmac_hash = mod_hash_create_extended("softmac_hash",
164 	    SOFTMAC_HASHSZ, mod_hash_null_keydtor, mod_hash_null_valdtor,
165 	    mod_hash_bystr, NULL, mod_hash_strkey_cmp, KM_SLEEP);
166 
167 	rw_init(&softmac_hash_lock, NULL, RW_DEFAULT, NULL);
168 	mutex_init(&smac_global_lock, NULL, MUTEX_DRIVER, NULL);
169 	cv_init(&smac_global_cv, NULL, CV_DRIVER, NULL);
170 
171 	softmac_cachep = kmem_cache_create("softmac_cache",
172 	    sizeof (softmac_t), 0, softmac_constructor,
173 	    softmac_destructor, NULL, NULL, NULL, 0);
174 	ASSERT(softmac_cachep != NULL);
175 	softmac_fp_init();
176 }
177 
178 void
179 softmac_fini()
180 {
181 	softmac_fp_fini();
182 	kmem_cache_destroy(softmac_cachep);
183 	rw_destroy(&softmac_hash_lock);
184 	mod_hash_destroy_hash(softmac_hash);
185 	mutex_destroy(&smac_global_lock);
186 	cv_destroy(&smac_global_cv);
187 }
188 
189 /* ARGSUSED */
190 static uint_t
191 softmac_exist(mod_hash_key_t key, mod_hash_val_t *val, void *arg)
192 {
193 	boolean_t *pexist = arg;
194 
195 	*pexist = B_TRUE;
196 	return (MH_WALK_TERMINATE);
197 }
198 
199 boolean_t
200 softmac_busy()
201 {
202 	boolean_t exist = B_FALSE;
203 
204 	rw_enter(&softmac_hash_lock, RW_READER);
205 	mod_hash_walk(softmac_hash, softmac_exist, &exist);
206 	rw_exit(&softmac_hash_lock);
207 	return (exist);
208 }
209 
210 /*
211  *
212  * softmac_create() is called for each minor node during the post-attach of
213  * each DDI_NT_NET device instance.  Note that it is possible that a device
214  * instance has two minor nodes (DLPI style-1 and style-2), so that for that
215  * specific device, softmac_create() could be called twice.
216  *
217  * A softmac_t is used to track each DDI_NT_NET device, and a softmac_dev_t
218  * is created to track each minor node.
219  *
220  * For each minor node of a legacy device, a taskq is started to finish
221  * softmac_mac_register(), which will finish the rest of work (see comments
222  * above softmac_mac_register()).
223  *
224  *			softmac state machine
225  * --------------------------------------------------------------------------
226  * OLD STATE		EVENT					NEW STATE
227  * --------------------------------------------------------------------------
228  * UNINIT		attach of 1st minor node 		ATTACH_INPROG
229  * okcnt = 0		net_postattach -> softmac_create	okcnt = 1
230  *
231  * ATTACH_INPROG	attach of 2nd minor node (GLDv3)	ATTACH_DONE
232  * okcnt = 1		net_postattach -> softmac_create	okcnt = 2
233  *
234  * ATTACH_INPROG	attach of 2nd minor node (legacy)	ATTACH_INPROG
235  * okcnt = 1		net_postattach -> softmac_create	okcnt = 2
236  *			schedule softmac_mac_register
237  *
238  * ATTACH_INPROG	legacy device node			ATTACH_DONE
239  * okcnt = 2		softmac_mac_register			okcnt = 2
240  *
241  * ATTACH_DONE		detach of 1st minor node		DETACH_INPROG
242  * okcnt = 2		(success)				okcnt = 1
243  *
244  * DETACH_INPROG	detach of 2nd minor node		UNINIT (or free)
245  * okcnt = 1		(success)				okcnt = 0
246  *
247  * ATTACH_DONE		detach failure				state unchanged
248  * DETACH_INPROG						left = okcnt
249  *
250  * DETACH_INPROG	reattach				ATTACH_INPROG
251  * okcnt = 0,1		net_postattach -> softmac_create
252  *
253  * ATTACH_DONE		reattach				ATTACH_DONE
254  * left != 0		net_postattach -> softmac_create	left = 0
255  *
256  * Abbreviation notes:
257  * states have SOFTMAC_ prefix,
258  * okcnt - softmac_attach_okcnt,
259  * left - softmac_attached_left
260  */
261 
262 #ifdef DEBUG
263 void
264 softmac_state_verify(softmac_t *softmac)
265 {
266 	ASSERT(MUTEX_HELD(&softmac->smac_mutex));
267 
268 	/*
269 	 * There are at most 2 minor nodes, one per DLPI style
270 	 */
271 	ASSERT(softmac->smac_cnt <= 2 && softmac->smac_attachok_cnt <= 2);
272 
273 	/*
274 	 * The smac_attachok_cnt represents the number of attaches i.e. the
275 	 * number of times net_postattach -> softmac_create() has been called
276 	 * for a device instance.
277 	 */
278 	ASSERT(softmac->smac_attachok_cnt == SMAC_NONZERO_NODECNT(softmac));
279 
280 	/*
281 	 * softmac_create (or softmac_mac_register) ->  softmac_create_datalink
282 	 * happens only after all minor nodes have been attached
283 	 */
284 	ASSERT(softmac->smac_state != SOFTMAC_ATTACH_DONE ||
285 	    softmac->smac_attachok_cnt == softmac->smac_cnt);
286 
287 	if (softmac->smac_attachok_cnt == 0) {
288 		ASSERT(softmac->smac_state == SOFTMAC_UNINIT);
289 		ASSERT(softmac->smac_mh == NULL);
290 	} else if (softmac->smac_attachok_cnt < softmac->smac_cnt) {
291 		ASSERT(softmac->smac_state == SOFTMAC_ATTACH_INPROG ||
292 		    softmac->smac_state == SOFTMAC_DETACH_INPROG);
293 		ASSERT(softmac->smac_mh == NULL);
294 	} else {
295 		/*
296 		 * In the stable condition the state whould be
297 		 * SOFTMAC_ATTACH_DONE. But there is a small transient window
298 		 * in softmac_destroy where we change the state to
299 		 * SOFTMAC_DETACH_INPROG and drop the lock before doing
300 		 * the link destroy
301 		 */
302 		ASSERT(softmac->smac_attachok_cnt == softmac->smac_cnt);
303 		ASSERT(softmac->smac_state != SOFTMAC_UNINIT);
304 	}
305 	if (softmac->smac_mh != NULL)
306 		ASSERT(softmac->smac_attachok_cnt == softmac->smac_cnt);
307 }
308 #endif
309 
310 #ifdef DEBUG
311 #define	SOFTMAC_STATE_VERIFY(softmac)	softmac_state_verify(softmac)
312 #else
313 #define	SOFTMAC_STATE_VERIFY(softmac)
314 #endif
315 
316 int
317 softmac_create(dev_info_t *dip, dev_t dev)
318 {
319 	char		devname[MAXNAMELEN];
320 	softmac_t	*softmac;
321 	softmac_dev_t	*softmac_dev = NULL;
322 	int		index;
323 	int		ppa, err = 0;
324 
325 	/*
326 	 * Force the softmac driver to be attached.
327 	 */
328 	if (i_ddi_attach_pseudo_node(SOFTMAC_DEV_NAME) == NULL) {
329 		cmn_err(CE_WARN, "softmac_create:softmac attach fails");
330 		return (ENXIO);
331 	}
332 
333 	if (GLDV3_DRV(ddi_driver_major(dip))) {
334 		minor_t minor = getminor(dev);
335 		/*
336 		 * For GLDv3, we don't care about the DLPI style 2
337 		 * compatibility node.  (We know that all such devices
338 		 * have style 1 nodes.)
339 		 */
340 		if ((strcmp(ddi_driver_name(dip), "clone") == 0) ||
341 		    (getmajor(dev) == ddi_name_to_major("clone")) ||
342 		    (minor == 0)) {
343 			return (0);
344 		}
345 
346 		/*
347 		 * Likewise, we know that the minor number for DLPI style 1
348 		 * nodes is constrained to a maximum value.
349 		 */
350 		if (minor >= DLS_MAX_MINOR) {
351 			return (ENOTSUP);
352 		}
353 		/*
354 		 * Otherwise we can decode the instance from the minor number,
355 		 * which allows for situations with multiple mac instances
356 		 * for a single dev_info_t.
357 		 */
358 		ppa = DLS_MINOR2INST(minor);
359 	} else {
360 		/*
361 		 * For legacy drivers, we just have to limit them to
362 		 * two minor nodes, one style 1 and one style 2, and
363 		 * we assume the ddi_get_instance() is the PPA.
364 		 * Drivers that need more flexibility should be ported
365 		 * to GLDv3.
366 		 */
367 		ppa = ddi_get_instance(dip);
368 		if (i_ddi_minor_node_count(dip, DDI_NT_NET) > 2) {
369 			cmn_err(CE_WARN, "%s has more than 2 minor nodes; "
370 			    "unsupported", devname);
371 			return (ENOTSUP);
372 		}
373 	}
374 
375 	(void) snprintf(devname, MAXNAMELEN, "%s%d", ddi_driver_name(dip), ppa);
376 
377 	/*
378 	 * Check whether the softmac for the specified device already exists
379 	 */
380 	rw_enter(&softmac_hash_lock, RW_WRITER);
381 	if ((mod_hash_find(softmac_hash, (mod_hash_key_t)devname,
382 	    (mod_hash_val_t *)&softmac)) != 0) {
383 
384 		softmac = kmem_cache_alloc(softmac_cachep, KM_SLEEP);
385 		(void) strlcpy(softmac->smac_devname, devname, MAXNAMELEN);
386 
387 		err = mod_hash_insert(softmac_hash,
388 		    (mod_hash_key_t)softmac->smac_devname,
389 		    (mod_hash_val_t)softmac);
390 		ASSERT(err == 0);
391 		mutex_enter(&smac_global_lock);
392 		cv_broadcast(&smac_global_cv);
393 		mutex_exit(&smac_global_lock);
394 	}
395 
396 	mutex_enter(&softmac->smac_mutex);
397 	SOFTMAC_STATE_VERIFY(softmac);
398 	if (softmac->smac_state != SOFTMAC_ATTACH_DONE)
399 		softmac->smac_state = SOFTMAC_ATTACH_INPROG;
400 	if (softmac->smac_attachok_cnt == 0) {
401 		/*
402 		 * Initialize the softmac if this is the post-attach of the
403 		 * first minor node.
404 		 */
405 		softmac->smac_flags = 0;
406 		softmac->smac_umajor = ddi_driver_major(dip);
407 		softmac->smac_uppa = ppa;
408 
409 		/*
410 		 * For GLDv3, we ignore the style 2 node (see the logic
411 		 * above on that), and we should have exactly one attach
412 		 * per MAC instance (possibly more than one per dev_info_t).
413 		 */
414 		if (GLDV3_DRV(ddi_driver_major(dip))) {
415 			softmac->smac_flags |= SOFTMAC_GLDV3;
416 			softmac->smac_cnt = 1;
417 		} else {
418 			softmac->smac_cnt =
419 			    i_ddi_minor_node_count(dip, DDI_NT_NET);
420 		}
421 	}
422 
423 	index = (getmajor(dev) == ddi_name_to_major("clone"));
424 	if (softmac->smac_softmac[index] != NULL) {
425 		/*
426 		 * This is possible if the post_attach() is called after
427 		 * pre_detach() fails. This seems to be a defect of the DACF
428 		 * framework. We work around it by using a smac_attached_left
429 		 * field that tracks this
430 		 */
431 		ASSERT(softmac->smac_attached_left != 0);
432 		softmac->smac_attached_left--;
433 		mutex_exit(&softmac->smac_mutex);
434 		rw_exit(&softmac_hash_lock);
435 		return (0);
436 
437 	}
438 	mutex_exit(&softmac->smac_mutex);
439 	rw_exit(&softmac_hash_lock);
440 
441 	softmac_dev = kmem_zalloc(sizeof (softmac_dev_t), KM_SLEEP);
442 	softmac_dev->sd_dev = dev;
443 
444 	mutex_enter(&softmac->smac_mutex);
445 	softmac->smac_softmac[index] = softmac_dev;
446 	/*
447 	 * Continue to register the mac and create the datalink only when all
448 	 * the minor nodes are attached.
449 	 */
450 	if (++softmac->smac_attachok_cnt != softmac->smac_cnt) {
451 		mutex_exit(&softmac->smac_mutex);
452 		return (0);
453 	}
454 
455 	/*
456 	 * All of the minor nodes have been attached; start a taskq
457 	 * to do the rest of the work.  We use a taskq instead of
458 	 * doing the work here because:
459 	 *
460 	 * We could be called as a result of a open() system call
461 	 * where spec_open() already SLOCKED the snode. Using a taskq
462 	 * sidesteps the risk that our ldi_open_by_dev() call would
463 	 * deadlock trying to set SLOCKED on the snode again.
464 	 *
465 	 * The devfs design requires that the downcalls don't use any
466 	 * interruptible cv_wait which happens when we do door upcalls.
467 	 * Otherwise the downcalls which may be holding devfs resources
468 	 * may cause a deadlock if the thread is stopped. Also we need to make
469 	 * sure these downcalls into softmac_create or softmac_destroy
470 	 * don't cv_wait on any devfs related condition. Thus softmac_destroy
471 	 * returns EBUSY if the asynchronous threads started in softmac_create
472 	 * haven't finished.
473 	 */
474 	(void) taskq_dispatch(system_taskq, softmac_create_task,
475 	    softmac, TQ_SLEEP);
476 	mutex_exit(&softmac->smac_mutex);
477 	return (0);
478 }
479 
480 static boolean_t
481 softmac_m_getcapab(void *arg, mac_capab_t cap, void *cap_data)
482 {
483 	softmac_t *softmac = arg;
484 
485 	if (!(softmac->smac_capab_flags & cap))
486 		return (B_FALSE);
487 
488 	switch (cap) {
489 	case MAC_CAPAB_HCKSUM: {
490 		uint32_t *txflags = cap_data;
491 
492 		*txflags = softmac->smac_hcksum_txflags;
493 		break;
494 	}
495 	case MAC_CAPAB_LEGACY: {
496 		mac_capab_legacy_t *legacy = cap_data;
497 
498 		/*
499 		 * The caller is not interested in the details.
500 		 */
501 		if (legacy == NULL)
502 			break;
503 
504 		legacy->ml_unsup_note = ~softmac->smac_notifications &
505 		    (DL_NOTE_LINK_UP | DL_NOTE_LINK_DOWN | DL_NOTE_SPEED);
506 		legacy->ml_active_set = softmac_active_set;
507 		legacy->ml_active_clear = softmac_active_clear;
508 		legacy->ml_fastpath_disable = softmac_fastpath_disable;
509 		legacy->ml_fastpath_enable = softmac_fastpath_enable;
510 		legacy->ml_dev = makedevice(softmac->smac_umajor,
511 		    softmac->smac_uppa + 1);
512 		break;
513 	}
514 
515 	/*
516 	 * For the capabilities below, there's nothing for us to fill in;
517 	 * simply return B_TRUE if we support it.
518 	 */
519 	case MAC_CAPAB_NO_ZCOPY:
520 	case MAC_CAPAB_NO_NATIVEVLAN:
521 	default:
522 		break;
523 	}
524 	return (B_TRUE);
525 }
526 
527 static int
528 softmac_update_info(softmac_t *softmac, datalink_id_t *linkidp)
529 {
530 	datalink_id_t	linkid = DATALINK_INVALID_LINKID;
531 	uint32_t	media;
532 	int		err;
533 
534 	if ((err = dls_mgmt_update(softmac->smac_devname, softmac->smac_media,
535 	    softmac->smac_flags & SOFTMAC_NOSUPP, &media, &linkid)) == 0) {
536 		*linkidp = linkid;
537 	}
538 
539 	if (err == EEXIST) {
540 		/*
541 		 * There is a link name conflict.  Either:
542 		 *
543 		 * - An existing link with the same device name with a
544 		 *   different media type from of the given type.
545 		 *   Mark this link back to persistent only; or
546 		 *
547 		 * - We cannot assign the "suggested" name because
548 		 *   GLDv3 and therefore vanity naming is not supported
549 		 *   for this link type. Delete this link's <link name,
550 		 *   linkid> mapping.
551 		 */
552 		if (media != softmac->smac_media) {
553 			cmn_err(CE_WARN, "%s device %s conflicts with "
554 			    "existing %s device %s.",
555 			    dl_mactypestr(softmac->smac_media),
556 			    softmac->smac_devname, dl_mactypestr(media),
557 			    softmac->smac_devname);
558 			(void) dls_mgmt_destroy(linkid, B_FALSE);
559 		} else {
560 			cmn_err(CE_WARN, "link name %s is already in-use.",
561 			    softmac->smac_devname);
562 			(void) dls_mgmt_destroy(linkid, B_TRUE);
563 		}
564 
565 		cmn_err(CE_WARN, "%s device might not be available "
566 		    "for use.", softmac->smac_devname);
567 		cmn_err(CE_WARN, "See dladm(1M) for more information.");
568 	}
569 
570 	return (err);
571 }
572 
573 /*
574  * This function:
575  * 1. provides the link's media type to dlmgmtd.
576  * 2. creates the GLDv3 datalink if the media type is supported by GLDv3.
577  */
578 static int
579 softmac_create_datalink(softmac_t *softmac)
580 {
581 	datalink_id_t	linkid = DATALINK_INVALID_LINKID;
582 	int		err;
583 
584 	/*
585 	 * Inform dlmgmtd of this link so that softmac_hold_device() is able
586 	 * to know the existence of this link. If this failed with EBADF,
587 	 * it might be because dlmgmtd was not started in time (e.g.,
588 	 * diskless boot); ignore the failure and continue to create
589 	 * the GLDv3 datalink if needed.
590 	 */
591 	err = dls_mgmt_create(softmac->smac_devname,
592 	    makedevice(softmac->smac_umajor, softmac->smac_uppa + 1),
593 	    DATALINK_CLASS_PHYS, DL_OTHER, B_TRUE, &linkid);
594 	if (err != 0 && err != EBADF)
595 		return (err);
596 
597 	/*
598 	 * Provide the media type of the physical link to dlmgmtd.
599 	 */
600 	if ((err != EBADF) &&
601 	    ((err = softmac_update_info(softmac, &linkid)) != 0)) {
602 		return (err);
603 	}
604 
605 	/*
606 	 * Create the GLDv3 datalink.
607 	 */
608 	if (!(softmac->smac_flags & SOFTMAC_NOSUPP)) {
609 		err = dls_devnet_create(softmac->smac_mh, linkid,
610 		    crgetzoneid(CRED()));
611 		if (err != 0) {
612 			cmn_err(CE_WARN, "dls_devnet_create failed for %s",
613 			    softmac->smac_devname);
614 			return (err);
615 		}
616 	}
617 
618 	if (linkid == DATALINK_INVALID_LINKID) {
619 		mutex_enter(&softmac->smac_mutex);
620 		softmac->smac_flags |= SOFTMAC_NEED_RECREATE;
621 		mutex_exit(&softmac->smac_mutex);
622 	}
623 
624 	return (0);
625 }
626 
627 static void
628 softmac_create_task(void *arg)
629 {
630 	softmac_t	*softmac = arg;
631 	mac_handle_t	mh;
632 	int		err;
633 
634 	if (!GLDV3_DRV(softmac->smac_umajor)) {
635 		softmac_mac_register(softmac);
636 		return;
637 	}
638 
639 	if ((err = mac_open(softmac->smac_devname, &mh)) != 0)
640 		goto done;
641 
642 	mutex_enter(&softmac->smac_mutex);
643 	softmac->smac_media = (mac_info(mh))->mi_nativemedia;
644 	softmac->smac_mh = mh;
645 	mutex_exit(&softmac->smac_mutex);
646 
647 	/*
648 	 * We can safely release the reference on the mac because
649 	 * this mac will only be unregistered and destroyed when
650 	 * the device detaches, and the softmac will be destroyed
651 	 * before then (in the pre-detach routine of the device).
652 	 */
653 	mac_close(mh);
654 
655 	/*
656 	 * Create the GLDv3 datalink for this mac.
657 	 */
658 	err = softmac_create_datalink(softmac);
659 
660 done:
661 	mutex_enter(&softmac->smac_mutex);
662 	if (err != 0)
663 		softmac->smac_mh = NULL;
664 	softmac->smac_attacherr = err;
665 	softmac->smac_state = SOFTMAC_ATTACH_DONE;
666 	cv_broadcast(&softmac->smac_cv);
667 	mutex_exit(&softmac->smac_mutex);
668 }
669 
670 /*
671  * This function is only called for legacy devices. It:
672  * 1. registers the MAC for the legacy devices whose media type is supported
673  *    by the GLDv3 framework.
674  * 2. creates the GLDv3 datalink if the media type is supported by GLDv3.
675  */
676 static void
677 softmac_mac_register(softmac_t *softmac)
678 {
679 	softmac_dev_t	*softmac_dev;
680 	dev_t		dev;
681 	ldi_handle_t	lh = NULL;
682 	ldi_ident_t	li = NULL;
683 	int		index;
684 	boolean_t	native_vlan = B_FALSE;
685 	int		err;
686 
687 	/*
688 	 * Note that we do not need any locks to access this softmac pointer,
689 	 * as softmac_destroy() will wait until this function is called.
690 	 */
691 	ASSERT(softmac != NULL);
692 	ASSERT(softmac->smac_state == SOFTMAC_ATTACH_INPROG &&
693 	    softmac->smac_attachok_cnt == softmac->smac_cnt);
694 
695 	if ((err = ldi_ident_from_dip(softmac_dip, &li)) != 0) {
696 		mutex_enter(&softmac->smac_mutex);
697 		goto done;
698 	}
699 
700 	/*
701 	 * Determine whether this legacy device support VLANs by opening
702 	 * the style-2 device node (if it exists) and attaching to a VLAN
703 	 * PPA (1000 + ppa).
704 	 */
705 	dev = makedevice(ddi_name_to_major("clone"), softmac->smac_umajor);
706 	err = ldi_open_by_dev(&dev, OTYP_CHR, FREAD|FWRITE, kcred, &lh, li);
707 	if (err == 0) {
708 		if (dl_attach(lh, softmac->smac_uppa + 1 * 1000, NULL) == 0)
709 			native_vlan = B_TRUE;
710 		(void) ldi_close(lh, FREAD|FWRITE, kcred);
711 	}
712 
713 	err = EINVAL;
714 	for (index = 0; index < 2; index++) {
715 		dl_info_ack_t	dlia;
716 		dl_error_ack_t	dlea;
717 		uint32_t	notes;
718 		struct strioctl	iocb;
719 		uint32_t	margin;
720 		int		rval;
721 
722 		if ((softmac_dev = softmac->smac_softmac[index]) == NULL)
723 			continue;
724 
725 		softmac->smac_dev = dev = softmac_dev->sd_dev;
726 		if (ldi_open_by_dev(&dev, OTYP_CHR, FREAD|FWRITE, kcred, &lh,
727 		    li) != 0) {
728 			continue;
729 		}
730 
731 		/*
732 		 * Pop all the intermediate modules in order to negotiate
733 		 * capabilities correctly.
734 		 */
735 		while (ldi_ioctl(lh, I_POP, 0, FKIOCTL, kcred, &rval) == 0)
736 			;
737 
738 		/* DLPI style-1 or DLPI style-2? */
739 		if ((rval = dl_info(lh, &dlia, NULL, NULL, &dlea)) != 0) {
740 			if (rval == ENOTSUP) {
741 				cmn_err(CE_NOTE, "softmac: received "
742 				    "DL_ERROR_ACK to DL_INFO_ACK; "
743 				    "DLPI errno 0x%x, UNIX errno %d",
744 				    dlea.dl_errno, dlea.dl_unix_errno);
745 			}
746 			(void) ldi_close(lh, FREAD|FWRITE, kcred);
747 			continue;
748 		}
749 
750 		/*
751 		 * Currently only DL_ETHER has GLDv3 mac plugin support.
752 		 * For media types that GLDv3 does not support, create a
753 		 * link id for it.
754 		 */
755 		if ((softmac->smac_media = dlia.dl_mac_type) != DL_ETHER) {
756 			(void) ldi_close(lh, FREAD|FWRITE, kcred);
757 			err = 0;
758 			break;
759 		}
760 
761 		if ((dlia.dl_provider_style == DL_STYLE2) &&
762 		    (dl_attach(lh, softmac->smac_uppa, NULL) != 0)) {
763 			(void) ldi_close(lh, FREAD|FWRITE, kcred);
764 			continue;
765 		}
766 
767 		if ((rval = dl_bind(lh, 0, NULL)) != 0) {
768 			if (rval == ENOTSUP) {
769 				cmn_err(CE_NOTE, "softmac: received "
770 				    "DL_ERROR_ACK to DL_BIND_ACK; "
771 				    "DLPI errno 0x%x, UNIX errno %d",
772 				    dlea.dl_errno, dlea.dl_unix_errno);
773 			}
774 			(void) ldi_close(lh, FREAD|FWRITE, kcred);
775 			continue;
776 		}
777 
778 		/*
779 		 * Call dl_info() after dl_bind() because some drivers only
780 		 * provide correct information (e.g. MAC address) once bound.
781 		 */
782 		softmac->smac_addrlen = sizeof (softmac->smac_unicst_addr);
783 		if ((rval = dl_info(lh, &dlia, softmac->smac_unicst_addr,
784 		    &softmac->smac_addrlen, &dlea)) != 0) {
785 			if (rval == ENOTSUP) {
786 				cmn_err(CE_NOTE, "softmac: received "
787 				    "DL_ERROR_ACK to DL_INFO_ACK; "
788 				    "DLPI errno 0x%x, UNIX errno %d",
789 				    dlea.dl_errno, dlea.dl_unix_errno);
790 			}
791 			(void) ldi_close(lh, FREAD|FWRITE, kcred);
792 			continue;
793 		}
794 
795 		softmac->smac_style = dlia.dl_provider_style;
796 		softmac->smac_saplen = ABS(dlia.dl_sap_length);
797 		softmac->smac_min_sdu = dlia.dl_min_sdu;
798 		softmac->smac_max_sdu = dlia.dl_max_sdu;
799 
800 		if ((softmac->smac_saplen != sizeof (uint16_t)) ||
801 		    (softmac->smac_addrlen != ETHERADDRL) ||
802 		    (dlia.dl_brdcst_addr_length != ETHERADDRL) ||
803 		    (dlia.dl_brdcst_addr_offset == 0)) {
804 			(void) ldi_close(lh, FREAD|FWRITE, kcred);
805 			continue;
806 		}
807 
808 		/*
809 		 * Check other DLPI capabilities. Note that this must be after
810 		 * dl_bind() because some drivers return DL_ERROR_ACK if the
811 		 * stream is not bound. It is also before mac_register(), so
812 		 * we don't need any lock protection here.
813 		 */
814 		softmac->smac_capab_flags =
815 		    (MAC_CAPAB_NO_ZCOPY | MAC_CAPAB_LEGACY);
816 
817 		softmac->smac_no_capability_req = B_FALSE;
818 		if (softmac_fill_capab(lh, softmac) != 0)
819 			softmac->smac_no_capability_req = B_TRUE;
820 
821 		/*
822 		 * Check the margin of the underlying driver.
823 		 */
824 		margin = 0;
825 		iocb.ic_cmd = DLIOCMARGININFO;
826 		iocb.ic_timout = INFTIM;
827 		iocb.ic_len = sizeof (margin);
828 		iocb.ic_dp = (char *)&margin;
829 		softmac->smac_margin = 0;
830 
831 		if (ldi_ioctl(lh, I_STR, (intptr_t)&iocb, FKIOCTL, kcred,
832 		    &rval) == 0) {
833 			softmac->smac_margin = margin;
834 		}
835 
836 		/*
837 		 * If the legacy driver doesn't support DLIOCMARGININFO, but
838 		 * it can support native VLAN, correct its margin value to 4.
839 		 */
840 		if (native_vlan) {
841 			if (softmac->smac_margin == 0)
842 				softmac->smac_margin = VLAN_TAGSZ;
843 		} else {
844 			softmac->smac_capab_flags |= MAC_CAPAB_NO_NATIVEVLAN;
845 		}
846 
847 		/*
848 		 * Not all drivers support DL_NOTIFY_REQ, so ignore ENOTSUP.
849 		 */
850 		softmac->smac_notifications = 0;
851 		notes = DL_NOTE_PHYS_ADDR | DL_NOTE_LINK_UP | DL_NOTE_LINK_DOWN;
852 		switch (dl_notify(lh, &notes, NULL)) {
853 		case 0:
854 			softmac->smac_notifications = notes;
855 			break;
856 		case ENOTSUP:
857 			break;
858 		default:
859 			(void) ldi_close(lh, FREAD|FWRITE, kcred);
860 			continue;
861 		}
862 
863 		(void) ldi_close(lh, FREAD|FWRITE, kcred);
864 		err = 0;
865 		break;
866 	}
867 	ldi_ident_release(li);
868 
869 	mutex_enter(&softmac->smac_mutex);
870 
871 	if (err != 0)
872 		goto done;
873 
874 	if (softmac->smac_media != DL_ETHER)
875 		softmac->smac_flags |= SOFTMAC_NOSUPP;
876 
877 	/*
878 	 * Finally, we're ready to register ourselves with the MAC layer
879 	 * interface; if this succeeds, we're all ready to start()
880 	 */
881 	if (!(softmac->smac_flags & SOFTMAC_NOSUPP)) {
882 		mac_register_t	*macp;
883 
884 		if ((macp = mac_alloc(MAC_VERSION)) == NULL) {
885 			err = ENOMEM;
886 			goto done;
887 		}
888 
889 		macp->m_type_ident = MAC_PLUGIN_IDENT_ETHER;
890 		macp->m_driver = softmac;
891 		macp->m_dip = softmac_dip;
892 
893 		macp->m_margin = softmac->smac_margin;
894 		macp->m_src_addr = softmac->smac_unicst_addr;
895 		macp->m_min_sdu = softmac->smac_min_sdu;
896 		macp->m_max_sdu = softmac->smac_max_sdu;
897 		macp->m_callbacks = &softmac_m_callbacks;
898 		macp->m_instance = (uint_t)-1;
899 
900 		err = mac_register(macp, &softmac->smac_mh);
901 		mac_free(macp);
902 		if (err != 0) {
903 			cmn_err(CE_WARN, "mac_register failed for %s",
904 			    softmac->smac_devname);
905 			goto done;
906 		}
907 	}
908 	mutex_exit(&softmac->smac_mutex);
909 
910 	/*
911 	 * Try to create the datalink for this softmac.
912 	 */
913 	if ((err = softmac_create_datalink(softmac)) != 0) {
914 		if (!(softmac->smac_flags & SOFTMAC_NOSUPP))
915 			(void) mac_unregister(softmac->smac_mh);
916 		mutex_enter(&softmac->smac_mutex);
917 		softmac->smac_mh = NULL;
918 		goto done;
919 	}
920 	/*
921 	 * If succeed, create the thread which handles the DL_NOTIFY_IND from
922 	 * the lower stream.
923 	 */
924 	mutex_enter(&softmac->smac_mutex);
925 	if (softmac->smac_mh != NULL) {
926 		softmac->smac_notify_thread = thread_create(NULL, 0,
927 		    softmac_notify_thread, softmac, 0, &p0,
928 		    TS_RUN, minclsyspri);
929 	}
930 
931 done:
932 	ASSERT(softmac->smac_state == SOFTMAC_ATTACH_INPROG &&
933 	    softmac->smac_attachok_cnt == softmac->smac_cnt);
934 	softmac->smac_state = SOFTMAC_ATTACH_DONE;
935 	softmac->smac_attacherr = err;
936 	cv_broadcast(&softmac->smac_cv);
937 	mutex_exit(&softmac->smac_mutex);
938 }
939 
940 int
941 softmac_destroy(dev_info_t *dip, dev_t dev)
942 {
943 	char			devname[MAXNAMELEN];
944 	softmac_t		*softmac;
945 	softmac_dev_t		*softmac_dev;
946 	int			index;
947 	int			ppa, err;
948 	datalink_id_t		linkid;
949 	mac_handle_t		smac_mh;
950 	uint32_t		smac_flags;
951 
952 	if (GLDV3_DRV(ddi_driver_major(dip))) {
953 		minor_t minor = getminor(dev);
954 		/*
955 		 * For an explanation of this logic, see the
956 		 * equivalent code in softmac_create.
957 		 */
958 		if ((strcmp(ddi_driver_name(dip), "clone") == 0) ||
959 		    (getmajor(dev) == ddi_name_to_major("clone")) ||
960 		    (minor == 0)) {
961 			return (0);
962 		}
963 		if (minor >= DLS_MAX_MINOR) {
964 			return (ENOTSUP);
965 		}
966 		ppa = DLS_MINOR2INST(minor);
967 	} else {
968 		ppa = ddi_get_instance(dip);
969 	}
970 
971 	(void) snprintf(devname, MAXNAMELEN, "%s%d", ddi_driver_name(dip), ppa);
972 
973 	/*
974 	 * We are called only from the predetach entry point. The DACF
975 	 * framework ensures there can't be a concurrent postattach call
976 	 * for the same softmac. The softmac found out from the modhash
977 	 * below can't vanish beneath us since this is the only place where
978 	 * it is deleted.
979 	 */
980 	err = mod_hash_find(softmac_hash, (mod_hash_key_t)devname,
981 	    (mod_hash_val_t *)&softmac);
982 	ASSERT(err == 0);
983 
984 	mutex_enter(&softmac->smac_mutex);
985 	SOFTMAC_STATE_VERIFY(softmac);
986 
987 	/*
988 	 * Fail the predetach routine if this softmac is in-use.
989 	 * Make sure these downcalls into softmac_create or softmac_destroy
990 	 * don't cv_wait on any devfs related condition. Thus softmac_destroy
991 	 * returns EBUSY if the asynchronous thread started in softmac_create
992 	 * hasn't finished
993 	 */
994 	if ((softmac->smac_hold_cnt != 0) ||
995 	    (softmac->smac_state == SOFTMAC_ATTACH_INPROG)) {
996 		softmac->smac_attached_left = softmac->smac_attachok_cnt;
997 		mutex_exit(&softmac->smac_mutex);
998 		return (EBUSY);
999 	}
1000 
1001 	/*
1002 	 * Even if the predetach of one minor node has already failed
1003 	 * (smac_attached_left is not 0), the DACF framework will continue
1004 	 * to call the predetach routines of the other minor nodes,
1005 	 * so we fail these calls here.
1006 	 */
1007 	if (softmac->smac_attached_left != 0) {
1008 		mutex_exit(&softmac->smac_mutex);
1009 		return (EBUSY);
1010 	}
1011 
1012 	smac_mh = softmac->smac_mh;
1013 	smac_flags = softmac->smac_flags;
1014 	softmac->smac_state = SOFTMAC_DETACH_INPROG;
1015 	mutex_exit(&softmac->smac_mutex);
1016 
1017 	if (smac_mh != NULL) {
1018 		/*
1019 		 * This is the first minor node that is being detached for this
1020 		 * softmac.
1021 		 */
1022 		ASSERT(softmac->smac_attachok_cnt == softmac->smac_cnt);
1023 		if (!(smac_flags & SOFTMAC_NOSUPP)) {
1024 			if ((err = dls_devnet_destroy(smac_mh, &linkid,
1025 			    B_FALSE)) != 0) {
1026 				goto error;
1027 			}
1028 		}
1029 		/*
1030 		 * If softmac_mac_register() succeeds in registering the mac
1031 		 * of the legacy device, unregister it.
1032 		 */
1033 		if (!(smac_flags & (SOFTMAC_GLDV3 | SOFTMAC_NOSUPP))) {
1034 			if ((err = mac_disable_nowait(smac_mh)) != 0) {
1035 				(void) dls_devnet_create(smac_mh, linkid,
1036 				    crgetzoneid(CRED()));
1037 				goto error;
1038 			}
1039 			/*
1040 			 * Ask softmac_notify_thread to quit, and wait for
1041 			 * that to be done.
1042 			 */
1043 			mutex_enter(&softmac->smac_mutex);
1044 			softmac->smac_flags |= SOFTMAC_NOTIFY_QUIT;
1045 			cv_broadcast(&softmac->smac_cv);
1046 			while (softmac->smac_notify_thread != NULL) {
1047 				cv_wait(&softmac->smac_cv,
1048 				    &softmac->smac_mutex);
1049 			}
1050 			mutex_exit(&softmac->smac_mutex);
1051 			VERIFY(mac_unregister(smac_mh) == 0);
1052 		}
1053 		softmac->smac_mh = NULL;
1054 	}
1055 
1056 	/*
1057 	 * Free softmac_dev
1058 	 */
1059 	rw_enter(&softmac_hash_lock, RW_WRITER);
1060 	mutex_enter(&softmac->smac_mutex);
1061 
1062 	ASSERT(softmac->smac_state == SOFTMAC_DETACH_INPROG &&
1063 	    softmac->smac_attachok_cnt != 0);
1064 	softmac->smac_mh = NULL;
1065 	index = (getmajor(dev) == ddi_name_to_major("clone"));
1066 	softmac_dev = softmac->smac_softmac[index];
1067 	ASSERT(softmac_dev != NULL);
1068 	softmac->smac_softmac[index] = NULL;
1069 	kmem_free(softmac_dev, sizeof (softmac_dev_t));
1070 
1071 	if (--softmac->smac_attachok_cnt == 0) {
1072 		mod_hash_val_t	hashval;
1073 
1074 		softmac->smac_state = SOFTMAC_UNINIT;
1075 		if (softmac->smac_hold_cnt != 0) {
1076 			/*
1077 			 * Someone did a softmac_hold_device while we dropped
1078 			 * the locks. Leave the softmac itself intact which
1079 			 * will be reused by the reattach
1080 			 */
1081 			mutex_exit(&softmac->smac_mutex);
1082 			rw_exit(&softmac_hash_lock);
1083 			return (0);
1084 		}
1085 		err = mod_hash_remove(softmac_hash,
1086 		    (mod_hash_key_t)devname,
1087 		    (mod_hash_val_t *)&hashval);
1088 		ASSERT(err == 0);
1089 
1090 		mutex_exit(&softmac->smac_mutex);
1091 		rw_exit(&softmac_hash_lock);
1092 		ASSERT(softmac->smac_fp_disable_clients == 0);
1093 		softmac->smac_fastpath_admin_disabled = B_FALSE;
1094 		kmem_cache_free(softmac_cachep, softmac);
1095 		return (0);
1096 	}
1097 	mutex_exit(&softmac->smac_mutex);
1098 	rw_exit(&softmac_hash_lock);
1099 	return (0);
1100 
1101 error:
1102 	mutex_enter(&softmac->smac_mutex);
1103 	softmac->smac_attached_left = softmac->smac_attachok_cnt;
1104 	softmac->smac_state = SOFTMAC_ATTACH_DONE;
1105 	cv_broadcast(&softmac->smac_cv);
1106 	mutex_exit(&softmac->smac_mutex);
1107 	return (err);
1108 }
1109 
1110 /*
1111  * This function is called as the result of a newly started dlmgmtd daemon.
1112  *
1113  * We walk through every softmac that was created but failed to notify
1114  * dlmgmtd about it (whose SOFTMAC_NEED_RECREATE flag is set).  This occurs
1115  * when softmacs are created before dlmgmtd is ready.  For example, during
1116  * diskless boot, a network device is used (and therefore attached) before
1117  * the datalink-management service starts dlmgmtd.
1118  */
1119 /* ARGSUSED */
1120 static uint_t
1121 softmac_mac_recreate(mod_hash_key_t key, mod_hash_val_t *val, void *arg)
1122 {
1123 	softmac_t	*softmac = (softmac_t *)val;
1124 	datalink_id_t	linkid;
1125 	int		err;
1126 	softmac_walk_t	*smwp = arg;
1127 
1128 	/*
1129 	 * The framework itself must not hold any locks across calls to the
1130 	 * mac perimeter. Thus this function does not call any framework
1131 	 * function that needs to grab the mac perimeter.
1132 	 */
1133 	ASSERT(RW_READ_HELD(&softmac_hash_lock));
1134 
1135 	smwp->smw_retry = B_FALSE;
1136 	mutex_enter(&softmac->smac_mutex);
1137 	SOFTMAC_STATE_VERIFY(softmac);
1138 	if (softmac->smac_state == SOFTMAC_ATTACH_INPROG) {
1139 		/*
1140 		 * Wait till softmac_create or softmac_mac_register finishes
1141 		 * Hold the softmac to ensure it stays around. The wait itself
1142 		 * is done in the caller, since we need to drop all locks
1143 		 * including the mod hash's internal lock before calling
1144 		 * cv_wait.
1145 		 */
1146 		smwp->smw_retry = B_TRUE;
1147 		smwp->smw_softmac = softmac;
1148 		softmac->smac_hold_cnt++;
1149 		return (MH_WALK_TERMINATE);
1150 	}
1151 
1152 	if ((softmac->smac_state != SOFTMAC_ATTACH_DONE) ||
1153 	    !(softmac->smac_flags & SOFTMAC_NEED_RECREATE)) {
1154 		mutex_exit(&softmac->smac_mutex);
1155 		return (MH_WALK_CONTINUE);
1156 	}
1157 
1158 	/*
1159 	 * Bumping up the smac_hold_cnt allows us to drop the lock. It also
1160 	 * makes softmac_destroy() return failure on an attempted device detach.
1161 	 * We don't want to hold the lock across calls to other subsystems
1162 	 * like kstats, which will happen in the call to dls_devnet_recreate
1163 	 */
1164 	softmac->smac_hold_cnt++;
1165 	mutex_exit(&softmac->smac_mutex);
1166 
1167 	if (dls_mgmt_create(softmac->smac_devname,
1168 	    makedevice(softmac->smac_umajor, softmac->smac_uppa + 1),
1169 	    DATALINK_CLASS_PHYS, softmac->smac_media, B_TRUE, &linkid) != 0) {
1170 		softmac_rele_device((dls_dev_handle_t)softmac);
1171 		return (MH_WALK_CONTINUE);
1172 	}
1173 
1174 	if ((err = softmac_update_info(softmac, &linkid)) != 0) {
1175 		cmn_err(CE_WARN, "softmac: softmac_update_info() for %s "
1176 		    "failed (%d)", softmac->smac_devname, err);
1177 		softmac_rele_device((dls_dev_handle_t)softmac);
1178 		return (MH_WALK_CONTINUE);
1179 	}
1180 
1181 	/*
1182 	 * Create a link for this MAC. The link name will be the same
1183 	 * as the MAC name.
1184 	 */
1185 	if (!(softmac->smac_flags & SOFTMAC_NOSUPP)) {
1186 		err = dls_devnet_recreate(softmac->smac_mh, linkid);
1187 		if (err != 0) {
1188 			cmn_err(CE_WARN, "softmac: dls_devnet_recreate() for "
1189 			    "%s (linkid %d) failed (%d)",
1190 			    softmac->smac_devname, linkid, err);
1191 		}
1192 	}
1193 
1194 	mutex_enter(&softmac->smac_mutex);
1195 	softmac->smac_flags &= ~SOFTMAC_NEED_RECREATE;
1196 	ASSERT(softmac->smac_hold_cnt != 0);
1197 	softmac->smac_hold_cnt--;
1198 	mutex_exit(&softmac->smac_mutex);
1199 
1200 	return (MH_WALK_CONTINUE);
1201 }
1202 
1203 /*
1204  * See comments above softmac_mac_recreate().
1205  */
1206 void
1207 softmac_recreate()
1208 {
1209 	softmac_walk_t	smw;
1210 	softmac_t	*softmac;
1211 
1212 	/*
1213 	 * Walk through the softmac_hash table. Request to create the
1214 	 * [link name, linkid] mapping if we failed to do so.
1215 	 */
1216 	do {
1217 		smw.smw_retry = B_FALSE;
1218 		rw_enter(&softmac_hash_lock, RW_READER);
1219 		mod_hash_walk(softmac_hash, softmac_mac_recreate, &smw);
1220 		rw_exit(&softmac_hash_lock);
1221 		if (smw.smw_retry) {
1222 			/*
1223 			 * softmac_create or softmac_mac_register hasn't yet
1224 			 * finished and the softmac is not yet in the
1225 			 * SOFTMAC_ATTACH_DONE state.
1226 			 */
1227 			softmac = smw.smw_softmac;
1228 			cv_wait(&softmac->smac_cv, &softmac->smac_mutex);
1229 			softmac->smac_hold_cnt--;
1230 			mutex_exit(&softmac->smac_mutex);
1231 		}
1232 	} while (smw.smw_retry);
1233 }
1234 
1235 static int
1236 softmac_m_start(void *arg)
1237 {
1238 	softmac_t	*softmac = arg;
1239 	softmac_lower_t	*slp = softmac->smac_lower;
1240 	int		err;
1241 
1242 	ASSERT(MAC_PERIM_HELD(softmac->smac_mh));
1243 	/*
1244 	 * Bind to SAP 2 on token ring, 0 on other interface types.
1245 	 * (SAP 0 has special significance on token ring).
1246 	 * Note that the receive-side packets could come anytime after bind.
1247 	 */
1248 	err = softmac_send_bind_req(slp, softmac->smac_media == DL_TPR ? 2 : 0);
1249 	if (err != 0)
1250 		return (err);
1251 
1252 	/*
1253 	 * Put the lower stream to the DL_PROMISC_SAP mode in order to receive
1254 	 * all packets of interest.
1255 	 *
1256 	 * some driver (e.g. the old legacy eri driver) incorrectly passes up
1257 	 * packets to DL_PROMISC_SAP stream when the lower stream is not bound,
1258 	 * so that we send DL_PROMISON_REQ after DL_BIND_REQ.
1259 	 */
1260 	err = softmac_send_promisc_req(slp, DL_PROMISC_SAP, B_TRUE);
1261 	if (err != 0) {
1262 		(void) softmac_send_unbind_req(slp);
1263 		return (err);
1264 	}
1265 
1266 	/*
1267 	 * Enable capabilities the underlying driver claims to support.
1268 	 * Some driver requires this being called after the stream is bound.
1269 	 */
1270 	if ((err = softmac_capab_enable(slp)) != 0) {
1271 		(void) softmac_send_promisc_req(slp, DL_PROMISC_SAP, B_FALSE);
1272 		(void) softmac_send_unbind_req(slp);
1273 	}
1274 
1275 	return (err);
1276 }
1277 
1278 /* ARGSUSED */
1279 static void
1280 softmac_m_stop(void *arg)
1281 {
1282 	softmac_t	*softmac = arg;
1283 	softmac_lower_t	*slp = softmac->smac_lower;
1284 
1285 	ASSERT(MAC_PERIM_HELD(softmac->smac_mh));
1286 
1287 	/*
1288 	 * It is not needed to reset zerocopy, MDT or HCKSUM capabilities.
1289 	 */
1290 	(void) softmac_send_promisc_req(slp, DL_PROMISC_SAP, B_FALSE);
1291 	(void) softmac_send_unbind_req(slp);
1292 }
1293 
1294 /*
1295  * Set up the lower stream above the legacy device. There are two different
1296  * type of lower streams:
1297  *
1298  * - Shared lower-stream
1299  *
1300  * Shared by all GLDv3 MAC clients. Put the lower stream to the DLIOCRAW
1301  * mode to send and receive the raw data. Further, put the lower stream into
1302  * DL_PROMISC_SAP mode to receive all packets of interest.
1303  *
1304  * - Dedicated lower-stream
1305  *
1306  * The lower-stream which is dedicated to upper IP/ARP stream. This is used
1307  * as fast-path for IP. In this case, the second argument is the pointer to
1308  * the softmac upper-stream.
1309  */
1310 int
1311 softmac_lower_setup(softmac_t *softmac, softmac_upper_t *sup,
1312     softmac_lower_t **slpp)
1313 {
1314 	ldi_ident_t		li;
1315 	dev_t			dev;
1316 	ldi_handle_t		lh = NULL;
1317 	softmac_lower_t		*slp = NULL;
1318 	smac_ioc_start_t	start_arg;
1319 	struct strioctl		strioc;
1320 	uint32_t		notifications;
1321 	int			err, rval;
1322 
1323 	if ((err = ldi_ident_from_dip(softmac_dip, &li)) != 0)
1324 		return (err);
1325 
1326 	/*
1327 	 * The GLDv3 framework makes sure that mac_unregister(), mac_open(),
1328 	 * and mac_close() cannot be called at the same time. So we don't
1329 	 * need any protection to access softmac here.
1330 	 */
1331 	dev = softmac->smac_dev;
1332 
1333 	err = ldi_open_by_dev(&dev, OTYP_CHR, FREAD|FWRITE, kcred, &lh, li);
1334 	ldi_ident_release(li);
1335 	if (err != 0)
1336 		goto done;
1337 
1338 	/*
1339 	 * Pop all the intermediate modules. The autopushed modules will
1340 	 * be pushed when the softmac node is opened.
1341 	 */
1342 	while (ldi_ioctl(lh, I_POP, 0, FKIOCTL, kcred, &rval) == 0)
1343 		;
1344 
1345 	if ((softmac->smac_style == DL_STYLE2) &&
1346 	    ((err = dl_attach(lh, softmac->smac_uppa, NULL)) != 0)) {
1347 		goto done;
1348 	}
1349 
1350 	/*
1351 	 * If this is the shared-lower-stream, put the lower stream to
1352 	 * the DLIOCRAW mode to send/receive raw data.
1353 	 */
1354 	if ((sup == NULL) && (err = ldi_ioctl(lh, DLIOCRAW, 0, FKIOCTL,
1355 	    kcred, &rval)) != 0) {
1356 		goto done;
1357 	}
1358 
1359 	/*
1360 	 * Then push the softmac shim layer atop the lower stream.
1361 	 */
1362 	if ((err = ldi_ioctl(lh, I_PUSH, (intptr_t)SOFTMAC_DEV_NAME, FKIOCTL,
1363 	    kcred, &rval)) != 0) {
1364 		goto done;
1365 	}
1366 
1367 	/*
1368 	 * Send the ioctl to get the slp pointer.
1369 	 */
1370 	strioc.ic_cmd = SMAC_IOC_START;
1371 	strioc.ic_timout = INFTIM;
1372 	strioc.ic_len = sizeof (start_arg);
1373 	strioc.ic_dp = (char *)&start_arg;
1374 
1375 	if ((err = ldi_ioctl(lh, I_STR, (intptr_t)&strioc, FKIOCTL,
1376 	    kcred, &rval)) != 0) {
1377 		goto done;
1378 	}
1379 	slp = start_arg.si_slp;
1380 	slp->sl_sup = sup;
1381 	slp->sl_lh = lh;
1382 	slp->sl_softmac = softmac;
1383 	*slpp = slp;
1384 
1385 	if (sup != NULL) {
1386 		slp->sl_rxinfo = &sup->su_rxinfo;
1387 	} else {
1388 		/*
1389 		 * Send DL_NOTIFY_REQ to enable certain DL_NOTIFY_IND.
1390 		 * We don't have to wait for the ack.
1391 		 */
1392 		notifications = DL_NOTE_PHYS_ADDR | DL_NOTE_LINK_UP |
1393 		    DL_NOTE_LINK_DOWN | DL_NOTE_PROMISC_ON_PHYS |
1394 		    DL_NOTE_PROMISC_OFF_PHYS;
1395 
1396 		(void) softmac_send_notify_req(slp,
1397 		    (notifications & softmac->smac_notifications));
1398 	}
1399 
1400 done:
1401 	if (err != 0)
1402 		(void) ldi_close(lh, FREAD|FWRITE, kcred);
1403 	return (err);
1404 }
1405 
1406 static int
1407 softmac_m_open(void *arg)
1408 {
1409 	softmac_t	*softmac = arg;
1410 	softmac_lower_t	*slp;
1411 	int		err;
1412 
1413 	ASSERT(MAC_PERIM_HELD(softmac->smac_mh));
1414 
1415 	if ((err = softmac_lower_setup(softmac, NULL, &slp)) != 0)
1416 		return (err);
1417 
1418 	softmac->smac_lower = slp;
1419 	return (0);
1420 }
1421 
1422 static void
1423 softmac_m_close(void *arg)
1424 {
1425 	softmac_t	*softmac = arg;
1426 	softmac_lower_t	*slp;
1427 
1428 	ASSERT(MAC_PERIM_HELD(softmac->smac_mh));
1429 	slp = softmac->smac_lower;
1430 	ASSERT(slp != NULL);
1431 
1432 	/*
1433 	 * Note that slp is destroyed when lh is closed.
1434 	 */
1435 	(void) ldi_close(slp->sl_lh, FREAD|FWRITE, kcred);
1436 	softmac->smac_lower = NULL;
1437 }
1438 
1439 /*
1440  * Softmac supports two priviate link properteis:
1441  *
1442  * - "_fastpath"
1443  *
1444  *    This is a read-only link property which points out the current data-path
1445  *    model of the given legacy link. The possible values are "disabled" and
1446  *    "enabled".
1447  *
1448  * - "_disable_fastpath"
1449  *
1450  *    This is a read-write link property which can be used to disable or enable
1451  *    the fast-path of the given legacy link. The possible values are "true"
1452  *    and "false". Note that even when "_disable_fastpath" is set to be
1453  *    "false", the fast-path may still not be enabled since there may be
1454  *    other mac cleints that request the fast-path to be disabled.
1455  */
1456 /* ARGSUSED */
1457 static int
1458 softmac_m_setprop(void *arg, const char *name, mac_prop_id_t id,
1459     uint_t valsize, const void *val)
1460 {
1461 	softmac_t	*softmac = arg;
1462 
1463 	if (id != MAC_PROP_PRIVATE || strcmp(name, "_disable_fastpath") != 0)
1464 		return (ENOTSUP);
1465 
1466 	if (strcmp(val, "true") == 0)
1467 		return (softmac_datapath_switch(softmac, B_TRUE, B_TRUE));
1468 	else if (strcmp(val, "false") == 0)
1469 		return (softmac_datapath_switch(softmac, B_FALSE, B_TRUE));
1470 	else
1471 		return (EINVAL);
1472 }
1473 
1474 static int
1475 softmac_m_getprop(void *arg, const char *name, mac_prop_id_t id,
1476     uint_t valsize, void *val)
1477 {
1478 	softmac_t	*softmac = arg;
1479 	char		*fpstr;
1480 
1481 	if (id != MAC_PROP_PRIVATE)
1482 		return (ENOTSUP);
1483 
1484 	if (strcmp(name, "_fastpath") == 0) {
1485 		mutex_enter(&softmac->smac_fp_mutex);
1486 		fpstr = (DATAPATH_MODE(softmac) == SOFTMAC_SLOWPATH) ?
1487 		    "disabled" : "enabled";
1488 		mutex_exit(&softmac->smac_fp_mutex);
1489 	} else if (strcmp(name, "_disable_fastpath") == 0) {
1490 		fpstr = softmac->smac_fastpath_admin_disabled ?
1491 		    "true" : "false";
1492 	} else if (strcmp(name, "_softmac") == 0) {
1493 		fpstr = "true";
1494 	} else {
1495 		return (ENOTSUP);
1496 	}
1497 
1498 	return (strlcpy(val, fpstr, valsize) >= valsize ? EINVAL : 0);
1499 }
1500 
1501 static void
1502 softmac_m_propinfo(void *arg, const char *name, mac_prop_id_t id,
1503     mac_prop_info_handle_t prh)
1504 {
1505         _NOTE(ARGUNUSED(arg));
1506 
1507 	if (id != MAC_PROP_PRIVATE)
1508 		return;
1509 
1510 	if (strcmp(name, "_fastpath") == 0) {
1511 		mac_prop_info_set_perm(prh, MAC_PROP_PERM_READ);
1512 	} else if (strcmp(name, "_disable_fastpath") == 0) {
1513 		mac_prop_info_set_default_str(prh, "false");
1514 	}
1515 
1516 }
1517 
1518 int
1519 softmac_hold_device(dev_t dev, dls_dev_handle_t *ddhp)
1520 {
1521 	dev_info_t	*dip;
1522 	char		devname[MAXNAMELEN];
1523 	softmac_t	*softmac;
1524 	major_t		major;
1525 	int		ppa, err = 0, inst;
1526 
1527 	major = getmajor(dev);
1528 	ppa = getminor(dev) - 1;
1529 
1530 	/*
1531 	 * For GLDv3 devices, look up the device instance using getinfo(9e).
1532 	 * Otherwise, fall back to the old assumption that inst == ppa.  The
1533 	 * GLDV3_DRV() macro depends on the driver module being loaded, hence
1534 	 * the call to ddi_hold_driver().
1535 	 */
1536 	if (ddi_hold_driver(major) == NULL)
1537 		return (ENXIO);
1538 	if (GLDV3_DRV(major)) {
1539 		if ((inst = dev_to_instance(dev)) < 0)
1540 			err = ENOENT;
1541 	} else {
1542 		inst = ppa;
1543 	}
1544 	ddi_rele_driver(major);
1545 	if (err != 0)
1546 		return (err);
1547 
1548 	/*
1549 	 * First try to hold this device instance to force device to attach
1550 	 * and ensure that the softmac entry gets created in net_postattach().
1551 	 */
1552 	if ((dip = ddi_hold_devi_by_instance(major, inst, 0)) == NULL)
1553 		return (ENOENT);
1554 
1555 	/*
1556 	 * Exclude non-physical network device instances, for example, aggr0.
1557 	 * Note: this check *must* occur after the dip is held, or else
1558 	 * NETWORK_PHYSDRV might return false incorrectly.  The
1559 	 * DN_NETWORK_PHYSDRIVER flag used by NETWORK_PHYSDRV() gets set if
1560 	 * ddi_create_minor_node() is called during the device's attach
1561 	 * phase.
1562 	 */
1563 	if (!NETWORK_PHYSDRV(major)) {
1564 		ddi_release_devi(dip);
1565 		return (ENOENT);
1566 	}
1567 
1568 	/* Now wait for its softmac to be created. */
1569 	(void) snprintf(devname, MAXNAMELEN, "%s%d", ddi_major_to_name(major),
1570 	    ppa);
1571 again:
1572 	rw_enter(&softmac_hash_lock, RW_READER);
1573 
1574 	if (mod_hash_find(softmac_hash, (mod_hash_key_t)devname,
1575 	    (mod_hash_val_t *)&softmac) != 0) {
1576 		/*
1577 		 * This is rare but possible. It could happen when pre-detach
1578 		 * routine of the device succeeds. But the softmac will then
1579 		 * be recreated when device fails to detach (as this device
1580 		 * is held).
1581 		 */
1582 		mutex_enter(&smac_global_lock);
1583 		rw_exit(&softmac_hash_lock);
1584 		cv_wait(&smac_global_cv, &smac_global_lock);
1585 		mutex_exit(&smac_global_lock);
1586 		goto again;
1587 	}
1588 
1589 	/*
1590 	 * Bump smac_hold_cnt to prevent device detach.
1591 	 */
1592 	mutex_enter(&softmac->smac_mutex);
1593 	softmac->smac_hold_cnt++;
1594 	rw_exit(&softmac_hash_lock);
1595 
1596 	/*
1597 	 * Wait till the device is fully attached.
1598 	 */
1599 	while (softmac->smac_state != SOFTMAC_ATTACH_DONE)
1600 		cv_wait(&softmac->smac_cv, &softmac->smac_mutex);
1601 
1602 	SOFTMAC_STATE_VERIFY(softmac);
1603 
1604 	if ((err = softmac->smac_attacherr) != 0)
1605 		softmac->smac_hold_cnt--;
1606 	else
1607 		*ddhp = (dls_dev_handle_t)softmac;
1608 	mutex_exit(&softmac->smac_mutex);
1609 
1610 	ddi_release_devi(dip);
1611 	return (err);
1612 }
1613 
1614 void
1615 softmac_rele_device(dls_dev_handle_t ddh)
1616 {
1617 	if (ddh != NULL)
1618 		softmac_rele((softmac_t *)ddh);
1619 }
1620 
1621 int
1622 softmac_hold(dev_t dev, softmac_t **softmacp)
1623 {
1624 	softmac_t	*softmac;
1625 	char		*drv;
1626 	mac_handle_t	mh;
1627 	char		mac[MAXNAMELEN];
1628 	int		err;
1629 
1630 	if ((drv = ddi_major_to_name(getmajor(dev))) == NULL)
1631 		return (EINVAL);
1632 
1633 	(void) snprintf(mac, MAXNAMELEN, "%s%d", drv, getminor(dev) - 1);
1634 	if ((err = mac_open(mac, &mh)) != 0)
1635 		return (err);
1636 
1637 	softmac = (softmac_t *)mac_driver(mh);
1638 
1639 	mutex_enter(&softmac->smac_mutex);
1640 	softmac->smac_hold_cnt++;
1641 	mutex_exit(&softmac->smac_mutex);
1642 	mac_close(mh);
1643 	*softmacp = softmac;
1644 	return (0);
1645 }
1646 
1647 void
1648 softmac_rele(softmac_t *softmac)
1649 {
1650 	mutex_enter(&softmac->smac_mutex);
1651 	softmac->smac_hold_cnt--;
1652 	mutex_exit(&softmac->smac_mutex);
1653 }
1654