xref: /illumos-gate/usr/src/uts/common/io/softmac/softmac_fp.c (revision 9b664393d4fdda96221e6ea9ea95790d3c15be70)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 /*
27  * Copyright 2019, Joyent, Inc.
28  * Copyright 2022 Garrett D'Amore
29  */
30 
31 /*
32  * Softmac data-path switching:
33  *
34  * - Fast-path model
35  *
36  * When the softmac fast-path is used, a dedicated lower-stream
37  * will be opened over the legacy device for each IP/ARP (upper-)stream
38  * over the softMAC, and all DLPI messages (including control messages
39  * and data messages) will be exchanged between the upper-stream and
40  * the corresponding lower-stream directly. Therefore, the data
41  * demultiplexing, filtering and classification processing will be done
42  * by the lower-stream, and the GLDv3 DLS/MAC layer processing will be
43  * no longer needed.
44  *
45  * - Slow-path model
46  *
47  * Some GLDv3 features requires the GLDv3 DLS/MAC layer processing to
48  * not be bypassed to assure its function correctness. For example,
49  * softmac fast-path must be disabled to support GLDv3 VNIC functionality.
50  * In this case, a shared lower-stream will be opened over the legacy
51  * device, which is responsible for implementing the GLDv3 callbacks
52  * and passing RAW data messages between the legacy devices and the GLDv3
53  * framework.
54  *
55  * By default, the softmac fast-path mode will be used to assure the
56  * performance; MAC clients will be able to request to disable the softmac
57  * fast-path mode to support certain features, and if that succeeds,
58  * the system will fallback to the slow-path softmac data-path model.
59  *
60  *
61  * The details of the softmac data fast-path model is stated as below
62  *
63  * 1. When a stream is opened on a softMAC, the softmac module will takes
64  *    over the DLPI processing on this stream;
65  *
66  * 2. For IP/ARP streams over a softMAC, softmac data fast-path will be
67  *    used by default, unless fast-path is disabled by any MAC client
68  *    explicitly. The softmac module first identifies an IP/ARP stream
69  *    by seeing whether there is a SIOCSLIFNAME ioctl sent from upstream,
70  *    if there is one, this stream is either an IP or an ARP stream
71  *    and will use fast-path potentially;
72  *
73  * 3. When the softmac fast-path is used, an dedicated lower-stream will
74  *    be setup for each IP/ARP stream (1-1 mapping). From that point on,
75  *    all control and data messages will be exchanged between the IP/ARP
76  *    upper-stream and the legacy device through this dedicated
77  *    lower-stream. As a result, the DLS/MAC layer processing in GLDv3
78  *    will be skipped, and this greatly improves the performance;
79  *
80  * 4. When the softmac data fast-path is disabled by a MAC client (e.g.,
81  *    by a VNIC), all the IP/ARP upper streams will try to switch from
82  *    the fast-path to the slow-path. The dedicated lower-stream will be
83  *    destroyed, and all the control and data-messages will go through the
84  *    existing GLDv3 code path and (in the end) the shared lower-stream;
85  *
86  * 5. On the other hand, when the last MAC client cancels its fast-path
87  *    disable request, all the IP/ARP streams will try to switch back to
88  *    the fast-path mode;
89  *
90  * Step 5 and 6 both rely on the data-path mode switching process
91  * described below:
92  *
93  * 1) To switch the softmac data-path mode (between fast-path and slow-path),
94  *    softmac will first send a DL_NOTE_REPLUMB DL_NOTIFY_IND message
95  *    upstream over each IP/ARP streams that needs data-path mode switching;
96  *
97  * 2) When IP receives this DL_NOTE_REPLUMB message, it will bring down
98  *    all the IP interfaces on the corresponding ill (IP Lower level
99  *    structure), and bring up those interfaces over again; this will in
100  *    turn cause the ARP to "replumb" the interface.
101  *
102  *    During the replumb process, both IP and ARP will send downstream the
103  *    necessary DL_DISABMULTI_REQ and DL_UNBIND_REQ messages and cleanup
104  *    the old state of the underlying softMAC, following with the necessary
105  *    DL_BIND_REQ and DL_ENABMULTI_REQ messages to setup the new state.
106  *    Between the cleanup and re-setup process, IP/ARP will also send down
107  *    a DL_NOTE_REPLUMB_DONE DL_NOTIFY_CONF messages to the softMAC to
108  *    indicate the *switching point*;
109  *
110  * 3) When softmac receives the DL_NOTE_REPLUMB_DONE message, it either
111  *    creates or destroys the dedicated lower-stream (depending on which
112  *    data-path mode the softMAC switches to), and change the softmac
113  *    data-path mode. From then on, softmac will process all the succeeding
114  *    control messages (including the DL_BIND_REQ and DL_ENABMULTI_REQ
115  *    messages) and data messages based on new data-path mode.
116  */
117 
118 #include <sys/types.h>
119 #include <sys/disp.h>
120 #include <sys/callb.h>
121 #include <sys/sysmacros.h>
122 #include <sys/file.h>
123 #include <sys/vlan.h>
124 #include <sys/dld.h>
125 #include <sys/sockio.h>
126 #include <sys/softmac_impl.h>
127 #include <net/if.h>
128 
129 static kmutex_t		softmac_taskq_lock;
130 static kcondvar_t	softmac_taskq_cv;
131 static list_t		softmac_taskq_list;	/* List of softmac_upper_t */
132 boolean_t		softmac_taskq_quit;
133 boolean_t		softmac_taskq_done;
134 
135 static void		softmac_taskq_dispatch();
136 static int		softmac_fastpath_setup(softmac_upper_t *);
137 static mac_tx_cookie_t	softmac_fastpath_wput_data(softmac_upper_t *, mblk_t *,
138 			    uintptr_t, uint16_t);
139 static void		softmac_datapath_switch_done(softmac_upper_t *);
140 
141 void
softmac_fp_init()142 softmac_fp_init()
143 {
144 	mutex_init(&softmac_taskq_lock, NULL, MUTEX_DRIVER, NULL);
145 	cv_init(&softmac_taskq_cv, NULL, CV_DRIVER, NULL);
146 
147 	softmac_taskq_quit = B_FALSE;
148 	softmac_taskq_done = B_FALSE;
149 	list_create(&softmac_taskq_list, sizeof (softmac_upper_t),
150 	    offsetof(softmac_upper_t, su_taskq_list_node));
151 	(void) thread_create(NULL, 0, softmac_taskq_dispatch, NULL, 0,
152 	    &p0, TS_RUN, minclsyspri);
153 }
154 
155 void
softmac_fp_fini()156 softmac_fp_fini()
157 {
158 	/*
159 	 * Request the softmac_taskq thread to quit and wait for it to be done.
160 	 */
161 	mutex_enter(&softmac_taskq_lock);
162 	softmac_taskq_quit = B_TRUE;
163 	cv_signal(&softmac_taskq_cv);
164 	while (!softmac_taskq_done)
165 		cv_wait(&softmac_taskq_cv, &softmac_taskq_lock);
166 	mutex_exit(&softmac_taskq_lock);
167 	list_destroy(&softmac_taskq_list);
168 
169 	mutex_destroy(&softmac_taskq_lock);
170 	cv_destroy(&softmac_taskq_cv);
171 }
172 
173 static boolean_t
check_ip_above(queue_t * q)174 check_ip_above(queue_t *q)
175 {
176 	queue_t		*next_q;
177 	boolean_t	ret = B_TRUE;
178 
179 	claimstr(q);
180 	next_q = q->q_next;
181 	if (strcmp(next_q->q_qinfo->qi_minfo->mi_idname, "ip") != 0)
182 		ret = B_FALSE;
183 	releasestr(q);
184 	return (ret);
185 }
186 
187 /* ARGSUSED */
188 static int
softmac_capab_perim(softmac_upper_t * sup,void * data,uint_t flags)189 softmac_capab_perim(softmac_upper_t *sup, void *data, uint_t flags)
190 {
191 	switch (flags) {
192 	case DLD_ENABLE:
193 		mutex_enter(&sup->su_mutex);
194 		break;
195 	case DLD_DISABLE:
196 		mutex_exit(&sup->su_mutex);
197 		break;
198 	case DLD_QUERY:
199 		return (MUTEX_HELD(&sup->su_mutex));
200 	}
201 	return (0);
202 }
203 
204 static mac_tx_notify_handle_t
softmac_client_tx_notify(softmac_upper_t * sup,mac_tx_notify_t func,void * arg)205 softmac_client_tx_notify(softmac_upper_t *sup, mac_tx_notify_t func, void *arg)
206 {
207 	ASSERT(MUTEX_HELD(&sup->su_mutex));
208 
209 	if (func != NULL) {
210 		sup->su_tx_notify_func = func;
211 		sup->su_tx_notify_arg = arg;
212 	} else {
213 		/*
214 		 * Wait for all tx_notify_func call to be done.
215 		 */
216 		while (sup->su_tx_inprocess != 0)
217 			cv_wait(&sup->su_cv, &sup->su_mutex);
218 
219 		sup->su_tx_notify_func = NULL;
220 		sup->su_tx_notify_arg = NULL;
221 	}
222 	return ((mac_tx_notify_handle_t)sup);
223 }
224 
225 static boolean_t
softmac_tx_is_flow_blocked(softmac_upper_t * sup,mac_tx_cookie_t cookie)226 softmac_tx_is_flow_blocked(softmac_upper_t *sup, mac_tx_cookie_t cookie)
227 {
228 	ASSERT(cookie == (mac_tx_cookie_t)sup);
229 	return (sup->su_tx_busy);
230 }
231 
232 static int
softmac_capab_direct(softmac_upper_t * sup,void * data,uint_t flags)233 softmac_capab_direct(softmac_upper_t *sup, void *data, uint_t flags)
234 {
235 	dld_capab_direct_t	*direct = data;
236 	softmac_lower_t		*slp = sup->su_slp;
237 
238 	ASSERT(MUTEX_HELD(&sup->su_mutex));
239 
240 	ASSERT(sup->su_mode == SOFTMAC_FASTPATH);
241 
242 	switch (flags) {
243 	case DLD_ENABLE:
244 		if (sup->su_direct)
245 			return (0);
246 
247 		sup->su_direct_rxinfo.slr_rx = (softmac_rx_t)direct->di_rx_cf;
248 		sup->su_direct_rxinfo.slr_arg = direct->di_rx_ch;
249 		slp->sl_rxinfo = &sup->su_direct_rxinfo;
250 		direct->di_tx_df = (uintptr_t)softmac_fastpath_wput_data;
251 		direct->di_tx_dh = sup;
252 		direct->di_tx_fctl_df = (uintptr_t)softmac_tx_is_flow_blocked;
253 		direct->di_tx_fctl_dh = sup;
254 		direct->di_tx_cb_df = (uintptr_t)softmac_client_tx_notify;
255 		direct->di_tx_cb_dh = sup;
256 		sup->su_direct = B_TRUE;
257 		return (0);
258 
259 	case DLD_DISABLE:
260 		if (!sup->su_direct)
261 			return (0);
262 
263 		slp->sl_rxinfo = &sup->su_rxinfo;
264 		sup->su_direct = B_FALSE;
265 		return (0);
266 	}
267 	return (ENOTSUP);
268 }
269 
270 static int
softmac_dld_capab(softmac_upper_t * sup,uint_t type,void * data,uint_t flags)271 softmac_dld_capab(softmac_upper_t *sup, uint_t type, void *data, uint_t flags)
272 {
273 	int	err;
274 
275 	/*
276 	 * Don't enable direct callback capabilities unless the caller is
277 	 * the IP client. When a module is inserted in a stream (_I_INSERT)
278 	 * the stack initiates capability disable, but due to races, the
279 	 * module insertion may complete before the capability disable
280 	 * completes. So we limit the check to DLD_ENABLE case.
281 	 */
282 	if ((flags == DLD_ENABLE && type != DLD_CAPAB_PERIM) &&
283 	    !check_ip_above(sup->su_rq)) {
284 		return (ENOTSUP);
285 	}
286 
287 	switch (type) {
288 	case DLD_CAPAB_DIRECT:
289 		err = softmac_capab_direct(sup, data, flags);
290 		break;
291 
292 	case DLD_CAPAB_PERIM:
293 		err = softmac_capab_perim(sup, data, flags);
294 		break;
295 
296 	default:
297 		err = ENOTSUP;
298 		break;
299 	}
300 	return (err);
301 }
302 
303 static void
softmac_capability_advertise(softmac_upper_t * sup,mblk_t * mp)304 softmac_capability_advertise(softmac_upper_t *sup, mblk_t *mp)
305 {
306 	dl_capability_ack_t	*dlap;
307 	dl_capability_sub_t	*dlsp;
308 	t_uscalar_t		subsize;
309 	uint8_t			*ptr;
310 	queue_t			*q = sup->su_wq;
311 	mblk_t			*mp1;
312 	softmac_t		*softmac = sup->su_softmac;
313 	boolean_t		dld_capable = B_FALSE;
314 	boolean_t		hcksum_capable = B_FALSE;
315 	boolean_t		zcopy_capable = B_FALSE;
316 
317 	ASSERT(sup->su_mode == SOFTMAC_FASTPATH);
318 
319 	/*
320 	 * Initially assume no capabilities.
321 	 */
322 	subsize = 0;
323 
324 	/*
325 	 * Direct capability negotiation interface between IP and softmac
326 	 */
327 	if (check_ip_above(sup->su_rq)) {
328 		dld_capable = B_TRUE;
329 		subsize += sizeof (dl_capability_sub_t) +
330 		    sizeof (dl_capab_dld_t);
331 	}
332 
333 	/*
334 	 * Check if checksum offload is supported on this MAC.
335 	 */
336 	if (softmac->smac_capab_flags & MAC_CAPAB_HCKSUM) {
337 		hcksum_capable = B_TRUE;
338 		subsize += sizeof (dl_capability_sub_t) +
339 		    sizeof (dl_capab_hcksum_t);
340 	}
341 
342 	/*
343 	 * Check if zerocopy is supported on this interface.
344 	 */
345 	if (!(softmac->smac_capab_flags & MAC_CAPAB_NO_ZCOPY)) {
346 		zcopy_capable = B_TRUE;
347 		subsize += sizeof (dl_capability_sub_t) +
348 		    sizeof (dl_capab_zerocopy_t);
349 	}
350 
351 	/*
352 	 * If there are no capabilities to advertise or if we
353 	 * can't allocate a response, send a DL_ERROR_ACK.
354 	 */
355 	if ((subsize == 0) || (mp1 = reallocb(mp,
356 	    sizeof (dl_capability_ack_t) + subsize, 0)) == NULL) {
357 		dlerrorack(q, mp, DL_CAPABILITY_REQ, DL_NOTSUPPORTED, 0);
358 		return;
359 	}
360 
361 	mp = mp1;
362 	DB_TYPE(mp) = M_PROTO;
363 	mp->b_wptr = mp->b_rptr + sizeof (dl_capability_ack_t) + subsize;
364 	bzero(mp->b_rptr, MBLKL(mp));
365 	dlap = (dl_capability_ack_t *)mp->b_rptr;
366 	dlap->dl_primitive = DL_CAPABILITY_ACK;
367 	dlap->dl_sub_offset = sizeof (dl_capability_ack_t);
368 	dlap->dl_sub_length = subsize;
369 	ptr = (uint8_t *)&dlap[1];
370 
371 	/*
372 	 * IP polling interface.
373 	 */
374 	if (dld_capable) {
375 		dl_capab_dld_t		dld;
376 
377 		dlsp = (dl_capability_sub_t *)ptr;
378 		dlsp->dl_cap = DL_CAPAB_DLD;
379 		dlsp->dl_length = sizeof (dl_capab_dld_t);
380 		ptr += sizeof (dl_capability_sub_t);
381 
382 		bzero(&dld, sizeof (dl_capab_dld_t));
383 		dld.dld_version = DLD_CURRENT_VERSION;
384 		dld.dld_capab = (uintptr_t)softmac_dld_capab;
385 		dld.dld_capab_handle = (uintptr_t)sup;
386 
387 		dlcapabsetqid(&(dld.dld_mid), sup->su_rq);
388 		bcopy(&dld, ptr, sizeof (dl_capab_dld_t));
389 		ptr += sizeof (dl_capab_dld_t);
390 	}
391 
392 	/*
393 	 * TCP/IP checksum offload.
394 	 */
395 	if (hcksum_capable) {
396 		dl_capab_hcksum_t	hcksum;
397 
398 		dlsp = (dl_capability_sub_t *)ptr;
399 
400 		dlsp->dl_cap = DL_CAPAB_HCKSUM;
401 		dlsp->dl_length = sizeof (dl_capab_hcksum_t);
402 		ptr += sizeof (dl_capability_sub_t);
403 
404 		bzero(&hcksum, sizeof (dl_capab_hcksum_t));
405 		hcksum.hcksum_version = HCKSUM_VERSION_1;
406 		hcksum.hcksum_txflags = softmac->smac_hcksum_txflags;
407 		dlcapabsetqid(&(hcksum.hcksum_mid), sup->su_rq);
408 		bcopy(&hcksum, ptr, sizeof (dl_capab_hcksum_t));
409 		ptr += sizeof (dl_capab_hcksum_t);
410 	}
411 
412 	/*
413 	 * Zero copy
414 	 */
415 	if (zcopy_capable) {
416 		dl_capab_zerocopy_t	zcopy;
417 
418 		dlsp = (dl_capability_sub_t *)ptr;
419 
420 		dlsp->dl_cap = DL_CAPAB_ZEROCOPY;
421 		dlsp->dl_length = sizeof (dl_capab_zerocopy_t);
422 		ptr += sizeof (dl_capability_sub_t);
423 
424 		bzero(&zcopy, sizeof (dl_capab_zerocopy_t));
425 		zcopy.zerocopy_version = ZEROCOPY_VERSION_1;
426 		zcopy.zerocopy_flags = DL_CAPAB_VMSAFE_MEM;
427 		dlcapabsetqid(&(zcopy.zerocopy_mid), sup->su_rq);
428 		bcopy(&zcopy, ptr, sizeof (dl_capab_zerocopy_t));
429 		ptr += sizeof (dl_capab_zerocopy_t);
430 	}
431 
432 	ASSERT(ptr == mp->b_rptr + sizeof (dl_capability_ack_t) + subsize);
433 	qreply(q, mp);
434 }
435 
436 static void
softmac_capability_req(softmac_upper_t * sup,mblk_t * mp)437 softmac_capability_req(softmac_upper_t *sup, mblk_t *mp)
438 {
439 	dl_capability_req_t	*dlp = (dl_capability_req_t *)mp->b_rptr;
440 	dl_capability_sub_t	*sp;
441 	size_t			size, len;
442 	offset_t		off, end;
443 	t_uscalar_t		dl_err;
444 	queue_t			*q = sup->su_wq;
445 
446 	ASSERT(sup->su_mode == SOFTMAC_FASTPATH);
447 	if (MBLKL(mp) < sizeof (dl_capability_req_t)) {
448 		dl_err = DL_BADPRIM;
449 		goto failed;
450 	}
451 
452 	if (!sup->su_bound) {
453 		dl_err = DL_OUTSTATE;
454 		goto failed;
455 	}
456 
457 	/*
458 	 * This request is overloaded. If there are no requested capabilities
459 	 * then we just want to acknowledge with all the capabilities we
460 	 * support. Otherwise we enable the set of capabilities requested.
461 	 */
462 	if (dlp->dl_sub_length == 0) {
463 		softmac_capability_advertise(sup, mp);
464 		return;
465 	}
466 
467 	if (!MBLKIN(mp, dlp->dl_sub_offset, dlp->dl_sub_length)) {
468 		dl_err = DL_BADPRIM;
469 		goto failed;
470 	}
471 
472 	dlp->dl_primitive = DL_CAPABILITY_ACK;
473 
474 	off = dlp->dl_sub_offset;
475 	len = dlp->dl_sub_length;
476 
477 	/*
478 	 * Walk the list of capabilities to be enabled.
479 	 */
480 	for (end = off + len; off < end; ) {
481 		sp = (dl_capability_sub_t *)(mp->b_rptr + off);
482 		size = sizeof (dl_capability_sub_t) + sp->dl_length;
483 
484 		if (off + size > end ||
485 		    !IS_P2ALIGNED(off, sizeof (uint32_t))) {
486 			dl_err = DL_BADPRIM;
487 			goto failed;
488 		}
489 
490 		switch (sp->dl_cap) {
491 		/*
492 		 * TCP/IP checksum offload to hardware.
493 		 */
494 		case DL_CAPAB_HCKSUM: {
495 			dl_capab_hcksum_t *hcksump;
496 			dl_capab_hcksum_t hcksum;
497 
498 			hcksump = (dl_capab_hcksum_t *)&sp[1];
499 			/*
500 			 * Copy for alignment.
501 			 */
502 			bcopy(hcksump, &hcksum, sizeof (dl_capab_hcksum_t));
503 			dlcapabsetqid(&(hcksum.hcksum_mid), sup->su_rq);
504 			bcopy(&hcksum, hcksump, sizeof (dl_capab_hcksum_t));
505 			break;
506 		}
507 
508 		default:
509 			break;
510 		}
511 
512 		off += size;
513 	}
514 	qreply(q, mp);
515 	return;
516 failed:
517 	dlerrorack(q, mp, DL_CAPABILITY_REQ, dl_err, 0);
518 }
519 
520 static void
softmac_bind_req(softmac_upper_t * sup,mblk_t * mp)521 softmac_bind_req(softmac_upper_t *sup, mblk_t *mp)
522 {
523 	softmac_lower_t	*slp = sup->su_slp;
524 	softmac_t	*softmac = sup->su_softmac;
525 	mblk_t		*ackmp, *mp1;
526 	int		err;
527 
528 	if (MBLKL(mp) < DL_BIND_REQ_SIZE) {
529 		freemsg(mp);
530 		return;
531 	}
532 
533 	/*
534 	 * Allocate ackmp incase the underlying driver does not ack timely.
535 	 */
536 	if ((mp1 = allocb(sizeof (dl_error_ack_t), BPRI_HI)) == NULL) {
537 		dlerrorack(sup->su_wq, mp, DL_BIND_REQ, DL_SYSERR, ENOMEM);
538 		return;
539 	}
540 
541 	err = softmac_output(slp, mp, DL_BIND_REQ, DL_BIND_ACK, &ackmp);
542 	if (ackmp != NULL) {
543 		freemsg(mp1);
544 	} else {
545 		/*
546 		 * The driver does not ack timely.
547 		 */
548 		ASSERT(err == ENOMSG);
549 		ackmp = mp1;
550 	}
551 	if (err != 0)
552 		goto failed;
553 
554 	/*
555 	 * Enable capabilities the underlying driver claims to support.
556 	 */
557 	if ((err = softmac_capab_enable(slp)) != 0)
558 		goto failed;
559 
560 	/*
561 	 * Check whether this softmac is already marked as exclusively used,
562 	 * e.g., an aggregation is created over it. Fail the BIND_REQ if so.
563 	 */
564 	mutex_enter(&softmac->smac_active_mutex);
565 	if (softmac->smac_active) {
566 		mutex_exit(&softmac->smac_active_mutex);
567 		err = EBUSY;
568 		goto failed;
569 	}
570 	softmac->smac_nactive++;
571 	sup->su_active = B_TRUE;
572 	mutex_exit(&softmac->smac_active_mutex);
573 	sup->su_bound = B_TRUE;
574 
575 	qreply(sup->su_wq, ackmp);
576 	return;
577 failed:
578 	if (err != 0) {
579 		dlerrorack(sup->su_wq, ackmp, DL_BIND_REQ, DL_SYSERR, err);
580 		return;
581 	}
582 }
583 
584 static void
softmac_unbind_req(softmac_upper_t * sup,mblk_t * mp)585 softmac_unbind_req(softmac_upper_t *sup, mblk_t *mp)
586 {
587 	softmac_lower_t	*slp = sup->su_slp;
588 	softmac_t	*softmac = sup->su_softmac;
589 	mblk_t		*ackmp, *mp1;
590 	int		err;
591 
592 	if (MBLKL(mp) < DL_UNBIND_REQ_SIZE) {
593 		freemsg(mp);
594 		return;
595 	}
596 
597 	if (!sup->su_bound) {
598 		dlerrorack(sup->su_wq, mp, DL_UNBIND_REQ, DL_OUTSTATE, 0);
599 		return;
600 	}
601 
602 	/*
603 	 * Allocate ackmp incase the underlying driver does not ack timely.
604 	 */
605 	if ((mp1 = allocb(sizeof (dl_error_ack_t), BPRI_HI)) == NULL) {
606 		dlerrorack(sup->su_wq, mp, DL_UNBIND_REQ, DL_SYSERR, ENOMEM);
607 		return;
608 	}
609 
610 	err = softmac_output(slp, mp, DL_UNBIND_REQ, DL_OK_ACK, &ackmp);
611 	if (ackmp != NULL) {
612 		freemsg(mp1);
613 	} else {
614 		/*
615 		 * The driver does not ack timely.
616 		 */
617 		ASSERT(err == ENOMSG);
618 		ackmp = mp1;
619 	}
620 	if (err != 0) {
621 		dlerrorack(sup->su_wq, ackmp, DL_UNBIND_REQ, DL_SYSERR, err);
622 		return;
623 	}
624 
625 	sup->su_bound = B_FALSE;
626 
627 	mutex_enter(&softmac->smac_active_mutex);
628 	if (sup->su_active) {
629 		ASSERT(!softmac->smac_active);
630 		softmac->smac_nactive--;
631 		sup->su_active = B_FALSE;
632 	}
633 	mutex_exit(&softmac->smac_active_mutex);
634 
635 done:
636 	qreply(sup->su_wq, ackmp);
637 }
638 
639 /*
640  * Process the non-data mblk.
641  */
642 static void
softmac_wput_single_nondata(softmac_upper_t * sup,mblk_t * mp)643 softmac_wput_single_nondata(softmac_upper_t *sup, mblk_t *mp)
644 {
645 	softmac_t *softmac = sup->su_softmac;
646 	softmac_lower_t	*slp = sup->su_slp;
647 	unsigned char	dbtype;
648 	t_uscalar_t	prim;
649 
650 	dbtype = DB_TYPE(mp);
651 	sup->su_is_arp = 0;
652 	switch (dbtype) {
653 	case M_CTL:
654 		sup->su_is_arp = 1;
655 		/* FALLTHROUGH */
656 	case M_IOCTL: {
657 		uint32_t	expected_mode;
658 
659 		if (((struct iocblk *)(mp->b_rptr))->ioc_cmd != SIOCSLIFNAME)
660 			break;
661 
662 		/*
663 		 * Nak the M_IOCTL based on the STREAMS specification.
664 		 */
665 		if (dbtype == M_IOCTL)
666 			miocnak(sup->su_wq, mp, 0, EINVAL);
667 		else
668 			freemsg(mp);
669 
670 		/*
671 		 * This stream is either IP or ARP. See whether
672 		 * we need to setup a dedicated-lower-stream for it.
673 		 */
674 		mutex_enter(&softmac->smac_fp_mutex);
675 
676 		expected_mode = DATAPATH_MODE(softmac);
677 		if (expected_mode == SOFTMAC_SLOWPATH)
678 			sup->su_mode = SOFTMAC_SLOWPATH;
679 		list_insert_head(&softmac->smac_sup_list, sup);
680 		mutex_exit(&softmac->smac_fp_mutex);
681 
682 		/*
683 		 * Setup the fast-path dedicated lower stream if fast-path
684 		 * is expected. Note that no lock is held here, and if
685 		 * smac_expected_mode is changed from SOFTMAC_FASTPATH to
686 		 * SOFTMAC_SLOWPATH, the DL_NOTE_REPLUMB message used for
687 		 * data-path switching would already be queued and will
688 		 * be processed by softmac_wput_single_nondata() later.
689 		 */
690 		if (expected_mode == SOFTMAC_FASTPATH)
691 			(void) softmac_fastpath_setup(sup);
692 		return;
693 	}
694 	case M_PROTO:
695 	case M_PCPROTO:
696 		if (MBLKL(mp) < sizeof (t_uscalar_t)) {
697 			freemsg(mp);
698 			return;
699 		}
700 		prim = ((union DL_primitives *)mp->b_rptr)->dl_primitive;
701 		switch (prim) {
702 		case DL_NOTIFY_IND:
703 			if (MBLKL(mp) < sizeof (dl_notify_ind_t) ||
704 			    ((dl_notify_ind_t *)mp->b_rptr)->dl_notification !=
705 			    DL_NOTE_REPLUMB) {
706 				freemsg(mp);
707 				return;
708 			}
709 			/*
710 			 * This DL_NOTE_REPLUMB message is initiated
711 			 * and queued by the softmac itself, when the
712 			 * sup is trying to switching its datapath mode
713 			 * between SOFTMAC_SLOWPATH and SOFTMAC_FASTPATH.
714 			 * Send this message upstream.
715 			 */
716 			qreply(sup->su_wq, mp);
717 			return;
718 		case DL_NOTIFY_CONF:
719 			if (MBLKL(mp) < sizeof (dl_notify_conf_t) ||
720 			    ((dl_notify_conf_t *)mp->b_rptr)->dl_notification !=
721 			    DL_NOTE_REPLUMB_DONE) {
722 				freemsg(mp);
723 				return;
724 			}
725 			/*
726 			 * This is an indication from IP/ARP that the
727 			 * fastpath->slowpath switch is done.
728 			 */
729 			freemsg(mp);
730 			softmac_datapath_switch_done(sup);
731 			return;
732 		}
733 		break;
734 	}
735 
736 	/*
737 	 * No need to hold lock to check su_mode, since su_mode updating only
738 	 * operation is is serialized by softmac_wput_nondata_task().
739 	 */
740 	if (sup->su_mode != SOFTMAC_FASTPATH) {
741 		(void) dld_wput(sup->su_wq, mp);
742 		return;
743 	}
744 
745 	/*
746 	 * Fastpath non-data message processing. Most of non-data messages
747 	 * can be directly passed down to the dedicated-lower-stream, aside
748 	 * from the following M_PROTO/M_PCPROTO messages.
749 	 */
750 	switch (dbtype) {
751 	case M_PROTO:
752 	case M_PCPROTO:
753 		switch (prim) {
754 		case DL_BIND_REQ:
755 			softmac_bind_req(sup, mp);
756 			break;
757 		case DL_UNBIND_REQ:
758 			softmac_unbind_req(sup, mp);
759 			break;
760 		case DL_CAPABILITY_REQ:
761 			softmac_capability_req(sup, mp);
762 			break;
763 		default:
764 			putnext(slp->sl_wq, mp);
765 			break;
766 		}
767 		break;
768 	default:
769 		putnext(slp->sl_wq, mp);
770 		break;
771 	}
772 }
773 
774 /*
775  * The worker thread which processes non-data messages. Note we only process
776  * one message at one time in order to be able to "flush" the queued message
777  * and serialize the processing.
778  */
779 static void
softmac_wput_nondata_task(void * arg)780 softmac_wput_nondata_task(void *arg)
781 {
782 	softmac_upper_t	*sup = arg;
783 	mblk_t		*mp;
784 
785 	mutex_enter(&sup->su_disp_mutex);
786 
787 	while (sup->su_pending_head != NULL) {
788 		if (sup->su_closing)
789 			break;
790 
791 		SOFTMAC_DQ_PENDING(sup, &mp);
792 		mutex_exit(&sup->su_disp_mutex);
793 		softmac_wput_single_nondata(sup, mp);
794 		mutex_enter(&sup->su_disp_mutex);
795 	}
796 
797 	/*
798 	 * If the stream is closing, flush all queued messages and inform
799 	 * the stream to be closed.
800 	 */
801 	freemsgchain(sup->su_pending_head);
802 	sup->su_pending_head = sup->su_pending_tail = NULL;
803 	sup->su_dlpi_pending = B_FALSE;
804 	cv_signal(&sup->su_disp_cv);
805 	mutex_exit(&sup->su_disp_mutex);
806 }
807 
808 /*
809  * Kernel thread to handle taskq dispatch failures in softmac_wput_nondata().
810  * This thread is started when the softmac module is first loaded.
811  */
812 static void
softmac_taskq_dispatch(void)813 softmac_taskq_dispatch(void)
814 {
815 	callb_cpr_t	cprinfo;
816 	softmac_upper_t	*sup;
817 
818 	CALLB_CPR_INIT(&cprinfo, &softmac_taskq_lock, callb_generic_cpr,
819 	    "softmac_taskq_dispatch");
820 	mutex_enter(&softmac_taskq_lock);
821 
822 	while (!softmac_taskq_quit) {
823 		sup = list_head(&softmac_taskq_list);
824 		while (sup != NULL) {
825 			list_remove(&softmac_taskq_list, sup);
826 			sup->su_taskq_scheduled = B_FALSE;
827 			mutex_exit(&softmac_taskq_lock);
828 			VERIFY(taskq_dispatch(system_taskq,
829 			    softmac_wput_nondata_task, sup, TQ_SLEEP) !=
830 			    TASKQID_INVALID);
831 			mutex_enter(&softmac_taskq_lock);
832 			sup = list_head(&softmac_taskq_list);
833 		}
834 
835 		CALLB_CPR_SAFE_BEGIN(&cprinfo);
836 		cv_wait(&softmac_taskq_cv, &softmac_taskq_lock);
837 		CALLB_CPR_SAFE_END(&cprinfo, &softmac_taskq_lock);
838 	}
839 
840 	softmac_taskq_done = B_TRUE;
841 	cv_signal(&softmac_taskq_cv);
842 	CALLB_CPR_EXIT(&cprinfo);
843 	thread_exit();
844 }
845 
846 void
softmac_wput_nondata(softmac_upper_t * sup,mblk_t * mp)847 softmac_wput_nondata(softmac_upper_t *sup, mblk_t *mp)
848 {
849 	/*
850 	 * The processing of the message might block. Enqueue the
851 	 * message for later processing.
852 	 */
853 	mutex_enter(&sup->su_disp_mutex);
854 
855 	if (sup->su_closing) {
856 		mutex_exit(&sup->su_disp_mutex);
857 		freemsg(mp);
858 		return;
859 	}
860 
861 	SOFTMAC_EQ_PENDING(sup, mp);
862 
863 	if (sup->su_dlpi_pending) {
864 		mutex_exit(&sup->su_disp_mutex);
865 		return;
866 	}
867 	sup->su_dlpi_pending = B_TRUE;
868 	mutex_exit(&sup->su_disp_mutex);
869 
870 	if (taskq_dispatch(system_taskq, softmac_wput_nondata_task,
871 	    sup, TQ_NOSLEEP) != TASKQID_INVALID) {
872 		return;
873 	}
874 
875 	mutex_enter(&softmac_taskq_lock);
876 	if (!sup->su_taskq_scheduled) {
877 		list_insert_tail(&softmac_taskq_list, sup);
878 		cv_signal(&softmac_taskq_cv);
879 	}
880 	sup->su_taskq_scheduled = B_TRUE;
881 	mutex_exit(&softmac_taskq_lock);
882 }
883 
884 /*
885  * Setup the dedicated-lower-stream (fast-path) for the IP/ARP upperstream.
886  */
887 static int
softmac_fastpath_setup(softmac_upper_t * sup)888 softmac_fastpath_setup(softmac_upper_t *sup)
889 {
890 	softmac_t	*softmac = sup->su_softmac;
891 	softmac_lower_t	*slp;
892 	int		err;
893 
894 	err = softmac_lower_setup(softmac, sup, &slp);
895 
896 	mutex_enter(&sup->su_mutex);
897 	/*
898 	 * Wait for all data messages to be processed so that we can change
899 	 * the su_mode.
900 	 */
901 	while (sup->su_tx_inprocess != 0)
902 		cv_wait(&sup->su_cv, &sup->su_mutex);
903 
904 	ASSERT(sup->su_mode != SOFTMAC_FASTPATH);
905 	ASSERT(sup->su_slp == NULL);
906 	if (err != 0) {
907 		sup->su_mode = SOFTMAC_SLOWPATH;
908 	} else {
909 		sup->su_slp = slp;
910 		sup->su_mode = SOFTMAC_FASTPATH;
911 	}
912 	mutex_exit(&sup->su_mutex);
913 	return (err);
914 }
915 
916 /*
917  * Tear down the dedicated-lower-stream (fast-path) for the IP/ARP upperstream.
918  */
919 static void
softmac_fastpath_tear(softmac_upper_t * sup)920 softmac_fastpath_tear(softmac_upper_t *sup)
921 {
922 	mutex_enter(&sup->su_mutex);
923 	/*
924 	 * Wait for all data messages in the dedicated-lower-stream
925 	 * to be processed.
926 	 */
927 	while (sup->su_tx_inprocess != 0)
928 		cv_wait(&sup->su_cv, &sup->su_mutex);
929 
930 	/*
931 	 * Note that this function is called either when the stream is closed,
932 	 * or the stream is unbound (fastpath-slowpath-switch). Therefore,
933 	 * No need to call the tx_notify callback.
934 	 */
935 	sup->su_tx_notify_func = NULL;
936 	sup->su_tx_notify_arg = NULL;
937 	if (sup->su_tx_busy) {
938 		ASSERT(sup->su_tx_flow_mp == NULL);
939 		VERIFY((sup->su_tx_flow_mp = getq(sup->su_wq)) != NULL);
940 		sup->su_tx_busy = B_FALSE;
941 	}
942 
943 	sup->su_mode = SOFTMAC_SLOWPATH;
944 
945 	/*
946 	 * Destroy the dedicated-lower-stream. Note that slp is destroyed
947 	 * when lh is closed.
948 	 */
949 	(void) ldi_close(sup->su_slp->sl_lh, FREAD|FWRITE, kcred);
950 	sup->su_slp = NULL;
951 	mutex_exit(&sup->su_mutex);
952 }
953 
954 void
softmac_wput_data(softmac_upper_t * sup,mblk_t * mp)955 softmac_wput_data(softmac_upper_t *sup, mblk_t *mp)
956 {
957 	/*
958 	 * No lock is required to access the su_mode field since the data
959 	 * traffic is quiesce by IP when the data-path mode is in the
960 	 * process of switching.
961 	 */
962 	if (sup->su_mode != SOFTMAC_FASTPATH)
963 		(void) dld_wput(sup->su_wq, mp);
964 	else
965 		(void) softmac_fastpath_wput_data(sup, mp, (uintptr_t)NULL, 0);
966 }
967 
968 /*ARGSUSED*/
969 static mac_tx_cookie_t
softmac_fastpath_wput_data(softmac_upper_t * sup,mblk_t * mp,uintptr_t f_hint,uint16_t flag)970 softmac_fastpath_wput_data(softmac_upper_t *sup, mblk_t *mp, uintptr_t f_hint,
971     uint16_t flag)
972 {
973 	queue_t		*wq = sup->su_slp->sl_wq;
974 
975 	/*
976 	 * This function is called from IP, only the MAC_DROP_ON_NO_DESC
977 	 * flag can be specified.
978 	 */
979 	ASSERT((flag & ~MAC_DROP_ON_NO_DESC) == 0);
980 	ASSERT(mp->b_next == NULL);
981 
982 	/*
983 	 * Check wether the dedicated-lower-stream is able to handle more
984 	 * messages, and enable the flow-control if it is not.
985 	 *
986 	 * Note that in order not to introduce any packet reordering, we
987 	 * always send the message down to the dedicated-lower-stream:
988 	 *
989 	 * If the flow-control is already enabled, but we still get
990 	 * the messages from the upper-stream, it means that the upper
991 	 * stream does not respect STREAMS flow-control (e.g., TCP). Simply
992 	 * pass the message down to the lower-stream in that case.
993 	 */
994 	if (SOFTMAC_CANPUTNEXT(wq)) {
995 		putnext(wq, mp);
996 		return ((mac_tx_cookie_t)NULL);
997 	}
998 
999 	if (sup->su_tx_busy) {
1000 		if ((flag & MAC_DROP_ON_NO_DESC) != 0)
1001 			freemsg(mp);
1002 		else
1003 			putnext(wq, mp);
1004 		return ((mac_tx_cookie_t)sup);
1005 	}
1006 
1007 	mutex_enter(&sup->su_mutex);
1008 	if (!sup->su_tx_busy) {
1009 		/*
1010 		 * If DLD_CAPAB_DIRECT is enabled, the notify callback will be
1011 		 * called when the flow control can be disabled. Otherwise,
1012 		 * put the tx_flow_mp into the wq to make use of the old
1013 		 * streams flow control.
1014 		 */
1015 		ASSERT(sup->su_tx_flow_mp != NULL);
1016 		(void) putq(sup->su_wq, sup->su_tx_flow_mp);
1017 		sup->su_tx_flow_mp = NULL;
1018 		sup->su_tx_busy = B_TRUE;
1019 		qenable(wq);
1020 	}
1021 	mutex_exit(&sup->su_mutex);
1022 
1023 	if ((flag & MAC_DROP_ON_NO_DESC) != 0)
1024 		freemsg(mp);
1025 	else
1026 		putnext(wq, mp);
1027 	return ((mac_tx_cookie_t)sup);
1028 }
1029 
1030 boolean_t
softmac_active_set(void * arg)1031 softmac_active_set(void *arg)
1032 {
1033 	softmac_t	*softmac = arg;
1034 
1035 	mutex_enter(&softmac->smac_active_mutex);
1036 	if (softmac->smac_nactive != 0) {
1037 		mutex_exit(&softmac->smac_active_mutex);
1038 		return (B_FALSE);
1039 	}
1040 	softmac->smac_active = B_TRUE;
1041 	mutex_exit(&softmac->smac_active_mutex);
1042 	return (B_TRUE);
1043 }
1044 
1045 void
softmac_active_clear(void * arg)1046 softmac_active_clear(void *arg)
1047 {
1048 	softmac_t	*softmac = arg;
1049 
1050 	mutex_enter(&softmac->smac_active_mutex);
1051 	ASSERT(softmac->smac_active && (softmac->smac_nactive == 0));
1052 	softmac->smac_active = B_FALSE;
1053 	mutex_exit(&softmac->smac_active_mutex);
1054 }
1055 
1056 /*
1057  * Disable/reenable fastpath on given softmac. This request could come from a
1058  * MAC client or directly from administrators.
1059  */
1060 int
softmac_datapath_switch(softmac_t * softmac,boolean_t disable,boolean_t admin)1061 softmac_datapath_switch(softmac_t *softmac, boolean_t disable, boolean_t admin)
1062 {
1063 	softmac_upper_t		*sup;
1064 	mblk_t			*head = NULL, *tail = NULL, *mp;
1065 	list_t			reqlist;
1066 	softmac_switch_req_t	*req;
1067 	uint32_t		current_mode, expected_mode;
1068 	int			err = 0;
1069 
1070 	mutex_enter(&softmac->smac_fp_mutex);
1071 
1072 	current_mode = DATAPATH_MODE(softmac);
1073 	if (admin) {
1074 		if (softmac->smac_fastpath_admin_disabled == disable) {
1075 			mutex_exit(&softmac->smac_fp_mutex);
1076 			return (0);
1077 		}
1078 		softmac->smac_fastpath_admin_disabled = disable;
1079 	} else if (disable) {
1080 		softmac->smac_fp_disable_clients++;
1081 	} else {
1082 		ASSERT(softmac->smac_fp_disable_clients != 0);
1083 		softmac->smac_fp_disable_clients--;
1084 	}
1085 
1086 	expected_mode = DATAPATH_MODE(softmac);
1087 	if (current_mode == expected_mode) {
1088 		mutex_exit(&softmac->smac_fp_mutex);
1089 		return (0);
1090 	}
1091 
1092 	/*
1093 	 * The expected mode is different from whatever datapath mode
1094 	 * this softmac is expected from last request, enqueue the data-path
1095 	 * switch request.
1096 	 */
1097 	list_create(&reqlist, sizeof (softmac_switch_req_t),
1098 	    offsetof(softmac_switch_req_t, ssq_req_list_node));
1099 
1100 	/*
1101 	 * Allocate all DL_NOTIFY_IND messages and request structures that
1102 	 * are required to switch each IP/ARP stream to the expected mode.
1103 	 */
1104 	for (sup = list_head(&softmac->smac_sup_list); sup != NULL;
1105 	    sup = list_next(&softmac->smac_sup_list, sup)) {
1106 		dl_notify_ind_t	*dlip;
1107 
1108 		req = kmem_alloc(sizeof (softmac_switch_req_t), KM_NOSLEEP);
1109 		if (req == NULL)
1110 			break;
1111 
1112 		req->ssq_expected_mode = expected_mode;
1113 		if (sup->su_is_arp) {
1114 			list_insert_tail(&reqlist, req);
1115 			continue;
1116 		}
1117 		/*
1118 		 * Allocate the DL_NOTE_REPLUMB message.
1119 		 */
1120 		if ((mp = allocb(sizeof (dl_notify_ind_t), BPRI_LO)) == NULL) {
1121 			kmem_free(req, sizeof (softmac_switch_req_t));
1122 			break;
1123 		}
1124 
1125 		list_insert_tail(&reqlist, req);
1126 
1127 		mp->b_wptr = mp->b_rptr + sizeof (dl_notify_ind_t);
1128 		mp->b_datap->db_type = M_PROTO;
1129 		bzero(mp->b_rptr, sizeof (dl_notify_ind_t));
1130 		dlip = (dl_notify_ind_t *)mp->b_rptr;
1131 		dlip->dl_primitive = DL_NOTIFY_IND;
1132 		dlip->dl_notification = DL_NOTE_REPLUMB;
1133 		if (head == NULL) {
1134 			head = tail = mp;
1135 		} else {
1136 			tail->b_next = mp;
1137 			tail = mp;
1138 		}
1139 	}
1140 
1141 	/*
1142 	 * Note that it is fine if the expected data-path mode is fast-path
1143 	 * and some of streams fails to switch. Only return failure if we
1144 	 * are expected to switch to the slow-path.
1145 	 */
1146 	if (sup != NULL && expected_mode == SOFTMAC_SLOWPATH) {
1147 		err = ENOMEM;
1148 		goto fail;
1149 	}
1150 
1151 	/*
1152 	 * Start switching for each IP/ARP stream. The switching operation
1153 	 * will eventually succeed and there is no need to wait for it
1154 	 * to finish.
1155 	 */
1156 	for (sup = list_head(&softmac->smac_sup_list); sup != NULL;
1157 	    sup = list_next(&softmac->smac_sup_list, sup)) {
1158 		if (!sup->su_is_arp) {
1159 			mp = head->b_next;
1160 			head->b_next = NULL;
1161 			softmac_wput_nondata(sup, head);
1162 			head = mp;
1163 		}
1164 		/*
1165 		 * Add the switch request to the requests list of the stream.
1166 		 */
1167 		req = list_head(&reqlist);
1168 		ASSERT(req != NULL);
1169 		list_remove(&reqlist, req);
1170 		list_insert_tail(&sup->su_req_list, req);
1171 	}
1172 
1173 	mutex_exit(&softmac->smac_fp_mutex);
1174 	ASSERT(list_is_empty(&reqlist));
1175 	list_destroy(&reqlist);
1176 	return (0);
1177 fail:
1178 	if (admin) {
1179 		softmac->smac_fastpath_admin_disabled = !disable;
1180 	} else if (disable) {
1181 		softmac->smac_fp_disable_clients--;
1182 	} else {
1183 		softmac->smac_fp_disable_clients++;
1184 	}
1185 
1186 	mutex_exit(&softmac->smac_fp_mutex);
1187 	while ((req = list_head(&reqlist)) != NULL) {
1188 		list_remove(&reqlist, req);
1189 		kmem_free(req, sizeof (softmac_switch_req_t));
1190 	}
1191 	freemsgchain(head);
1192 	list_destroy(&reqlist);
1193 	return (err);
1194 }
1195 
1196 int
softmac_fastpath_disable(void * arg)1197 softmac_fastpath_disable(void *arg)
1198 {
1199 	return (softmac_datapath_switch((softmac_t *)arg, B_TRUE, B_FALSE));
1200 }
1201 
1202 void
softmac_fastpath_enable(void * arg)1203 softmac_fastpath_enable(void *arg)
1204 {
1205 	VERIFY(softmac_datapath_switch((softmac_t *)arg, B_FALSE,
1206 	    B_FALSE) == 0);
1207 }
1208 
1209 void
softmac_upperstream_close(softmac_upper_t * sup)1210 softmac_upperstream_close(softmac_upper_t *sup)
1211 {
1212 	softmac_t		*softmac = sup->su_softmac;
1213 	softmac_switch_req_t	*req;
1214 
1215 	mutex_enter(&softmac->smac_fp_mutex);
1216 
1217 	if (sup->su_mode == SOFTMAC_FASTPATH)
1218 		softmac_fastpath_tear(sup);
1219 
1220 	if (sup->su_mode != SOFTMAC_UNKNOWN) {
1221 		list_remove(&softmac->smac_sup_list, sup);
1222 		sup->su_mode = SOFTMAC_UNKNOWN;
1223 	}
1224 
1225 	/*
1226 	 * Cleanup all the switch requests queueed on this stream.
1227 	 */
1228 	while ((req = list_head(&sup->su_req_list)) != NULL) {
1229 		list_remove(&sup->su_req_list, req);
1230 		kmem_free(req, sizeof (softmac_switch_req_t));
1231 	}
1232 	mutex_exit(&softmac->smac_fp_mutex);
1233 }
1234 
1235 /*
1236  * Handle the DL_NOTE_REPLUMB_DONE indication from IP/ARP. Change the upper
1237  * stream from the fastpath mode to the slowpath mode.
1238  */
1239 static void
softmac_datapath_switch_done(softmac_upper_t * sup)1240 softmac_datapath_switch_done(softmac_upper_t *sup)
1241 {
1242 	softmac_t		*softmac = sup->su_softmac;
1243 	softmac_switch_req_t	*req;
1244 	uint32_t		expected_mode;
1245 
1246 	mutex_enter(&softmac->smac_fp_mutex);
1247 	req = list_head(&sup->su_req_list);
1248 	list_remove(&sup->su_req_list, req);
1249 	expected_mode = req->ssq_expected_mode;
1250 	kmem_free(req, sizeof (softmac_switch_req_t));
1251 
1252 	if (expected_mode == sup->su_mode) {
1253 		mutex_exit(&softmac->smac_fp_mutex);
1254 		return;
1255 	}
1256 
1257 	ASSERT(!sup->su_bound);
1258 	mutex_exit(&softmac->smac_fp_mutex);
1259 
1260 	/*
1261 	 * It is fine if the expected mode is fast-path and we fail
1262 	 * to enable fastpath on this stream.
1263 	 */
1264 	if (expected_mode == SOFTMAC_SLOWPATH)
1265 		softmac_fastpath_tear(sup);
1266 	else
1267 		(void) softmac_fastpath_setup(sup);
1268 }
1269