1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 /*
27 * Softmac data-path switching:
28 *
29 * - Fast-path model
30 *
31 * When the softmac fast-path is used, a dedicated lower-stream
32 * will be opened over the legacy device for each IP/ARP (upper-)stream
33 * over the softMAC, and all DLPI messages (including control messages
34 * and data messages) will be exchanged between the upper-stream and
35 * the corresponding lower-stream directly. Therefore, the data
36 * demultiplexing, filtering and classification processing will be done
37 * by the lower-stream, and the GLDv3 DLS/MAC layer processing will be
38 * no longer needed.
39 *
40 * - Slow-path model
41 *
42 * Some GLDv3 features requires the GLDv3 DLS/MAC layer processing to
43 * not be bypassed to assure its function correctness. For example,
44 * softmac fast-path must be disabled to support GLDv3 VNIC functionality.
45 * In this case, a shared lower-stream will be opened over the legacy
46 * device, which is responsible for implementing the GLDv3 callbacks
47 * and passing RAW data messages between the legacy devices and the GLDv3
48 * framework.
49 *
50 * By default, the softmac fast-path mode will be used to assure the
51 * performance; MAC clients will be able to request to disable the softmac
52 * fast-path mode to support certain features, and if that succeeds,
53 * the system will fallback to the slow-path softmac data-path model.
54 *
55 *
56 * The details of the softmac data fast-path model is stated as below
57 *
58 * 1. When a stream is opened on a softMAC, the softmac module will takes
59 * over the DLPI processing on this stream;
60 *
61 * 2. For IP/ARP streams over a softMAC, softmac data fast-path will be
62 * used by default, unless fast-path is disabled by any MAC client
63 * explicitly. The softmac module first identifies an IP/ARP stream
64 * by seeing whether there is a SIOCSLIFNAME ioctl sent from upstream,
65 * if there is one, this stream is either an IP or an ARP stream
66 * and will use fast-path potentially;
67 *
68 * 3. When the softmac fast-path is used, an dedicated lower-stream will
69 * be setup for each IP/ARP stream (1-1 mapping). From that point on,
70 * all control and data messages will be exchanged between the IP/ARP
71 * upper-stream and the legacy device through this dedicated
72 * lower-stream. As a result, the DLS/MAC layer processing in GLDv3
73 * will be skipped, and this greatly improves the performance;
74 *
75 * 4. When the softmac data fast-path is disabled by a MAC client (e.g.,
76 * by a VNIC), all the IP/ARP upper streams will try to switch from
77 * the fast-path to the slow-path. The dedicated lower-stream will be
78 * destroyed, and all the control and data-messages will go through the
79 * existing GLDv3 code path and (in the end) the shared lower-stream;
80 *
81 * 5. On the other hand, when the last MAC client cancels its fast-path
82 * disable request, all the IP/ARP streams will try to switch back to
83 * the fast-path mode;
84 *
85 * Step 5 and 6 both rely on the data-path mode switching process
86 * described below:
87 *
88 * 1) To switch the softmac data-path mode (between fast-path and slow-path),
89 * softmac will first send a DL_NOTE_REPLUMB DL_NOTIFY_IND message
90 * upstream over each IP/ARP streams that needs data-path mode switching;
91 *
92 * 2) When IP receives this DL_NOTE_REPLUMB message, it will bring down
93 * all the IP interfaces on the corresponding ill (IP Lower level
94 * structure), and bring up those interfaces over again; this will in
95 * turn cause the ARP to "replumb" the interface.
96 *
97 * During the replumb process, both IP and ARP will send downstream the
98 * necessary DL_DISABMULTI_REQ and DL_UNBIND_REQ messages and cleanup
99 * the old state of the underlying softMAC, following with the necessary
100 * DL_BIND_REQ and DL_ENABMULTI_REQ messages to setup the new state.
101 * Between the cleanup and re-setup process, IP/ARP will also send down
102 * a DL_NOTE_REPLUMB_DONE DL_NOTIFY_CONF messages to the softMAC to
103 * indicate the *switching point*;
104 *
105 * 3) When softmac receives the DL_NOTE_REPLUMB_DONE message, it either
106 * creates or destroys the dedicated lower-stream (depending on which
107 * data-path mode the softMAC switches to), and change the softmac
108 * data-path mode. From then on, softmac will process all the succeeding
109 * control messages (including the DL_BIND_REQ and DL_ENABMULTI_REQ
110 * messages) and data messages based on new data-path mode.
111 */
112
113 #include <sys/types.h>
114 #include <sys/disp.h>
115 #include <sys/callb.h>
116 #include <sys/sysmacros.h>
117 #include <sys/file.h>
118 #include <sys/vlan.h>
119 #include <sys/dld.h>
120 #include <sys/sockio.h>
121 #include <sys/softmac_impl.h>
122 #include <net/if.h>
123
124 static kmutex_t softmac_taskq_lock;
125 static kcondvar_t softmac_taskq_cv;
126 static list_t softmac_taskq_list; /* List of softmac_upper_t */
127 boolean_t softmac_taskq_quit;
128 boolean_t softmac_taskq_done;
129
130 static void softmac_taskq_dispatch();
131 static int softmac_fastpath_setup(softmac_upper_t *);
132 static mac_tx_cookie_t softmac_fastpath_wput_data(softmac_upper_t *, mblk_t *,
133 uintptr_t, uint16_t);
134 static void softmac_datapath_switch_done(softmac_upper_t *);
135
136 void
softmac_fp_init()137 softmac_fp_init()
138 {
139 mutex_init(&softmac_taskq_lock, NULL, MUTEX_DRIVER, NULL);
140 cv_init(&softmac_taskq_cv, NULL, CV_DRIVER, NULL);
141
142 softmac_taskq_quit = B_FALSE;
143 softmac_taskq_done = B_FALSE;
144 list_create(&softmac_taskq_list, sizeof (softmac_upper_t),
145 offsetof(softmac_upper_t, su_taskq_list_node));
146 (void) thread_create(NULL, 0, softmac_taskq_dispatch, NULL, 0,
147 &p0, TS_RUN, minclsyspri);
148 }
149
150 void
softmac_fp_fini()151 softmac_fp_fini()
152 {
153 /*
154 * Request the softmac_taskq thread to quit and wait for it to be done.
155 */
156 mutex_enter(&softmac_taskq_lock);
157 softmac_taskq_quit = B_TRUE;
158 cv_signal(&softmac_taskq_cv);
159 while (!softmac_taskq_done)
160 cv_wait(&softmac_taskq_cv, &softmac_taskq_lock);
161 mutex_exit(&softmac_taskq_lock);
162 list_destroy(&softmac_taskq_list);
163
164 mutex_destroy(&softmac_taskq_lock);
165 cv_destroy(&softmac_taskq_cv);
166 }
167
168 static boolean_t
check_ip_above(queue_t * q)169 check_ip_above(queue_t *q)
170 {
171 queue_t *next_q;
172 boolean_t ret = B_TRUE;
173
174 claimstr(q);
175 next_q = q->q_next;
176 if (strcmp(next_q->q_qinfo->qi_minfo->mi_idname, "ip") != 0)
177 ret = B_FALSE;
178 releasestr(q);
179 return (ret);
180 }
181
182 /* ARGSUSED */
183 static int
softmac_capab_perim(softmac_upper_t * sup,void * data,uint_t flags)184 softmac_capab_perim(softmac_upper_t *sup, void *data, uint_t flags)
185 {
186 switch (flags) {
187 case DLD_ENABLE:
188 mutex_enter(&sup->su_mutex);
189 break;
190 case DLD_DISABLE:
191 mutex_exit(&sup->su_mutex);
192 break;
193 case DLD_QUERY:
194 return (MUTEX_HELD(&sup->su_mutex));
195 }
196 return (0);
197 }
198
199 static mac_tx_notify_handle_t
softmac_client_tx_notify(softmac_upper_t * sup,mac_tx_notify_t func,void * arg)200 softmac_client_tx_notify(softmac_upper_t *sup, mac_tx_notify_t func, void *arg)
201 {
202 ASSERT(MUTEX_HELD(&sup->su_mutex));
203
204 if (func != NULL) {
205 sup->su_tx_notify_func = func;
206 sup->su_tx_notify_arg = arg;
207 } else {
208 /*
209 * Wait for all tx_notify_func call to be done.
210 */
211 while (sup->su_tx_inprocess != 0)
212 cv_wait(&sup->su_cv, &sup->su_mutex);
213
214 sup->su_tx_notify_func = NULL;
215 sup->su_tx_notify_arg = NULL;
216 }
217 return ((mac_tx_notify_handle_t)sup);
218 }
219
220 static boolean_t
softmac_tx_is_flow_blocked(softmac_upper_t * sup,mac_tx_cookie_t cookie)221 softmac_tx_is_flow_blocked(softmac_upper_t *sup, mac_tx_cookie_t cookie)
222 {
223 ASSERT(cookie == (mac_tx_cookie_t)sup);
224 return (sup->su_tx_busy);
225 }
226
227 static int
softmac_capab_direct(softmac_upper_t * sup,void * data,uint_t flags)228 softmac_capab_direct(softmac_upper_t *sup, void *data, uint_t flags)
229 {
230 dld_capab_direct_t *direct = data;
231 softmac_lower_t *slp = sup->su_slp;
232
233 ASSERT(MUTEX_HELD(&sup->su_mutex));
234
235 ASSERT(sup->su_mode == SOFTMAC_FASTPATH);
236
237 switch (flags) {
238 case DLD_ENABLE:
239 if (sup->su_direct)
240 return (0);
241
242 sup->su_direct_rxinfo.slr_rx = (softmac_rx_t)direct->di_rx_cf;
243 sup->su_direct_rxinfo.slr_arg = direct->di_rx_ch;
244 slp->sl_rxinfo = &sup->su_direct_rxinfo;
245 direct->di_tx_df = (uintptr_t)softmac_fastpath_wput_data;
246 direct->di_tx_dh = sup;
247 direct->di_tx_fctl_df = (uintptr_t)softmac_tx_is_flow_blocked;
248 direct->di_tx_fctl_dh = sup;
249 direct->di_tx_cb_df = (uintptr_t)softmac_client_tx_notify;
250 direct->di_tx_cb_dh = sup;
251 sup->su_direct = B_TRUE;
252 return (0);
253
254 case DLD_DISABLE:
255 if (!sup->su_direct)
256 return (0);
257
258 slp->sl_rxinfo = &sup->su_rxinfo;
259 sup->su_direct = B_FALSE;
260 return (0);
261 }
262 return (ENOTSUP);
263 }
264
265 static int
softmac_dld_capab(softmac_upper_t * sup,uint_t type,void * data,uint_t flags)266 softmac_dld_capab(softmac_upper_t *sup, uint_t type, void *data, uint_t flags)
267 {
268 int err;
269
270 /*
271 * Don't enable direct callback capabilities unless the caller is
272 * the IP client. When a module is inserted in a stream (_I_INSERT)
273 * the stack initiates capability disable, but due to races, the
274 * module insertion may complete before the capability disable
275 * completes. So we limit the check to DLD_ENABLE case.
276 */
277 if ((flags == DLD_ENABLE && type != DLD_CAPAB_PERIM) &&
278 !check_ip_above(sup->su_rq)) {
279 return (ENOTSUP);
280 }
281
282 switch (type) {
283 case DLD_CAPAB_DIRECT:
284 err = softmac_capab_direct(sup, data, flags);
285 break;
286
287 case DLD_CAPAB_PERIM:
288 err = softmac_capab_perim(sup, data, flags);
289 break;
290
291 default:
292 err = ENOTSUP;
293 break;
294 }
295 return (err);
296 }
297
298 static void
softmac_capability_advertise(softmac_upper_t * sup,mblk_t * mp)299 softmac_capability_advertise(softmac_upper_t *sup, mblk_t *mp)
300 {
301 dl_capability_ack_t *dlap;
302 dl_capability_sub_t *dlsp;
303 t_uscalar_t subsize;
304 uint8_t *ptr;
305 queue_t *q = sup->su_wq;
306 mblk_t *mp1;
307 softmac_t *softmac = sup->su_softmac;
308 boolean_t dld_capable = B_FALSE;
309 boolean_t hcksum_capable = B_FALSE;
310 boolean_t zcopy_capable = B_FALSE;
311 boolean_t mdt_capable = B_FALSE;
312
313 ASSERT(sup->su_mode == SOFTMAC_FASTPATH);
314
315 /*
316 * Initially assume no capabilities.
317 */
318 subsize = 0;
319
320 /*
321 * Direct capability negotiation interface between IP and softmac
322 */
323 if (check_ip_above(sup->su_rq)) {
324 dld_capable = B_TRUE;
325 subsize += sizeof (dl_capability_sub_t) +
326 sizeof (dl_capab_dld_t);
327 }
328
329 /*
330 * Check if checksum offload is supported on this MAC.
331 */
332 if (softmac->smac_capab_flags & MAC_CAPAB_HCKSUM) {
333 hcksum_capable = B_TRUE;
334 subsize += sizeof (dl_capability_sub_t) +
335 sizeof (dl_capab_hcksum_t);
336 }
337
338 /*
339 * Check if zerocopy is supported on this interface.
340 */
341 if (!(softmac->smac_capab_flags & MAC_CAPAB_NO_ZCOPY)) {
342 zcopy_capable = B_TRUE;
343 subsize += sizeof (dl_capability_sub_t) +
344 sizeof (dl_capab_zerocopy_t);
345 }
346
347 if (softmac->smac_mdt) {
348 mdt_capable = B_TRUE;
349 subsize += sizeof (dl_capability_sub_t) +
350 sizeof (dl_capab_mdt_t);
351 }
352
353 /*
354 * If there are no capabilities to advertise or if we
355 * can't allocate a response, send a DL_ERROR_ACK.
356 */
357 if ((subsize == 0) || (mp1 = reallocb(mp,
358 sizeof (dl_capability_ack_t) + subsize, 0)) == NULL) {
359 dlerrorack(q, mp, DL_CAPABILITY_REQ, DL_NOTSUPPORTED, 0);
360 return;
361 }
362
363 mp = mp1;
364 DB_TYPE(mp) = M_PROTO;
365 mp->b_wptr = mp->b_rptr + sizeof (dl_capability_ack_t) + subsize;
366 bzero(mp->b_rptr, MBLKL(mp));
367 dlap = (dl_capability_ack_t *)mp->b_rptr;
368 dlap->dl_primitive = DL_CAPABILITY_ACK;
369 dlap->dl_sub_offset = sizeof (dl_capability_ack_t);
370 dlap->dl_sub_length = subsize;
371 ptr = (uint8_t *)&dlap[1];
372
373 /*
374 * IP polling interface.
375 */
376 if (dld_capable) {
377 dl_capab_dld_t dld;
378
379 dlsp = (dl_capability_sub_t *)ptr;
380 dlsp->dl_cap = DL_CAPAB_DLD;
381 dlsp->dl_length = sizeof (dl_capab_dld_t);
382 ptr += sizeof (dl_capability_sub_t);
383
384 bzero(&dld, sizeof (dl_capab_dld_t));
385 dld.dld_version = DLD_CURRENT_VERSION;
386 dld.dld_capab = (uintptr_t)softmac_dld_capab;
387 dld.dld_capab_handle = (uintptr_t)sup;
388
389 dlcapabsetqid(&(dld.dld_mid), sup->su_rq);
390 bcopy(&dld, ptr, sizeof (dl_capab_dld_t));
391 ptr += sizeof (dl_capab_dld_t);
392 }
393
394 /*
395 * TCP/IP checksum offload.
396 */
397 if (hcksum_capable) {
398 dl_capab_hcksum_t hcksum;
399
400 dlsp = (dl_capability_sub_t *)ptr;
401
402 dlsp->dl_cap = DL_CAPAB_HCKSUM;
403 dlsp->dl_length = sizeof (dl_capab_hcksum_t);
404 ptr += sizeof (dl_capability_sub_t);
405
406 bzero(&hcksum, sizeof (dl_capab_hcksum_t));
407 hcksum.hcksum_version = HCKSUM_VERSION_1;
408 hcksum.hcksum_txflags = softmac->smac_hcksum_txflags;
409 dlcapabsetqid(&(hcksum.hcksum_mid), sup->su_rq);
410 bcopy(&hcksum, ptr, sizeof (dl_capab_hcksum_t));
411 ptr += sizeof (dl_capab_hcksum_t);
412 }
413
414 /*
415 * Zero copy
416 */
417 if (zcopy_capable) {
418 dl_capab_zerocopy_t zcopy;
419
420 dlsp = (dl_capability_sub_t *)ptr;
421
422 dlsp->dl_cap = DL_CAPAB_ZEROCOPY;
423 dlsp->dl_length = sizeof (dl_capab_zerocopy_t);
424 ptr += sizeof (dl_capability_sub_t);
425
426 bzero(&zcopy, sizeof (dl_capab_zerocopy_t));
427 zcopy.zerocopy_version = ZEROCOPY_VERSION_1;
428 zcopy.zerocopy_flags = DL_CAPAB_VMSAFE_MEM;
429 dlcapabsetqid(&(zcopy.zerocopy_mid), sup->su_rq);
430 bcopy(&zcopy, ptr, sizeof (dl_capab_zerocopy_t));
431 ptr += sizeof (dl_capab_zerocopy_t);
432 }
433
434 /*
435 * MDT
436 */
437 if (mdt_capable) {
438 dl_capab_mdt_t mdt;
439
440 dlsp = (dl_capability_sub_t *)ptr;
441
442 dlsp->dl_cap = DL_CAPAB_MDT;
443 dlsp->dl_length = sizeof (dl_capab_mdt_t);
444 ptr += sizeof (dl_capability_sub_t);
445
446 bzero(&mdt, sizeof (dl_capab_mdt_t));
447 mdt.mdt_version = MDT_VERSION_2;
448 mdt.mdt_flags = DL_CAPAB_MDT_ENABLE;
449 mdt.mdt_hdr_head = softmac->smac_mdt_capab.mdt_hdr_head;
450 mdt.mdt_hdr_tail = softmac->smac_mdt_capab.mdt_hdr_tail;
451 mdt.mdt_max_pld = softmac->smac_mdt_capab.mdt_max_pld;
452 mdt.mdt_span_limit = softmac->smac_mdt_capab.mdt_span_limit;
453 dlcapabsetqid(&(mdt.mdt_mid), sup->su_rq);
454 bcopy(&mdt, ptr, sizeof (dl_capab_mdt_t));
455 ptr += sizeof (dl_capab_mdt_t);
456 }
457
458 ASSERT(ptr == mp->b_rptr + sizeof (dl_capability_ack_t) + subsize);
459 qreply(q, mp);
460 }
461
462 static void
softmac_capability_req(softmac_upper_t * sup,mblk_t * mp)463 softmac_capability_req(softmac_upper_t *sup, mblk_t *mp)
464 {
465 dl_capability_req_t *dlp = (dl_capability_req_t *)mp->b_rptr;
466 dl_capability_sub_t *sp;
467 size_t size, len;
468 offset_t off, end;
469 t_uscalar_t dl_err;
470 queue_t *q = sup->su_wq;
471
472 ASSERT(sup->su_mode == SOFTMAC_FASTPATH);
473 if (MBLKL(mp) < sizeof (dl_capability_req_t)) {
474 dl_err = DL_BADPRIM;
475 goto failed;
476 }
477
478 if (!sup->su_bound) {
479 dl_err = DL_OUTSTATE;
480 goto failed;
481 }
482
483 /*
484 * This request is overloaded. If there are no requested capabilities
485 * then we just want to acknowledge with all the capabilities we
486 * support. Otherwise we enable the set of capabilities requested.
487 */
488 if (dlp->dl_sub_length == 0) {
489 softmac_capability_advertise(sup, mp);
490 return;
491 }
492
493 if (!MBLKIN(mp, dlp->dl_sub_offset, dlp->dl_sub_length)) {
494 dl_err = DL_BADPRIM;
495 goto failed;
496 }
497
498 dlp->dl_primitive = DL_CAPABILITY_ACK;
499
500 off = dlp->dl_sub_offset;
501 len = dlp->dl_sub_length;
502
503 /*
504 * Walk the list of capabilities to be enabled.
505 */
506 for (end = off + len; off < end; ) {
507 sp = (dl_capability_sub_t *)(mp->b_rptr + off);
508 size = sizeof (dl_capability_sub_t) + sp->dl_length;
509
510 if (off + size > end ||
511 !IS_P2ALIGNED(off, sizeof (uint32_t))) {
512 dl_err = DL_BADPRIM;
513 goto failed;
514 }
515
516 switch (sp->dl_cap) {
517 /*
518 * TCP/IP checksum offload to hardware.
519 */
520 case DL_CAPAB_HCKSUM: {
521 dl_capab_hcksum_t *hcksump;
522 dl_capab_hcksum_t hcksum;
523
524 hcksump = (dl_capab_hcksum_t *)&sp[1];
525 /*
526 * Copy for alignment.
527 */
528 bcopy(hcksump, &hcksum, sizeof (dl_capab_hcksum_t));
529 dlcapabsetqid(&(hcksum.hcksum_mid), sup->su_rq);
530 bcopy(&hcksum, hcksump, sizeof (dl_capab_hcksum_t));
531 break;
532 }
533
534 default:
535 break;
536 }
537
538 off += size;
539 }
540 qreply(q, mp);
541 return;
542 failed:
543 dlerrorack(q, mp, DL_CAPABILITY_REQ, dl_err, 0);
544 }
545
546 static void
softmac_bind_req(softmac_upper_t * sup,mblk_t * mp)547 softmac_bind_req(softmac_upper_t *sup, mblk_t *mp)
548 {
549 softmac_lower_t *slp = sup->su_slp;
550 softmac_t *softmac = sup->su_softmac;
551 mblk_t *ackmp, *mp1;
552 int err;
553
554 if (MBLKL(mp) < DL_BIND_REQ_SIZE) {
555 freemsg(mp);
556 return;
557 }
558
559 /*
560 * Allocate ackmp incase the underlying driver does not ack timely.
561 */
562 if ((mp1 = allocb(sizeof (dl_error_ack_t), BPRI_HI)) == NULL) {
563 dlerrorack(sup->su_wq, mp, DL_BIND_REQ, DL_SYSERR, ENOMEM);
564 return;
565 }
566
567 err = softmac_output(slp, mp, DL_BIND_REQ, DL_BIND_ACK, &ackmp);
568 if (ackmp != NULL) {
569 freemsg(mp1);
570 } else {
571 /*
572 * The driver does not ack timely.
573 */
574 ASSERT(err == ENOMSG);
575 ackmp = mp1;
576 }
577 if (err != 0)
578 goto failed;
579
580 /*
581 * Enable capabilities the underlying driver claims to support.
582 */
583 if ((err = softmac_capab_enable(slp)) != 0)
584 goto failed;
585
586 /*
587 * Check whether this softmac is already marked as exclusively used,
588 * e.g., an aggregation is created over it. Fail the BIND_REQ if so.
589 */
590 mutex_enter(&softmac->smac_active_mutex);
591 if (softmac->smac_active) {
592 mutex_exit(&softmac->smac_active_mutex);
593 err = EBUSY;
594 goto failed;
595 }
596 softmac->smac_nactive++;
597 sup->su_active = B_TRUE;
598 mutex_exit(&softmac->smac_active_mutex);
599 sup->su_bound = B_TRUE;
600
601 qreply(sup->su_wq, ackmp);
602 return;
603 failed:
604 if (err != 0) {
605 dlerrorack(sup->su_wq, ackmp, DL_BIND_REQ, DL_SYSERR, err);
606 return;
607 }
608 }
609
610 static void
softmac_unbind_req(softmac_upper_t * sup,mblk_t * mp)611 softmac_unbind_req(softmac_upper_t *sup, mblk_t *mp)
612 {
613 softmac_lower_t *slp = sup->su_slp;
614 softmac_t *softmac = sup->su_softmac;
615 mblk_t *ackmp, *mp1;
616 int err;
617
618 if (MBLKL(mp) < DL_UNBIND_REQ_SIZE) {
619 freemsg(mp);
620 return;
621 }
622
623 if (!sup->su_bound) {
624 dlerrorack(sup->su_wq, mp, DL_UNBIND_REQ, DL_OUTSTATE, 0);
625 return;
626 }
627
628 /*
629 * Allocate ackmp incase the underlying driver does not ack timely.
630 */
631 if ((mp1 = allocb(sizeof (dl_error_ack_t), BPRI_HI)) == NULL) {
632 dlerrorack(sup->su_wq, mp, DL_UNBIND_REQ, DL_SYSERR, ENOMEM);
633 return;
634 }
635
636 err = softmac_output(slp, mp, DL_UNBIND_REQ, DL_OK_ACK, &ackmp);
637 if (ackmp != NULL) {
638 freemsg(mp1);
639 } else {
640 /*
641 * The driver does not ack timely.
642 */
643 ASSERT(err == ENOMSG);
644 ackmp = mp1;
645 }
646 if (err != 0) {
647 dlerrorack(sup->su_wq, ackmp, DL_UNBIND_REQ, DL_SYSERR, err);
648 return;
649 }
650
651 sup->su_bound = B_FALSE;
652
653 mutex_enter(&softmac->smac_active_mutex);
654 if (sup->su_active) {
655 ASSERT(!softmac->smac_active);
656 softmac->smac_nactive--;
657 sup->su_active = B_FALSE;
658 }
659 mutex_exit(&softmac->smac_active_mutex);
660
661 done:
662 qreply(sup->su_wq, ackmp);
663 }
664
665 /*
666 * Process the non-data mblk.
667 */
668 static void
softmac_wput_single_nondata(softmac_upper_t * sup,mblk_t * mp)669 softmac_wput_single_nondata(softmac_upper_t *sup, mblk_t *mp)
670 {
671 softmac_t *softmac = sup->su_softmac;
672 softmac_lower_t *slp = sup->su_slp;
673 unsigned char dbtype;
674 t_uscalar_t prim;
675
676 dbtype = DB_TYPE(mp);
677 sup->su_is_arp = 0;
678 switch (dbtype) {
679 case M_CTL:
680 sup->su_is_arp = 1;
681 /* FALLTHROUGH */
682 case M_IOCTL: {
683 uint32_t expected_mode;
684
685 if (((struct iocblk *)(mp->b_rptr))->ioc_cmd != SIOCSLIFNAME)
686 break;
687
688 /*
689 * Nak the M_IOCTL based on the STREAMS specification.
690 */
691 if (dbtype == M_IOCTL)
692 miocnak(sup->su_wq, mp, 0, EINVAL);
693 else
694 freemsg(mp);
695
696 /*
697 * This stream is either IP or ARP. See whether
698 * we need to setup a dedicated-lower-stream for it.
699 */
700 mutex_enter(&softmac->smac_fp_mutex);
701
702 expected_mode = DATAPATH_MODE(softmac);
703 if (expected_mode == SOFTMAC_SLOWPATH)
704 sup->su_mode = SOFTMAC_SLOWPATH;
705 list_insert_head(&softmac->smac_sup_list, sup);
706 mutex_exit(&softmac->smac_fp_mutex);
707
708 /*
709 * Setup the fast-path dedicated lower stream if fast-path
710 * is expected. Note that no lock is held here, and if
711 * smac_expected_mode is changed from SOFTMAC_FASTPATH to
712 * SOFTMAC_SLOWPATH, the DL_NOTE_REPLUMB message used for
713 * data-path switching would already be queued and will
714 * be processed by softmac_wput_single_nondata() later.
715 */
716 if (expected_mode == SOFTMAC_FASTPATH)
717 (void) softmac_fastpath_setup(sup);
718 return;
719 }
720 case M_PROTO:
721 case M_PCPROTO:
722 if (MBLKL(mp) < sizeof (t_uscalar_t)) {
723 freemsg(mp);
724 return;
725 }
726 prim = ((union DL_primitives *)mp->b_rptr)->dl_primitive;
727 switch (prim) {
728 case DL_NOTIFY_IND:
729 if (MBLKL(mp) < sizeof (dl_notify_ind_t) ||
730 ((dl_notify_ind_t *)mp->b_rptr)->dl_notification !=
731 DL_NOTE_REPLUMB) {
732 freemsg(mp);
733 return;
734 }
735 /*
736 * This DL_NOTE_REPLUMB message is initiated
737 * and queued by the softmac itself, when the
738 * sup is trying to switching its datapath mode
739 * between SOFTMAC_SLOWPATH and SOFTMAC_FASTPATH.
740 * Send this message upstream.
741 */
742 qreply(sup->su_wq, mp);
743 return;
744 case DL_NOTIFY_CONF:
745 if (MBLKL(mp) < sizeof (dl_notify_conf_t) ||
746 ((dl_notify_conf_t *)mp->b_rptr)->dl_notification !=
747 DL_NOTE_REPLUMB_DONE) {
748 freemsg(mp);
749 return;
750 }
751 /*
752 * This is an indication from IP/ARP that the
753 * fastpath->slowpath switch is done.
754 */
755 freemsg(mp);
756 softmac_datapath_switch_done(sup);
757 return;
758 }
759 break;
760 }
761
762 /*
763 * No need to hold lock to check su_mode, since su_mode updating only
764 * operation is is serialized by softmac_wput_nondata_task().
765 */
766 if (sup->su_mode != SOFTMAC_FASTPATH) {
767 dld_wput(sup->su_wq, mp);
768 return;
769 }
770
771 /*
772 * Fastpath non-data message processing. Most of non-data messages
773 * can be directly passed down to the dedicated-lower-stream, aside
774 * from the following M_PROTO/M_PCPROTO messages.
775 */
776 switch (dbtype) {
777 case M_PROTO:
778 case M_PCPROTO:
779 switch (prim) {
780 case DL_BIND_REQ:
781 softmac_bind_req(sup, mp);
782 break;
783 case DL_UNBIND_REQ:
784 softmac_unbind_req(sup, mp);
785 break;
786 case DL_CAPABILITY_REQ:
787 softmac_capability_req(sup, mp);
788 break;
789 default:
790 putnext(slp->sl_wq, mp);
791 break;
792 }
793 break;
794 default:
795 putnext(slp->sl_wq, mp);
796 break;
797 }
798 }
799
800 /*
801 * The worker thread which processes non-data messages. Note we only process
802 * one message at one time in order to be able to "flush" the queued message
803 * and serialize the processing.
804 */
805 static void
softmac_wput_nondata_task(void * arg)806 softmac_wput_nondata_task(void *arg)
807 {
808 softmac_upper_t *sup = arg;
809 mblk_t *mp;
810
811 mutex_enter(&sup->su_disp_mutex);
812
813 while (sup->su_pending_head != NULL) {
814 if (sup->su_closing)
815 break;
816
817 SOFTMAC_DQ_PENDING(sup, &mp);
818 mutex_exit(&sup->su_disp_mutex);
819 softmac_wput_single_nondata(sup, mp);
820 mutex_enter(&sup->su_disp_mutex);
821 }
822
823 /*
824 * If the stream is closing, flush all queued messages and inform
825 * the stream to be closed.
826 */
827 freemsgchain(sup->su_pending_head);
828 sup->su_pending_head = sup->su_pending_tail = NULL;
829 sup->su_dlpi_pending = B_FALSE;
830 cv_signal(&sup->su_disp_cv);
831 mutex_exit(&sup->su_disp_mutex);
832 }
833
834 /*
835 * Kernel thread to handle taskq dispatch failures in softmac_wput_nondata().
836 * This thread is started when the softmac module is first loaded.
837 */
838 static void
softmac_taskq_dispatch(void)839 softmac_taskq_dispatch(void)
840 {
841 callb_cpr_t cprinfo;
842 softmac_upper_t *sup;
843
844 CALLB_CPR_INIT(&cprinfo, &softmac_taskq_lock, callb_generic_cpr,
845 "softmac_taskq_dispatch");
846 mutex_enter(&softmac_taskq_lock);
847
848 while (!softmac_taskq_quit) {
849 sup = list_head(&softmac_taskq_list);
850 while (sup != NULL) {
851 list_remove(&softmac_taskq_list, sup);
852 sup->su_taskq_scheduled = B_FALSE;
853 mutex_exit(&softmac_taskq_lock);
854 VERIFY(taskq_dispatch(system_taskq,
855 softmac_wput_nondata_task, sup, TQ_SLEEP) != NULL);
856 mutex_enter(&softmac_taskq_lock);
857 sup = list_head(&softmac_taskq_list);
858 }
859
860 CALLB_CPR_SAFE_BEGIN(&cprinfo);
861 cv_wait(&softmac_taskq_cv, &softmac_taskq_lock);
862 CALLB_CPR_SAFE_END(&cprinfo, &softmac_taskq_lock);
863 }
864
865 softmac_taskq_done = B_TRUE;
866 cv_signal(&softmac_taskq_cv);
867 CALLB_CPR_EXIT(&cprinfo);
868 thread_exit();
869 }
870
871 void
softmac_wput_nondata(softmac_upper_t * sup,mblk_t * mp)872 softmac_wput_nondata(softmac_upper_t *sup, mblk_t *mp)
873 {
874 /*
875 * The processing of the message might block. Enqueue the
876 * message for later processing.
877 */
878 mutex_enter(&sup->su_disp_mutex);
879
880 if (sup->su_closing) {
881 mutex_exit(&sup->su_disp_mutex);
882 freemsg(mp);
883 return;
884 }
885
886 SOFTMAC_EQ_PENDING(sup, mp);
887
888 if (sup->su_dlpi_pending) {
889 mutex_exit(&sup->su_disp_mutex);
890 return;
891 }
892 sup->su_dlpi_pending = B_TRUE;
893 mutex_exit(&sup->su_disp_mutex);
894
895 if (taskq_dispatch(system_taskq, softmac_wput_nondata_task,
896 sup, TQ_NOSLEEP) != NULL) {
897 return;
898 }
899
900 mutex_enter(&softmac_taskq_lock);
901 if (!sup->su_taskq_scheduled) {
902 list_insert_tail(&softmac_taskq_list, sup);
903 cv_signal(&softmac_taskq_cv);
904 }
905 sup->su_taskq_scheduled = B_TRUE;
906 mutex_exit(&softmac_taskq_lock);
907 }
908
909 /*
910 * Setup the dedicated-lower-stream (fast-path) for the IP/ARP upperstream.
911 */
912 static int
softmac_fastpath_setup(softmac_upper_t * sup)913 softmac_fastpath_setup(softmac_upper_t *sup)
914 {
915 softmac_t *softmac = sup->su_softmac;
916 softmac_lower_t *slp;
917 int err;
918
919 err = softmac_lower_setup(softmac, sup, &slp);
920
921 mutex_enter(&sup->su_mutex);
922 /*
923 * Wait for all data messages to be processed so that we can change
924 * the su_mode.
925 */
926 while (sup->su_tx_inprocess != 0)
927 cv_wait(&sup->su_cv, &sup->su_mutex);
928
929 ASSERT(sup->su_mode != SOFTMAC_FASTPATH);
930 ASSERT(sup->su_slp == NULL);
931 if (err != 0) {
932 sup->su_mode = SOFTMAC_SLOWPATH;
933 } else {
934 sup->su_slp = slp;
935 sup->su_mode = SOFTMAC_FASTPATH;
936 }
937 mutex_exit(&sup->su_mutex);
938 return (err);
939 }
940
941 /*
942 * Tear down the dedicated-lower-stream (fast-path) for the IP/ARP upperstream.
943 */
944 static void
softmac_fastpath_tear(softmac_upper_t * sup)945 softmac_fastpath_tear(softmac_upper_t *sup)
946 {
947 mutex_enter(&sup->su_mutex);
948 /*
949 * Wait for all data messages in the dedicated-lower-stream
950 * to be processed.
951 */
952 while (sup->su_tx_inprocess != 0)
953 cv_wait(&sup->su_cv, &sup->su_mutex);
954
955 /*
956 * Note that this function is called either when the stream is closed,
957 * or the stream is unbound (fastpath-slowpath-switch). Therefore,
958 * No need to call the tx_notify callback.
959 */
960 sup->su_tx_notify_func = NULL;
961 sup->su_tx_notify_arg = NULL;
962 if (sup->su_tx_busy) {
963 ASSERT(sup->su_tx_flow_mp == NULL);
964 VERIFY((sup->su_tx_flow_mp = getq(sup->su_wq)) != NULL);
965 sup->su_tx_busy = B_FALSE;
966 }
967
968 sup->su_mode = SOFTMAC_SLOWPATH;
969
970 /*
971 * Destroy the dedicated-lower-stream. Note that slp is destroyed
972 * when lh is closed.
973 */
974 (void) ldi_close(sup->su_slp->sl_lh, FREAD|FWRITE, kcred);
975 sup->su_slp = NULL;
976 mutex_exit(&sup->su_mutex);
977 }
978
979 void
softmac_wput_data(softmac_upper_t * sup,mblk_t * mp)980 softmac_wput_data(softmac_upper_t *sup, mblk_t *mp)
981 {
982 /*
983 * No lock is required to access the su_mode field since the data
984 * traffic is quiesce by IP when the data-path mode is in the
985 * process of switching.
986 */
987 if (sup->su_mode != SOFTMAC_FASTPATH)
988 dld_wput(sup->su_wq, mp);
989 else
990 (void) softmac_fastpath_wput_data(sup, mp, NULL, 0);
991 }
992
993 /*ARGSUSED*/
994 static mac_tx_cookie_t
softmac_fastpath_wput_data(softmac_upper_t * sup,mblk_t * mp,uintptr_t f_hint,uint16_t flag)995 softmac_fastpath_wput_data(softmac_upper_t *sup, mblk_t *mp, uintptr_t f_hint,
996 uint16_t flag)
997 {
998 queue_t *wq = sup->su_slp->sl_wq;
999
1000 /*
1001 * This function is called from IP, only the MAC_DROP_ON_NO_DESC
1002 * flag can be specified.
1003 */
1004 ASSERT((flag & ~MAC_DROP_ON_NO_DESC) == 0);
1005 ASSERT(mp->b_next == NULL);
1006
1007 /*
1008 * Check wether the dedicated-lower-stream is able to handle more
1009 * messages, and enable the flow-control if it is not.
1010 *
1011 * Note that in order not to introduce any packet reordering, we
1012 * always send the message down to the dedicated-lower-stream:
1013 *
1014 * If the flow-control is already enabled, but we still get
1015 * the messages from the upper-stream, it means that the upper
1016 * stream does not respect STREAMS flow-control (e.g., TCP). Simply
1017 * pass the message down to the lower-stream in that case.
1018 */
1019 if (SOFTMAC_CANPUTNEXT(wq)) {
1020 putnext(wq, mp);
1021 return (NULL);
1022 }
1023
1024 if (sup->su_tx_busy) {
1025 if ((flag & MAC_DROP_ON_NO_DESC) != 0)
1026 freemsg(mp);
1027 else
1028 putnext(wq, mp);
1029 return ((mac_tx_cookie_t)sup);
1030 }
1031
1032 mutex_enter(&sup->su_mutex);
1033 if (!sup->su_tx_busy) {
1034 /*
1035 * If DLD_CAPAB_DIRECT is enabled, the notify callback will be
1036 * called when the flow control can be disabled. Otherwise,
1037 * put the tx_flow_mp into the wq to make use of the old
1038 * streams flow control.
1039 */
1040 ASSERT(sup->su_tx_flow_mp != NULL);
1041 (void) putq(sup->su_wq, sup->su_tx_flow_mp);
1042 sup->su_tx_flow_mp = NULL;
1043 sup->su_tx_busy = B_TRUE;
1044 qenable(wq);
1045 }
1046 mutex_exit(&sup->su_mutex);
1047
1048 if ((flag & MAC_DROP_ON_NO_DESC) != 0)
1049 freemsg(mp);
1050 else
1051 putnext(wq, mp);
1052 return ((mac_tx_cookie_t)sup);
1053 }
1054
1055 boolean_t
softmac_active_set(void * arg)1056 softmac_active_set(void *arg)
1057 {
1058 softmac_t *softmac = arg;
1059
1060 mutex_enter(&softmac->smac_active_mutex);
1061 if (softmac->smac_nactive != 0) {
1062 mutex_exit(&softmac->smac_active_mutex);
1063 return (B_FALSE);
1064 }
1065 softmac->smac_active = B_TRUE;
1066 mutex_exit(&softmac->smac_active_mutex);
1067 return (B_TRUE);
1068 }
1069
1070 void
softmac_active_clear(void * arg)1071 softmac_active_clear(void *arg)
1072 {
1073 softmac_t *softmac = arg;
1074
1075 mutex_enter(&softmac->smac_active_mutex);
1076 ASSERT(softmac->smac_active && (softmac->smac_nactive == 0));
1077 softmac->smac_active = B_FALSE;
1078 mutex_exit(&softmac->smac_active_mutex);
1079 }
1080
1081 /*
1082 * Disable/reenable fastpath on given softmac. This request could come from a
1083 * MAC client or directly from administrators.
1084 */
1085 int
softmac_datapath_switch(softmac_t * softmac,boolean_t disable,boolean_t admin)1086 softmac_datapath_switch(softmac_t *softmac, boolean_t disable, boolean_t admin)
1087 {
1088 softmac_upper_t *sup;
1089 mblk_t *head = NULL, *tail = NULL, *mp;
1090 list_t reqlist;
1091 softmac_switch_req_t *req;
1092 uint32_t current_mode, expected_mode;
1093 int err = 0;
1094
1095 mutex_enter(&softmac->smac_fp_mutex);
1096
1097 current_mode = DATAPATH_MODE(softmac);
1098 if (admin) {
1099 if (softmac->smac_fastpath_admin_disabled == disable) {
1100 mutex_exit(&softmac->smac_fp_mutex);
1101 return (0);
1102 }
1103 softmac->smac_fastpath_admin_disabled = disable;
1104 } else if (disable) {
1105 softmac->smac_fp_disable_clients++;
1106 } else {
1107 ASSERT(softmac->smac_fp_disable_clients != 0);
1108 softmac->smac_fp_disable_clients--;
1109 }
1110
1111 expected_mode = DATAPATH_MODE(softmac);
1112 if (current_mode == expected_mode) {
1113 mutex_exit(&softmac->smac_fp_mutex);
1114 return (0);
1115 }
1116
1117 /*
1118 * The expected mode is different from whatever datapath mode
1119 * this softmac is expected from last request, enqueue the data-path
1120 * switch request.
1121 */
1122 list_create(&reqlist, sizeof (softmac_switch_req_t),
1123 offsetof(softmac_switch_req_t, ssq_req_list_node));
1124
1125 /*
1126 * Allocate all DL_NOTIFY_IND messages and request structures that
1127 * are required to switch each IP/ARP stream to the expected mode.
1128 */
1129 for (sup = list_head(&softmac->smac_sup_list); sup != NULL;
1130 sup = list_next(&softmac->smac_sup_list, sup)) {
1131 dl_notify_ind_t *dlip;
1132
1133 req = kmem_alloc(sizeof (softmac_switch_req_t), KM_NOSLEEP);
1134 if (req == NULL)
1135 break;
1136
1137 req->ssq_expected_mode = expected_mode;
1138 if (sup->su_is_arp) {
1139 list_insert_tail(&reqlist, req);
1140 continue;
1141 }
1142 /*
1143 * Allocate the DL_NOTE_REPLUMB message.
1144 */
1145 if ((mp = allocb(sizeof (dl_notify_ind_t), BPRI_LO)) == NULL) {
1146 kmem_free(req, sizeof (softmac_switch_req_t));
1147 break;
1148 }
1149
1150 list_insert_tail(&reqlist, req);
1151
1152 mp->b_wptr = mp->b_rptr + sizeof (dl_notify_ind_t);
1153 mp->b_datap->db_type = M_PROTO;
1154 bzero(mp->b_rptr, sizeof (dl_notify_ind_t));
1155 dlip = (dl_notify_ind_t *)mp->b_rptr;
1156 dlip->dl_primitive = DL_NOTIFY_IND;
1157 dlip->dl_notification = DL_NOTE_REPLUMB;
1158 if (head == NULL) {
1159 head = tail = mp;
1160 } else {
1161 tail->b_next = mp;
1162 tail = mp;
1163 }
1164 }
1165
1166 /*
1167 * Note that it is fine if the expected data-path mode is fast-path
1168 * and some of streams fails to switch. Only return failure if we
1169 * are expected to switch to the slow-path.
1170 */
1171 if (sup != NULL && expected_mode == SOFTMAC_SLOWPATH) {
1172 err = ENOMEM;
1173 goto fail;
1174 }
1175
1176 /*
1177 * Start switching for each IP/ARP stream. The switching operation
1178 * will eventually succeed and there is no need to wait for it
1179 * to finish.
1180 */
1181 for (sup = list_head(&softmac->smac_sup_list); sup != NULL;
1182 sup = list_next(&softmac->smac_sup_list, sup)) {
1183 if (!sup->su_is_arp) {
1184 mp = head->b_next;
1185 head->b_next = NULL;
1186 softmac_wput_nondata(sup, head);
1187 head = mp;
1188 }
1189 /*
1190 * Add the switch request to the requests list of the stream.
1191 */
1192 req = list_head(&reqlist);
1193 ASSERT(req != NULL);
1194 list_remove(&reqlist, req);
1195 list_insert_tail(&sup->su_req_list, req);
1196 }
1197
1198 mutex_exit(&softmac->smac_fp_mutex);
1199 ASSERT(list_is_empty(&reqlist));
1200 list_destroy(&reqlist);
1201 return (0);
1202 fail:
1203 if (admin) {
1204 softmac->smac_fastpath_admin_disabled = !disable;
1205 } else if (disable) {
1206 softmac->smac_fp_disable_clients--;
1207 } else {
1208 softmac->smac_fp_disable_clients++;
1209 }
1210
1211 mutex_exit(&softmac->smac_fp_mutex);
1212 while ((req = list_head(&reqlist)) != NULL) {
1213 list_remove(&reqlist, req);
1214 kmem_free(req, sizeof (softmac_switch_req_t));
1215 }
1216 freemsgchain(head);
1217 list_destroy(&reqlist);
1218 return (err);
1219 }
1220
1221 int
softmac_fastpath_disable(void * arg)1222 softmac_fastpath_disable(void *arg)
1223 {
1224 return (softmac_datapath_switch((softmac_t *)arg, B_TRUE, B_FALSE));
1225 }
1226
1227 void
softmac_fastpath_enable(void * arg)1228 softmac_fastpath_enable(void *arg)
1229 {
1230 VERIFY(softmac_datapath_switch((softmac_t *)arg, B_FALSE,
1231 B_FALSE) == 0);
1232 }
1233
1234 void
softmac_upperstream_close(softmac_upper_t * sup)1235 softmac_upperstream_close(softmac_upper_t *sup)
1236 {
1237 softmac_t *softmac = sup->su_softmac;
1238 softmac_switch_req_t *req;
1239
1240 mutex_enter(&softmac->smac_fp_mutex);
1241
1242 if (sup->su_mode == SOFTMAC_FASTPATH)
1243 softmac_fastpath_tear(sup);
1244
1245 if (sup->su_mode != SOFTMAC_UNKNOWN) {
1246 list_remove(&softmac->smac_sup_list, sup);
1247 sup->su_mode = SOFTMAC_UNKNOWN;
1248 }
1249
1250 /*
1251 * Cleanup all the switch requests queueed on this stream.
1252 */
1253 while ((req = list_head(&sup->su_req_list)) != NULL) {
1254 list_remove(&sup->su_req_list, req);
1255 kmem_free(req, sizeof (softmac_switch_req_t));
1256 }
1257 mutex_exit(&softmac->smac_fp_mutex);
1258 }
1259
1260 /*
1261 * Handle the DL_NOTE_REPLUMB_DONE indication from IP/ARP. Change the upper
1262 * stream from the fastpath mode to the slowpath mode.
1263 */
1264 static void
softmac_datapath_switch_done(softmac_upper_t * sup)1265 softmac_datapath_switch_done(softmac_upper_t *sup)
1266 {
1267 softmac_t *softmac = sup->su_softmac;
1268 softmac_switch_req_t *req;
1269 uint32_t expected_mode;
1270
1271 mutex_enter(&softmac->smac_fp_mutex);
1272 req = list_head(&sup->su_req_list);
1273 list_remove(&sup->su_req_list, req);
1274 expected_mode = req->ssq_expected_mode;
1275 kmem_free(req, sizeof (softmac_switch_req_t));
1276
1277 if (expected_mode == sup->su_mode) {
1278 mutex_exit(&softmac->smac_fp_mutex);
1279 return;
1280 }
1281
1282 ASSERT(!sup->su_bound);
1283 mutex_exit(&softmac->smac_fp_mutex);
1284
1285 /*
1286 * It is fine if the expected mode is fast-path and we fail
1287 * to enable fastpath on this stream.
1288 */
1289 if (expected_mode == SOFTMAC_SLOWPATH)
1290 softmac_fastpath_tear(sup);
1291 else
1292 (void) softmac_fastpath_setup(sup);
1293 }
1294