1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 /*
27 * Copyright 2019, Joyent, Inc.
28 * Copyright 2022 Garrett D'Amore
29 * Copyright 2025 Oxide Computer Company
30 */
31
32 /*
33 * Softmac data-path switching:
34 *
35 * - Fast-path model
36 *
37 * When the softmac fast-path is used, a dedicated lower-stream
38 * will be opened over the legacy device for each IP/ARP (upper-)stream
39 * over the softMAC, and all DLPI messages (including control messages
40 * and data messages) will be exchanged between the upper-stream and
41 * the corresponding lower-stream directly. Therefore, the data
42 * demultiplexing, filtering and classification processing will be done
43 * by the lower-stream, and the GLDv3 DLS/MAC layer processing will be
44 * no longer needed.
45 *
46 * - Slow-path model
47 *
48 * Some GLDv3 features requires the GLDv3 DLS/MAC layer processing to
49 * not be bypassed to assure its function correctness. For example,
50 * softmac fast-path must be disabled to support GLDv3 VNIC functionality.
51 * In this case, a shared lower-stream will be opened over the legacy
52 * device, which is responsible for implementing the GLDv3 callbacks
53 * and passing RAW data messages between the legacy devices and the GLDv3
54 * framework.
55 *
56 * By default, the softmac fast-path mode will be used to assure the
57 * performance; MAC clients will be able to request to disable the softmac
58 * fast-path mode to support certain features, and if that succeeds,
59 * the system will fallback to the slow-path softmac data-path model.
60 *
61 *
62 * The details of the softmac data fast-path model is stated as below
63 *
64 * 1. When a stream is opened on a softMAC, the softmac module will takes
65 * over the DLPI processing on this stream;
66 *
67 * 2. For IP/ARP streams over a softMAC, softmac data fast-path will be
68 * used by default, unless fast-path is disabled by any MAC client
69 * explicitly. The softmac module first identifies an IP/ARP stream
70 * by seeing whether there is a SIOCSLIFNAME ioctl sent from upstream,
71 * if there is one, this stream is either an IP or an ARP stream
72 * and will use fast-path potentially;
73 *
74 * 3. When the softmac fast-path is used, an dedicated lower-stream will
75 * be setup for each IP/ARP stream (1-1 mapping). From that point on,
76 * all control and data messages will be exchanged between the IP/ARP
77 * upper-stream and the legacy device through this dedicated
78 * lower-stream. As a result, the DLS/MAC layer processing in GLDv3
79 * will be skipped, and this greatly improves the performance;
80 *
81 * 4. When the softmac data fast-path is disabled by a MAC client (e.g.,
82 * by a VNIC), all the IP/ARP upper streams will try to switch from
83 * the fast-path to the slow-path. The dedicated lower-stream will be
84 * destroyed, and all the control and data-messages will go through the
85 * existing GLDv3 code path and (in the end) the shared lower-stream;
86 *
87 * 5. On the other hand, when the last MAC client cancels its fast-path
88 * disable request, all the IP/ARP streams will try to switch back to
89 * the fast-path mode;
90 *
91 * Step 5 and 6 both rely on the data-path mode switching process
92 * described below:
93 *
94 * 1) To switch the softmac data-path mode (between fast-path and slow-path),
95 * softmac will first send a DL_NOTE_REPLUMB DL_NOTIFY_IND message
96 * upstream over each IP/ARP streams that needs data-path mode switching;
97 *
98 * 2) When IP receives this DL_NOTE_REPLUMB message, it will bring down
99 * all the IP interfaces on the corresponding ill (IP Lower level
100 * structure), and bring up those interfaces over again; this will in
101 * turn cause the ARP to "replumb" the interface.
102 *
103 * During the replumb process, both IP and ARP will send downstream the
104 * necessary DL_DISABMULTI_REQ and DL_UNBIND_REQ messages and cleanup
105 * the old state of the underlying softMAC, following with the necessary
106 * DL_BIND_REQ and DL_ENABMULTI_REQ messages to setup the new state.
107 * Between the cleanup and re-setup process, IP/ARP will also send down
108 * a DL_NOTE_REPLUMB_DONE DL_NOTIFY_CONF messages to the softMAC to
109 * indicate the *switching point*;
110 *
111 * 3) When softmac receives the DL_NOTE_REPLUMB_DONE message, it either
112 * creates or destroys the dedicated lower-stream (depending on which
113 * data-path mode the softMAC switches to), and change the softmac
114 * data-path mode. From then on, softmac will process all the succeeding
115 * control messages (including the DL_BIND_REQ and DL_ENABMULTI_REQ
116 * messages) and data messages based on new data-path mode.
117 */
118
119 #include <sys/types.h>
120 #include <sys/disp.h>
121 #include <sys/callb.h>
122 #include <sys/sysmacros.h>
123 #include <sys/file.h>
124 #include <sys/vlan.h>
125 #include <sys/dld.h>
126 #include <sys/sockio.h>
127 #include <sys/softmac_impl.h>
128 #include <net/if.h>
129
130 static kmutex_t softmac_taskq_lock;
131 static kthread_t *softmac_taskq_thread;
132 static kcondvar_t softmac_taskq_cv;
133 static list_t softmac_taskq_list; /* List of softmac_upper_t */
134 boolean_t softmac_taskq_quit;
135 boolean_t softmac_taskq_done;
136
137 static void softmac_taskq_dispatch();
138 static int softmac_fastpath_setup(softmac_upper_t *);
139 static mac_tx_cookie_t softmac_fastpath_wput_data(softmac_upper_t *, mblk_t *,
140 uintptr_t, uint16_t);
141 static void softmac_datapath_switch_done(softmac_upper_t *);
142
143 void
softmac_fp_init()144 softmac_fp_init()
145 {
146 mutex_init(&softmac_taskq_lock, NULL, MUTEX_DRIVER, NULL);
147 cv_init(&softmac_taskq_cv, NULL, CV_DRIVER, NULL);
148
149 softmac_taskq_quit = B_FALSE;
150 softmac_taskq_done = B_FALSE;
151 list_create(&softmac_taskq_list, sizeof (softmac_upper_t),
152 offsetof(softmac_upper_t, su_taskq_list_node));
153 softmac_taskq_thread = thread_create(NULL, 0, softmac_taskq_dispatch,
154 NULL, 0, &p0, TS_RUN, minclsyspri);
155 }
156
157 void
softmac_fp_fini()158 softmac_fp_fini()
159 {
160 /*
161 * Request the softmac_taskq thread to quit and wait for it to be done.
162 */
163 mutex_enter(&softmac_taskq_lock);
164 softmac_taskq_quit = B_TRUE;
165 cv_signal(&softmac_taskq_cv);
166 while (!softmac_taskq_done)
167 cv_wait(&softmac_taskq_cv, &softmac_taskq_lock);
168 mutex_exit(&softmac_taskq_lock);
169 thread_join(softmac_taskq_thread->t_did);
170 list_destroy(&softmac_taskq_list);
171
172 mutex_destroy(&softmac_taskq_lock);
173 cv_destroy(&softmac_taskq_cv);
174 }
175
176 static boolean_t
check_ip_above(queue_t * q)177 check_ip_above(queue_t *q)
178 {
179 queue_t *next_q;
180 boolean_t ret = B_TRUE;
181
182 claimstr(q);
183 next_q = q->q_next;
184 if (strcmp(next_q->q_qinfo->qi_minfo->mi_idname, "ip") != 0)
185 ret = B_FALSE;
186 releasestr(q);
187 return (ret);
188 }
189
190 /* ARGSUSED */
191 static int
softmac_capab_perim(softmac_upper_t * sup,void * data,uint_t flags)192 softmac_capab_perim(softmac_upper_t *sup, void *data, uint_t flags)
193 {
194 switch (flags) {
195 case DLD_ENABLE:
196 mutex_enter(&sup->su_mutex);
197 break;
198 case DLD_DISABLE:
199 mutex_exit(&sup->su_mutex);
200 break;
201 case DLD_QUERY:
202 return (MUTEX_HELD(&sup->su_mutex));
203 }
204 return (0);
205 }
206
207 static mac_tx_notify_handle_t
softmac_client_tx_notify(softmac_upper_t * sup,mac_tx_notify_t func,void * arg)208 softmac_client_tx_notify(softmac_upper_t *sup, mac_tx_notify_t func, void *arg)
209 {
210 ASSERT(MUTEX_HELD(&sup->su_mutex));
211
212 if (func != NULL) {
213 sup->su_tx_notify_func = func;
214 sup->su_tx_notify_arg = arg;
215 } else {
216 /*
217 * Wait for all tx_notify_func call to be done.
218 */
219 while (sup->su_tx_inprocess != 0)
220 cv_wait(&sup->su_cv, &sup->su_mutex);
221
222 sup->su_tx_notify_func = NULL;
223 sup->su_tx_notify_arg = NULL;
224 }
225 return ((mac_tx_notify_handle_t)sup);
226 }
227
228 static boolean_t
softmac_tx_is_flow_blocked(softmac_upper_t * sup,mac_tx_cookie_t cookie)229 softmac_tx_is_flow_blocked(softmac_upper_t *sup, mac_tx_cookie_t cookie)
230 {
231 ASSERT(cookie == (mac_tx_cookie_t)sup);
232 return (sup->su_tx_busy);
233 }
234
235 static int
softmac_capab_direct(softmac_upper_t * sup,void * data,uint_t flags)236 softmac_capab_direct(softmac_upper_t *sup, void *data, uint_t flags)
237 {
238 dld_capab_direct_t *direct = data;
239 softmac_lower_t *slp = sup->su_slp;
240
241 ASSERT(MUTEX_HELD(&sup->su_mutex));
242
243 ASSERT(sup->su_mode == SOFTMAC_FASTPATH);
244
245 switch (flags) {
246 case DLD_ENABLE:
247 if (sup->su_direct)
248 return (0);
249
250 sup->su_direct_rxinfo.slr_rx = (softmac_rx_t)direct->di_rx_cf;
251 sup->su_direct_rxinfo.slr_arg = direct->di_rx_ch;
252 slp->sl_rxinfo = &sup->su_direct_rxinfo;
253 direct->di_tx_df = (uintptr_t)softmac_fastpath_wput_data;
254 direct->di_tx_dh = sup;
255 direct->di_tx_fctl_df = (uintptr_t)softmac_tx_is_flow_blocked;
256 direct->di_tx_fctl_dh = sup;
257 direct->di_tx_cb_df = (uintptr_t)softmac_client_tx_notify;
258 direct->di_tx_cb_dh = sup;
259 sup->su_direct = B_TRUE;
260 return (0);
261
262 case DLD_DISABLE:
263 if (!sup->su_direct)
264 return (0);
265
266 slp->sl_rxinfo = &sup->su_rxinfo;
267 sup->su_direct = B_FALSE;
268 return (0);
269 }
270 return (ENOTSUP);
271 }
272
273 static int
softmac_dld_capab(softmac_upper_t * sup,uint_t type,void * data,uint_t flags)274 softmac_dld_capab(softmac_upper_t *sup, uint_t type, void *data, uint_t flags)
275 {
276 int err;
277
278 /*
279 * Don't enable direct callback capabilities unless the caller is
280 * the IP client. When a module is inserted in a stream (_I_INSERT)
281 * the stack initiates capability disable, but due to races, the
282 * module insertion may complete before the capability disable
283 * completes. So we limit the check to DLD_ENABLE case.
284 */
285 if ((flags == DLD_ENABLE && type != DLD_CAPAB_PERIM) &&
286 !check_ip_above(sup->su_rq)) {
287 return (ENOTSUP);
288 }
289
290 switch (type) {
291 case DLD_CAPAB_DIRECT:
292 err = softmac_capab_direct(sup, data, flags);
293 break;
294
295 case DLD_CAPAB_PERIM:
296 err = softmac_capab_perim(sup, data, flags);
297 break;
298
299 default:
300 err = ENOTSUP;
301 break;
302 }
303 return (err);
304 }
305
306 static void
softmac_capability_advertise(softmac_upper_t * sup,mblk_t * mp)307 softmac_capability_advertise(softmac_upper_t *sup, mblk_t *mp)
308 {
309 dl_capability_ack_t *dlap;
310 dl_capability_sub_t *dlsp;
311 t_uscalar_t subsize;
312 uint8_t *ptr;
313 queue_t *q = sup->su_wq;
314 mblk_t *mp1;
315 softmac_t *softmac = sup->su_softmac;
316 boolean_t dld_capable = B_FALSE;
317 boolean_t hcksum_capable = B_FALSE;
318 boolean_t zcopy_capable = B_FALSE;
319
320 ASSERT(sup->su_mode == SOFTMAC_FASTPATH);
321
322 /*
323 * Initially assume no capabilities.
324 */
325 subsize = 0;
326
327 /*
328 * Direct capability negotiation interface between IP and softmac
329 */
330 if (check_ip_above(sup->su_rq)) {
331 dld_capable = B_TRUE;
332 subsize += sizeof (dl_capability_sub_t) +
333 sizeof (dl_capab_dld_t);
334 }
335
336 /*
337 * Check if checksum offload is supported on this MAC.
338 */
339 if (softmac->smac_capab_flags & MAC_CAPAB_HCKSUM) {
340 hcksum_capable = B_TRUE;
341 subsize += sizeof (dl_capability_sub_t) +
342 sizeof (dl_capab_hcksum_t);
343 }
344
345 /*
346 * Check if zerocopy is supported on this interface.
347 */
348 if (!(softmac->smac_capab_flags & MAC_CAPAB_NO_ZCOPY)) {
349 zcopy_capable = B_TRUE;
350 subsize += sizeof (dl_capability_sub_t) +
351 sizeof (dl_capab_zerocopy_t);
352 }
353
354 /*
355 * If there are no capabilities to advertise or if we
356 * can't allocate a response, send a DL_ERROR_ACK.
357 */
358 if ((subsize == 0) || (mp1 = reallocb(mp,
359 sizeof (dl_capability_ack_t) + subsize, 0)) == NULL) {
360 dlerrorack(q, mp, DL_CAPABILITY_REQ, DL_NOTSUPPORTED, 0);
361 return;
362 }
363
364 mp = mp1;
365 DB_TYPE(mp) = M_PROTO;
366 mp->b_wptr = mp->b_rptr + sizeof (dl_capability_ack_t) + subsize;
367 bzero(mp->b_rptr, MBLKL(mp));
368 dlap = (dl_capability_ack_t *)mp->b_rptr;
369 dlap->dl_primitive = DL_CAPABILITY_ACK;
370 dlap->dl_sub_offset = sizeof (dl_capability_ack_t);
371 dlap->dl_sub_length = subsize;
372 ptr = (uint8_t *)&dlap[1];
373
374 /*
375 * IP polling interface.
376 */
377 if (dld_capable) {
378 dl_capab_dld_t dld;
379
380 dlsp = (dl_capability_sub_t *)ptr;
381 dlsp->dl_cap = DL_CAPAB_DLD;
382 dlsp->dl_length = sizeof (dl_capab_dld_t);
383 ptr += sizeof (dl_capability_sub_t);
384
385 bzero(&dld, sizeof (dl_capab_dld_t));
386 dld.dld_version = DLD_CURRENT_VERSION;
387 dld.dld_capab = (uintptr_t)softmac_dld_capab;
388 dld.dld_capab_handle = (uintptr_t)sup;
389
390 dlcapabsetqid(&(dld.dld_mid), sup->su_rq);
391 bcopy(&dld, ptr, sizeof (dl_capab_dld_t));
392 ptr += sizeof (dl_capab_dld_t);
393 }
394
395 /*
396 * TCP/IP checksum offload.
397 */
398 if (hcksum_capable) {
399 dl_capab_hcksum_t hcksum;
400
401 dlsp = (dl_capability_sub_t *)ptr;
402
403 dlsp->dl_cap = DL_CAPAB_HCKSUM;
404 dlsp->dl_length = sizeof (dl_capab_hcksum_t);
405 ptr += sizeof (dl_capability_sub_t);
406
407 bzero(&hcksum, sizeof (dl_capab_hcksum_t));
408 hcksum.hcksum_version = HCKSUM_VERSION_1;
409 hcksum.hcksum_txflags = softmac->smac_hcksum_txflags;
410 dlcapabsetqid(&(hcksum.hcksum_mid), sup->su_rq);
411 bcopy(&hcksum, ptr, sizeof (dl_capab_hcksum_t));
412 ptr += sizeof (dl_capab_hcksum_t);
413 }
414
415 /*
416 * Zero copy
417 */
418 if (zcopy_capable) {
419 dl_capab_zerocopy_t zcopy;
420
421 dlsp = (dl_capability_sub_t *)ptr;
422
423 dlsp->dl_cap = DL_CAPAB_ZEROCOPY;
424 dlsp->dl_length = sizeof (dl_capab_zerocopy_t);
425 ptr += sizeof (dl_capability_sub_t);
426
427 bzero(&zcopy, sizeof (dl_capab_zerocopy_t));
428 zcopy.zerocopy_version = ZEROCOPY_VERSION_1;
429 zcopy.zerocopy_flags = DL_CAPAB_VMSAFE_MEM;
430 dlcapabsetqid(&(zcopy.zerocopy_mid), sup->su_rq);
431 bcopy(&zcopy, ptr, sizeof (dl_capab_zerocopy_t));
432 ptr += sizeof (dl_capab_zerocopy_t);
433 }
434
435 ASSERT(ptr == mp->b_rptr + sizeof (dl_capability_ack_t) + subsize);
436 qreply(q, mp);
437 }
438
439 static void
softmac_capability_req(softmac_upper_t * sup,mblk_t * mp)440 softmac_capability_req(softmac_upper_t *sup, mblk_t *mp)
441 {
442 dl_capability_req_t *dlp = (dl_capability_req_t *)mp->b_rptr;
443 dl_capability_sub_t *sp;
444 size_t size, len;
445 offset_t off, end;
446 t_uscalar_t dl_err;
447 queue_t *q = sup->su_wq;
448
449 ASSERT(sup->su_mode == SOFTMAC_FASTPATH);
450 if (MBLKL(mp) < sizeof (dl_capability_req_t)) {
451 dl_err = DL_BADPRIM;
452 goto failed;
453 }
454
455 if (!sup->su_bound) {
456 dl_err = DL_OUTSTATE;
457 goto failed;
458 }
459
460 /*
461 * This request is overloaded. If there are no requested capabilities
462 * then we just want to acknowledge with all the capabilities we
463 * support. Otherwise we enable the set of capabilities requested.
464 */
465 if (dlp->dl_sub_length == 0) {
466 softmac_capability_advertise(sup, mp);
467 return;
468 }
469
470 if (!MBLKIN(mp, dlp->dl_sub_offset, dlp->dl_sub_length)) {
471 dl_err = DL_BADPRIM;
472 goto failed;
473 }
474
475 dlp->dl_primitive = DL_CAPABILITY_ACK;
476
477 off = dlp->dl_sub_offset;
478 len = dlp->dl_sub_length;
479
480 /*
481 * Walk the list of capabilities to be enabled.
482 */
483 for (end = off + len; off < end; ) {
484 sp = (dl_capability_sub_t *)(mp->b_rptr + off);
485 size = sizeof (dl_capability_sub_t) + sp->dl_length;
486
487 if (off + size > end ||
488 !IS_P2ALIGNED(off, sizeof (uint32_t))) {
489 dl_err = DL_BADPRIM;
490 goto failed;
491 }
492
493 switch (sp->dl_cap) {
494 /*
495 * TCP/IP checksum offload to hardware.
496 */
497 case DL_CAPAB_HCKSUM: {
498 dl_capab_hcksum_t *hcksump;
499 dl_capab_hcksum_t hcksum;
500
501 hcksump = (dl_capab_hcksum_t *)&sp[1];
502 /*
503 * Copy for alignment.
504 */
505 bcopy(hcksump, &hcksum, sizeof (dl_capab_hcksum_t));
506 dlcapabsetqid(&(hcksum.hcksum_mid), sup->su_rq);
507 bcopy(&hcksum, hcksump, sizeof (dl_capab_hcksum_t));
508 break;
509 }
510
511 default:
512 break;
513 }
514
515 off += size;
516 }
517 qreply(q, mp);
518 return;
519 failed:
520 dlerrorack(q, mp, DL_CAPABILITY_REQ, dl_err, 0);
521 }
522
523 static void
softmac_bind_req(softmac_upper_t * sup,mblk_t * mp)524 softmac_bind_req(softmac_upper_t *sup, mblk_t *mp)
525 {
526 softmac_lower_t *slp = sup->su_slp;
527 softmac_t *softmac = sup->su_softmac;
528 mblk_t *ackmp, *mp1;
529 int err;
530
531 if (MBLKL(mp) < DL_BIND_REQ_SIZE) {
532 freemsg(mp);
533 return;
534 }
535
536 /*
537 * Allocate ackmp incase the underlying driver does not ack timely.
538 */
539 if ((mp1 = allocb(sizeof (dl_error_ack_t), BPRI_HI)) == NULL) {
540 dlerrorack(sup->su_wq, mp, DL_BIND_REQ, DL_SYSERR, ENOMEM);
541 return;
542 }
543
544 err = softmac_output(slp, mp, DL_BIND_REQ, DL_BIND_ACK, &ackmp);
545 if (ackmp != NULL) {
546 freemsg(mp1);
547 } else {
548 /*
549 * The driver does not ack timely.
550 */
551 ASSERT(err == ENOMSG);
552 ackmp = mp1;
553 }
554 if (err != 0)
555 goto failed;
556
557 /*
558 * Enable capabilities the underlying driver claims to support.
559 */
560 if ((err = softmac_capab_enable(slp)) != 0)
561 goto failed;
562
563 /*
564 * Check whether this softmac is already marked as exclusively used,
565 * e.g., an aggregation is created over it. Fail the BIND_REQ if so.
566 */
567 mutex_enter(&softmac->smac_active_mutex);
568 if (softmac->smac_active) {
569 mutex_exit(&softmac->smac_active_mutex);
570 err = EBUSY;
571 goto failed;
572 }
573 softmac->smac_nactive++;
574 sup->su_active = B_TRUE;
575 mutex_exit(&softmac->smac_active_mutex);
576 sup->su_bound = B_TRUE;
577
578 qreply(sup->su_wq, ackmp);
579 return;
580 failed:
581 if (err != 0) {
582 dlerrorack(sup->su_wq, ackmp, DL_BIND_REQ, DL_SYSERR, err);
583 return;
584 }
585 }
586
587 static void
softmac_unbind_req(softmac_upper_t * sup,mblk_t * mp)588 softmac_unbind_req(softmac_upper_t *sup, mblk_t *mp)
589 {
590 softmac_lower_t *slp = sup->su_slp;
591 softmac_t *softmac = sup->su_softmac;
592 mblk_t *ackmp, *mp1;
593 int err;
594
595 if (MBLKL(mp) < DL_UNBIND_REQ_SIZE) {
596 freemsg(mp);
597 return;
598 }
599
600 if (!sup->su_bound) {
601 dlerrorack(sup->su_wq, mp, DL_UNBIND_REQ, DL_OUTSTATE, 0);
602 return;
603 }
604
605 /*
606 * Allocate ackmp incase the underlying driver does not ack timely.
607 */
608 if ((mp1 = allocb(sizeof (dl_error_ack_t), BPRI_HI)) == NULL) {
609 dlerrorack(sup->su_wq, mp, DL_UNBIND_REQ, DL_SYSERR, ENOMEM);
610 return;
611 }
612
613 err = softmac_output(slp, mp, DL_UNBIND_REQ, DL_OK_ACK, &ackmp);
614 if (ackmp != NULL) {
615 freemsg(mp1);
616 } else {
617 /*
618 * The driver does not ack timely.
619 */
620 ASSERT(err == ENOMSG);
621 ackmp = mp1;
622 }
623 if (err != 0) {
624 dlerrorack(sup->su_wq, ackmp, DL_UNBIND_REQ, DL_SYSERR, err);
625 return;
626 }
627
628 sup->su_bound = B_FALSE;
629
630 mutex_enter(&softmac->smac_active_mutex);
631 if (sup->su_active) {
632 ASSERT(!softmac->smac_active);
633 softmac->smac_nactive--;
634 sup->su_active = B_FALSE;
635 }
636 mutex_exit(&softmac->smac_active_mutex);
637
638 done:
639 qreply(sup->su_wq, ackmp);
640 }
641
642 /*
643 * Process the non-data mblk.
644 */
645 static void
softmac_wput_single_nondata(softmac_upper_t * sup,mblk_t * mp)646 softmac_wput_single_nondata(softmac_upper_t *sup, mblk_t *mp)
647 {
648 softmac_t *softmac = sup->su_softmac;
649 softmac_lower_t *slp = sup->su_slp;
650 unsigned char dbtype;
651 t_uscalar_t prim;
652
653 dbtype = DB_TYPE(mp);
654 sup->su_is_arp = 0;
655 switch (dbtype) {
656 case M_CTL:
657 sup->su_is_arp = 1;
658 /* FALLTHROUGH */
659 case M_IOCTL: {
660 uint32_t expected_mode;
661
662 if (((struct iocblk *)(mp->b_rptr))->ioc_cmd != SIOCSLIFNAME)
663 break;
664
665 /*
666 * Nak the M_IOCTL based on the STREAMS specification.
667 */
668 if (dbtype == M_IOCTL)
669 miocnak(sup->su_wq, mp, 0, EINVAL);
670 else
671 freemsg(mp);
672
673 /*
674 * This stream is either IP or ARP. See whether
675 * we need to setup a dedicated-lower-stream for it.
676 */
677 mutex_enter(&softmac->smac_fp_mutex);
678
679 expected_mode = DATAPATH_MODE(softmac);
680 if (expected_mode == SOFTMAC_SLOWPATH)
681 sup->su_mode = SOFTMAC_SLOWPATH;
682 list_insert_head(&softmac->smac_sup_list, sup);
683 mutex_exit(&softmac->smac_fp_mutex);
684
685 /*
686 * Setup the fast-path dedicated lower stream if fast-path
687 * is expected. Note that no lock is held here, and if
688 * smac_expected_mode is changed from SOFTMAC_FASTPATH to
689 * SOFTMAC_SLOWPATH, the DL_NOTE_REPLUMB message used for
690 * data-path switching would already be queued and will
691 * be processed by softmac_wput_single_nondata() later.
692 */
693 if (expected_mode == SOFTMAC_FASTPATH)
694 (void) softmac_fastpath_setup(sup);
695 return;
696 }
697 case M_PROTO:
698 case M_PCPROTO:
699 if (MBLKL(mp) < sizeof (t_uscalar_t)) {
700 freemsg(mp);
701 return;
702 }
703 prim = ((union DL_primitives *)mp->b_rptr)->dl_primitive;
704 switch (prim) {
705 case DL_NOTIFY_IND:
706 if (MBLKL(mp) < sizeof (dl_notify_ind_t) ||
707 ((dl_notify_ind_t *)mp->b_rptr)->dl_notification !=
708 DL_NOTE_REPLUMB) {
709 freemsg(mp);
710 return;
711 }
712 /*
713 * This DL_NOTE_REPLUMB message is initiated
714 * and queued by the softmac itself, when the
715 * sup is trying to switching its datapath mode
716 * between SOFTMAC_SLOWPATH and SOFTMAC_FASTPATH.
717 * Send this message upstream.
718 */
719 qreply(sup->su_wq, mp);
720 return;
721 case DL_NOTIFY_CONF:
722 if (MBLKL(mp) < sizeof (dl_notify_conf_t) ||
723 ((dl_notify_conf_t *)mp->b_rptr)->dl_notification !=
724 DL_NOTE_REPLUMB_DONE) {
725 freemsg(mp);
726 return;
727 }
728 /*
729 * This is an indication from IP/ARP that the
730 * fastpath->slowpath switch is done.
731 */
732 freemsg(mp);
733 softmac_datapath_switch_done(sup);
734 return;
735 }
736 break;
737 }
738
739 /*
740 * No need to hold lock to check su_mode, since su_mode updating only
741 * operation is is serialized by softmac_wput_nondata_task().
742 */
743 if (sup->su_mode != SOFTMAC_FASTPATH) {
744 (void) dld_wput(sup->su_wq, mp);
745 return;
746 }
747
748 /*
749 * Fastpath non-data message processing. Most of non-data messages
750 * can be directly passed down to the dedicated-lower-stream, aside
751 * from the following M_PROTO/M_PCPROTO messages.
752 */
753 switch (dbtype) {
754 case M_PROTO:
755 case M_PCPROTO:
756 switch (prim) {
757 case DL_BIND_REQ:
758 softmac_bind_req(sup, mp);
759 break;
760 case DL_UNBIND_REQ:
761 softmac_unbind_req(sup, mp);
762 break;
763 case DL_CAPABILITY_REQ:
764 softmac_capability_req(sup, mp);
765 break;
766 default:
767 putnext(slp->sl_wq, mp);
768 break;
769 }
770 break;
771 default:
772 putnext(slp->sl_wq, mp);
773 break;
774 }
775 }
776
777 /*
778 * The worker thread which processes non-data messages. Note we only process
779 * one message at one time in order to be able to "flush" the queued message
780 * and serialize the processing.
781 */
782 static void
softmac_wput_nondata_task(void * arg)783 softmac_wput_nondata_task(void *arg)
784 {
785 softmac_upper_t *sup = arg;
786 mblk_t *mp;
787
788 mutex_enter(&sup->su_disp_mutex);
789
790 while (sup->su_pending_head != NULL) {
791 if (sup->su_closing)
792 break;
793
794 SOFTMAC_DQ_PENDING(sup, &mp);
795 mutex_exit(&sup->su_disp_mutex);
796 softmac_wput_single_nondata(sup, mp);
797 mutex_enter(&sup->su_disp_mutex);
798 }
799
800 /*
801 * If the stream is closing, flush all queued messages and inform
802 * the stream to be closed.
803 */
804 freemsgchain(sup->su_pending_head);
805 sup->su_pending_head = sup->su_pending_tail = NULL;
806 sup->su_dlpi_pending = B_FALSE;
807 cv_signal(&sup->su_disp_cv);
808 mutex_exit(&sup->su_disp_mutex);
809 }
810
811 /*
812 * Kernel thread to handle taskq dispatch failures in softmac_wput_nondata().
813 * This thread is started when the softmac module is first loaded.
814 */
815 static void
softmac_taskq_dispatch(void)816 softmac_taskq_dispatch(void)
817 {
818 callb_cpr_t cprinfo;
819 softmac_upper_t *sup;
820
821 CALLB_CPR_INIT(&cprinfo, &softmac_taskq_lock, callb_generic_cpr,
822 "softmac_taskq_dispatch");
823 mutex_enter(&softmac_taskq_lock);
824
825 while (!softmac_taskq_quit) {
826 sup = list_head(&softmac_taskq_list);
827 while (sup != NULL) {
828 list_remove(&softmac_taskq_list, sup);
829 sup->su_taskq_scheduled = B_FALSE;
830 mutex_exit(&softmac_taskq_lock);
831 VERIFY(taskq_dispatch(system_taskq,
832 softmac_wput_nondata_task, sup, TQ_SLEEP) !=
833 TASKQID_INVALID);
834 mutex_enter(&softmac_taskq_lock);
835 sup = list_head(&softmac_taskq_list);
836 }
837
838 CALLB_CPR_SAFE_BEGIN(&cprinfo);
839 cv_wait(&softmac_taskq_cv, &softmac_taskq_lock);
840 CALLB_CPR_SAFE_END(&cprinfo, &softmac_taskq_lock);
841 }
842
843 softmac_taskq_done = B_TRUE;
844 cv_signal(&softmac_taskq_cv);
845 CALLB_CPR_EXIT(&cprinfo);
846 thread_exit();
847 }
848
849 void
softmac_wput_nondata(softmac_upper_t * sup,mblk_t * mp)850 softmac_wput_nondata(softmac_upper_t *sup, mblk_t *mp)
851 {
852 /*
853 * The processing of the message might block. Enqueue the
854 * message for later processing.
855 */
856 mutex_enter(&sup->su_disp_mutex);
857
858 if (sup->su_closing) {
859 mutex_exit(&sup->su_disp_mutex);
860 freemsg(mp);
861 return;
862 }
863
864 SOFTMAC_EQ_PENDING(sup, mp);
865
866 if (sup->su_dlpi_pending) {
867 mutex_exit(&sup->su_disp_mutex);
868 return;
869 }
870 sup->su_dlpi_pending = B_TRUE;
871 mutex_exit(&sup->su_disp_mutex);
872
873 if (taskq_dispatch(system_taskq, softmac_wput_nondata_task,
874 sup, TQ_NOSLEEP) != TASKQID_INVALID) {
875 return;
876 }
877
878 mutex_enter(&softmac_taskq_lock);
879 if (!sup->su_taskq_scheduled) {
880 list_insert_tail(&softmac_taskq_list, sup);
881 cv_signal(&softmac_taskq_cv);
882 }
883 sup->su_taskq_scheduled = B_TRUE;
884 mutex_exit(&softmac_taskq_lock);
885 }
886
887 /*
888 * Setup the dedicated-lower-stream (fast-path) for the IP/ARP upperstream.
889 */
890 static int
softmac_fastpath_setup(softmac_upper_t * sup)891 softmac_fastpath_setup(softmac_upper_t *sup)
892 {
893 softmac_t *softmac = sup->su_softmac;
894 softmac_lower_t *slp;
895 int err;
896
897 err = softmac_lower_setup(softmac, sup, &slp);
898
899 mutex_enter(&sup->su_mutex);
900 /*
901 * Wait for all data messages to be processed so that we can change
902 * the su_mode.
903 */
904 while (sup->su_tx_inprocess != 0)
905 cv_wait(&sup->su_cv, &sup->su_mutex);
906
907 ASSERT(sup->su_mode != SOFTMAC_FASTPATH);
908 ASSERT(sup->su_slp == NULL);
909 if (err != 0) {
910 sup->su_mode = SOFTMAC_SLOWPATH;
911 } else {
912 sup->su_slp = slp;
913 sup->su_mode = SOFTMAC_FASTPATH;
914 }
915 mutex_exit(&sup->su_mutex);
916 return (err);
917 }
918
919 /*
920 * Tear down the dedicated-lower-stream (fast-path) for the IP/ARP upperstream.
921 */
922 static void
softmac_fastpath_tear(softmac_upper_t * sup)923 softmac_fastpath_tear(softmac_upper_t *sup)
924 {
925 mutex_enter(&sup->su_mutex);
926 /*
927 * Wait for all data messages in the dedicated-lower-stream
928 * to be processed.
929 */
930 while (sup->su_tx_inprocess != 0)
931 cv_wait(&sup->su_cv, &sup->su_mutex);
932
933 /*
934 * Note that this function is called either when the stream is closed,
935 * or the stream is unbound (fastpath-slowpath-switch). Therefore,
936 * No need to call the tx_notify callback.
937 */
938 sup->su_tx_notify_func = NULL;
939 sup->su_tx_notify_arg = NULL;
940 if (sup->su_tx_busy) {
941 ASSERT(sup->su_tx_flow_mp == NULL);
942 VERIFY((sup->su_tx_flow_mp = getq(sup->su_wq)) != NULL);
943 sup->su_tx_busy = B_FALSE;
944 }
945
946 sup->su_mode = SOFTMAC_SLOWPATH;
947
948 /*
949 * Destroy the dedicated-lower-stream. Note that slp is destroyed
950 * when lh is closed.
951 */
952 (void) ldi_close(sup->su_slp->sl_lh, FREAD|FWRITE, kcred);
953 sup->su_slp = NULL;
954 mutex_exit(&sup->su_mutex);
955 }
956
957 void
softmac_wput_data(softmac_upper_t * sup,mblk_t * mp)958 softmac_wput_data(softmac_upper_t *sup, mblk_t *mp)
959 {
960 /*
961 * No lock is required to access the su_mode field since the data
962 * traffic is quiesce by IP when the data-path mode is in the
963 * process of switching.
964 */
965 if (sup->su_mode != SOFTMAC_FASTPATH)
966 (void) dld_wput(sup->su_wq, mp);
967 else
968 (void) softmac_fastpath_wput_data(sup, mp, (uintptr_t)NULL, 0);
969 }
970
971 /*ARGSUSED*/
972 static mac_tx_cookie_t
softmac_fastpath_wput_data(softmac_upper_t * sup,mblk_t * mp,uintptr_t f_hint,uint16_t flag)973 softmac_fastpath_wput_data(softmac_upper_t *sup, mblk_t *mp, uintptr_t f_hint,
974 uint16_t flag)
975 {
976 queue_t *wq = sup->su_slp->sl_wq;
977
978 /*
979 * This function is called from IP, only the MAC_DROP_ON_NO_DESC
980 * flag can be specified.
981 */
982 ASSERT((flag & ~MAC_DROP_ON_NO_DESC) == 0);
983 ASSERT(mp->b_next == NULL);
984
985 /*
986 * Check wether the dedicated-lower-stream is able to handle more
987 * messages, and enable the flow-control if it is not.
988 *
989 * Note that in order not to introduce any packet reordering, we
990 * always send the message down to the dedicated-lower-stream:
991 *
992 * If the flow-control is already enabled, but we still get
993 * the messages from the upper-stream, it means that the upper
994 * stream does not respect STREAMS flow-control (e.g., TCP). Simply
995 * pass the message down to the lower-stream in that case.
996 */
997 if (SOFTMAC_CANPUTNEXT(wq)) {
998 putnext(wq, mp);
999 return ((mac_tx_cookie_t)NULL);
1000 }
1001
1002 if (sup->su_tx_busy) {
1003 if ((flag & MAC_DROP_ON_NO_DESC) != 0)
1004 freemsg(mp);
1005 else
1006 putnext(wq, mp);
1007 return ((mac_tx_cookie_t)sup);
1008 }
1009
1010 mutex_enter(&sup->su_mutex);
1011 if (!sup->su_tx_busy) {
1012 /*
1013 * If DLD_CAPAB_DIRECT is enabled, the notify callback will be
1014 * called when the flow control can be disabled. Otherwise,
1015 * put the tx_flow_mp into the wq to make use of the old
1016 * streams flow control.
1017 */
1018 ASSERT(sup->su_tx_flow_mp != NULL);
1019 (void) putq(sup->su_wq, sup->su_tx_flow_mp);
1020 sup->su_tx_flow_mp = NULL;
1021 sup->su_tx_busy = B_TRUE;
1022 qenable(wq);
1023 }
1024 mutex_exit(&sup->su_mutex);
1025
1026 if ((flag & MAC_DROP_ON_NO_DESC) != 0)
1027 freemsg(mp);
1028 else
1029 putnext(wq, mp);
1030 return ((mac_tx_cookie_t)sup);
1031 }
1032
1033 boolean_t
softmac_active_set(void * arg)1034 softmac_active_set(void *arg)
1035 {
1036 softmac_t *softmac = arg;
1037
1038 mutex_enter(&softmac->smac_active_mutex);
1039 if (softmac->smac_nactive != 0) {
1040 mutex_exit(&softmac->smac_active_mutex);
1041 return (B_FALSE);
1042 }
1043 softmac->smac_active = B_TRUE;
1044 mutex_exit(&softmac->smac_active_mutex);
1045 return (B_TRUE);
1046 }
1047
1048 void
softmac_active_clear(void * arg)1049 softmac_active_clear(void *arg)
1050 {
1051 softmac_t *softmac = arg;
1052
1053 mutex_enter(&softmac->smac_active_mutex);
1054 ASSERT(softmac->smac_active && (softmac->smac_nactive == 0));
1055 softmac->smac_active = B_FALSE;
1056 mutex_exit(&softmac->smac_active_mutex);
1057 }
1058
1059 /*
1060 * Disable/reenable fastpath on given softmac. This request could come from a
1061 * MAC client or directly from administrators.
1062 */
1063 int
softmac_datapath_switch(softmac_t * softmac,boolean_t disable,boolean_t admin)1064 softmac_datapath_switch(softmac_t *softmac, boolean_t disable, boolean_t admin)
1065 {
1066 softmac_upper_t *sup;
1067 mblk_t *head = NULL, *tail = NULL, *mp;
1068 list_t reqlist;
1069 softmac_switch_req_t *req;
1070 uint32_t current_mode, expected_mode;
1071 int err = 0;
1072
1073 mutex_enter(&softmac->smac_fp_mutex);
1074
1075 current_mode = DATAPATH_MODE(softmac);
1076 if (admin) {
1077 if (softmac->smac_fastpath_admin_disabled == disable) {
1078 mutex_exit(&softmac->smac_fp_mutex);
1079 return (0);
1080 }
1081 softmac->smac_fastpath_admin_disabled = disable;
1082 } else if (disable) {
1083 softmac->smac_fp_disable_clients++;
1084 } else {
1085 ASSERT(softmac->smac_fp_disable_clients != 0);
1086 softmac->smac_fp_disable_clients--;
1087 }
1088
1089 expected_mode = DATAPATH_MODE(softmac);
1090 if (current_mode == expected_mode) {
1091 mutex_exit(&softmac->smac_fp_mutex);
1092 return (0);
1093 }
1094
1095 /*
1096 * The expected mode is different from whatever datapath mode
1097 * this softmac is expected from last request, enqueue the data-path
1098 * switch request.
1099 */
1100 list_create(&reqlist, sizeof (softmac_switch_req_t),
1101 offsetof(softmac_switch_req_t, ssq_req_list_node));
1102
1103 /*
1104 * Allocate all DL_NOTIFY_IND messages and request structures that
1105 * are required to switch each IP/ARP stream to the expected mode.
1106 */
1107 for (sup = list_head(&softmac->smac_sup_list); sup != NULL;
1108 sup = list_next(&softmac->smac_sup_list, sup)) {
1109 dl_notify_ind_t *dlip;
1110
1111 req = kmem_alloc(sizeof (softmac_switch_req_t), KM_NOSLEEP);
1112 if (req == NULL)
1113 break;
1114
1115 req->ssq_expected_mode = expected_mode;
1116 if (sup->su_is_arp) {
1117 list_insert_tail(&reqlist, req);
1118 continue;
1119 }
1120 /*
1121 * Allocate the DL_NOTE_REPLUMB message.
1122 */
1123 if ((mp = allocb(sizeof (dl_notify_ind_t), BPRI_LO)) == NULL) {
1124 kmem_free(req, sizeof (softmac_switch_req_t));
1125 break;
1126 }
1127
1128 list_insert_tail(&reqlist, req);
1129
1130 mp->b_wptr = mp->b_rptr + sizeof (dl_notify_ind_t);
1131 mp->b_datap->db_type = M_PROTO;
1132 bzero(mp->b_rptr, sizeof (dl_notify_ind_t));
1133 dlip = (dl_notify_ind_t *)mp->b_rptr;
1134 dlip->dl_primitive = DL_NOTIFY_IND;
1135 dlip->dl_notification = DL_NOTE_REPLUMB;
1136 if (head == NULL) {
1137 head = tail = mp;
1138 } else {
1139 tail->b_next = mp;
1140 tail = mp;
1141 }
1142 }
1143
1144 /*
1145 * Note that it is fine if the expected data-path mode is fast-path
1146 * and some of streams fails to switch. Only return failure if we
1147 * are expected to switch to the slow-path.
1148 */
1149 if (sup != NULL && expected_mode == SOFTMAC_SLOWPATH) {
1150 err = ENOMEM;
1151 goto fail;
1152 }
1153
1154 /*
1155 * Start switching for each IP/ARP stream. The switching operation
1156 * will eventually succeed and there is no need to wait for it
1157 * to finish.
1158 */
1159 for (sup = list_head(&softmac->smac_sup_list); sup != NULL;
1160 sup = list_next(&softmac->smac_sup_list, sup)) {
1161 if (!sup->su_is_arp) {
1162 mp = head->b_next;
1163 head->b_next = NULL;
1164 softmac_wput_nondata(sup, head);
1165 head = mp;
1166 }
1167 /*
1168 * Add the switch request to the requests list of the stream.
1169 */
1170 req = list_head(&reqlist);
1171 ASSERT(req != NULL);
1172 list_remove(&reqlist, req);
1173 list_insert_tail(&sup->su_req_list, req);
1174 }
1175
1176 mutex_exit(&softmac->smac_fp_mutex);
1177 ASSERT(list_is_empty(&reqlist));
1178 list_destroy(&reqlist);
1179 return (0);
1180 fail:
1181 if (admin) {
1182 softmac->smac_fastpath_admin_disabled = !disable;
1183 } else if (disable) {
1184 softmac->smac_fp_disable_clients--;
1185 } else {
1186 softmac->smac_fp_disable_clients++;
1187 }
1188
1189 mutex_exit(&softmac->smac_fp_mutex);
1190 while ((req = list_head(&reqlist)) != NULL) {
1191 list_remove(&reqlist, req);
1192 kmem_free(req, sizeof (softmac_switch_req_t));
1193 }
1194 freemsgchain(head);
1195 list_destroy(&reqlist);
1196 return (err);
1197 }
1198
1199 int
softmac_fastpath_disable(void * arg)1200 softmac_fastpath_disable(void *arg)
1201 {
1202 return (softmac_datapath_switch((softmac_t *)arg, B_TRUE, B_FALSE));
1203 }
1204
1205 void
softmac_fastpath_enable(void * arg)1206 softmac_fastpath_enable(void *arg)
1207 {
1208 VERIFY(softmac_datapath_switch((softmac_t *)arg, B_FALSE,
1209 B_FALSE) == 0);
1210 }
1211
1212 void
softmac_upperstream_close(softmac_upper_t * sup)1213 softmac_upperstream_close(softmac_upper_t *sup)
1214 {
1215 softmac_t *softmac = sup->su_softmac;
1216 softmac_switch_req_t *req;
1217
1218 mutex_enter(&softmac->smac_fp_mutex);
1219
1220 if (sup->su_mode == SOFTMAC_FASTPATH)
1221 softmac_fastpath_tear(sup);
1222
1223 if (sup->su_mode != SOFTMAC_UNKNOWN) {
1224 list_remove(&softmac->smac_sup_list, sup);
1225 sup->su_mode = SOFTMAC_UNKNOWN;
1226 }
1227
1228 /*
1229 * Cleanup all the switch requests queueed on this stream.
1230 */
1231 while ((req = list_head(&sup->su_req_list)) != NULL) {
1232 list_remove(&sup->su_req_list, req);
1233 kmem_free(req, sizeof (softmac_switch_req_t));
1234 }
1235 mutex_exit(&softmac->smac_fp_mutex);
1236 }
1237
1238 /*
1239 * Handle the DL_NOTE_REPLUMB_DONE indication from IP/ARP. Change the upper
1240 * stream from the fastpath mode to the slowpath mode.
1241 */
1242 static void
softmac_datapath_switch_done(softmac_upper_t * sup)1243 softmac_datapath_switch_done(softmac_upper_t *sup)
1244 {
1245 softmac_t *softmac = sup->su_softmac;
1246 softmac_switch_req_t *req;
1247 uint32_t expected_mode;
1248
1249 mutex_enter(&softmac->smac_fp_mutex);
1250 req = list_head(&sup->su_req_list);
1251 list_remove(&sup->su_req_list, req);
1252 expected_mode = req->ssq_expected_mode;
1253 kmem_free(req, sizeof (softmac_switch_req_t));
1254
1255 if (expected_mode == sup->su_mode) {
1256 mutex_exit(&softmac->smac_fp_mutex);
1257 return;
1258 }
1259
1260 ASSERT(!sup->su_bound);
1261 mutex_exit(&softmac->smac_fp_mutex);
1262
1263 /*
1264 * It is fine if the expected mode is fast-path and we fail
1265 * to enable fastpath on this stream.
1266 */
1267 if (expected_mode == SOFTMAC_SLOWPATH)
1268 softmac_fastpath_tear(sup);
1269 else
1270 (void) softmac_fastpath_setup(sup);
1271 }
1272