/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2008 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * Data-Link Driver */ #include #include #include #include #include #include #include #include #include #include #include static int str_constructor(void *, void *, int); static void str_destructor(void *, void *); static mblk_t *str_unitdata_ind(dld_str_t *, mblk_t *, boolean_t); static void str_notify_promisc_on_phys(dld_str_t *); static void str_notify_promisc_off_phys(dld_str_t *); static void str_notify_phys_addr(dld_str_t *, const uint8_t *); static void str_notify_link_up(dld_str_t *); static void str_notify_link_down(dld_str_t *); static void str_notify_capab_reneg(dld_str_t *); static void str_notify_speed(dld_str_t *, uint32_t); static void ioc_native(dld_str_t *, mblk_t *); static void ioc_margin(dld_str_t *, mblk_t *); static void ioc_raw(dld_str_t *, mblk_t *); static void ioc_fast(dld_str_t *, mblk_t *); static void ioc(dld_str_t *, mblk_t *); static void dld_ioc(dld_str_t *, mblk_t *); static void dld_wput_nondata(dld_str_t *, mblk_t *); static void str_mdata_raw_put(dld_str_t *, mblk_t *); static mblk_t *i_dld_ether_header_update_tag(mblk_t *, uint_t, uint16_t); static mblk_t *i_dld_ether_header_strip_tag(mblk_t *); static uint32_t str_count; static kmem_cache_t *str_cachep; static mod_hash_t *str_hashp; #define STR_HASHSZ 64 #define STR_HASH_KEY(key) ((mod_hash_key_t)(uintptr_t)(key)) #define dld_taskq system_taskq static kmutex_t dld_taskq_lock; static kcondvar_t dld_taskq_cv; static list_t dld_taskq_list; /* List of dld_str_t */ boolean_t dld_taskq_quit; boolean_t dld_taskq_done; static void dld_taskq_dispatch(void); /* * Some notes on entry points, flow-control, queueing. * * This driver exports the traditional STREAMS put entry point as well as * the non-STREAMS fast-path transmit routine which is provided to IP via * the DL_CAPAB_POLL negotiation. The put procedure handles all control * and data operations, while the fast-path routine deals only with M_DATA * fast-path packets. Regardless of the entry point, all outbound packets * will end up in DLD_TX(), where they will be delivered to the MAC layer. * * The transmit logic operates in the following way: All packets coming * into DLD will be sent to the MAC layer through DLD_TX(). Flow-control * happens when the MAC layer indicates the packets couldn't be * transmitted due to 1) lack of resources (e.g. running out of * descriptors), or 2) reaching the allowed bandwidth limit for this * particular flow. The indication comes in the form of a Tx cookie that * identifies the blocked ring. In such case, DLD will place a * dummy message on its write-side STREAMS queue so that the queue is * marked as "full". Any subsequent packets arriving at the driver will * still be sent to the MAC layer where it either gets queued in the Tx * SRS or discarded it if queue limit is exceeded. The write-side STREAMS * queue gets enabled when MAC layer notifies DLD through MAC_NOTE_TX. * When the write service procedure runs, it will remove the dummy * message from the write-side STREAMS queue; in effect this will trigger * backenabling. The sizes of q_hiwat and q_lowat are set to 1 and 0, * respectively, due to the above reasons. * * All non-data operations, both DLPI and ioctls are single threaded on a per * dld_str_t endpoint. This is done using a taskq so that the control operation * has kernel context and can cv_wait for resources. In addition all set type * operations that involve mac level state modification are serialized on a * per mac end point using the perimeter mechanism provided by the mac layer. * This serializes all mac clients trying to modify a single mac end point over * the entire sequence of mac calls made by that client as an atomic unit. The * mac framework locking is described in mac.c. A critical element is that * DLD/DLS does not hold any locks across the mac perimeter. * * dld_finddevinfo() returns the dev_info_t * corresponding to a particular * dev_t. It searches str_hashp (a table of dld_str_t's) for streams that * match dev_t. If a stream is found and it is attached, its dev_info_t * * is returned. If the mac handle is non-null, it can be safely accessed * below. The mac handle won't be freed until the mac_unregister which * won't happen until the driver detaches. The DDI framework ensures that * the detach won't happen while a getinfo is in progress. */ typedef struct i_dld_str_state_s { major_t ds_major; minor_t ds_minor; dev_info_t *ds_dip; } i_dld_str_state_t; /* ARGSUSED */ static uint_t i_dld_str_walker(mod_hash_key_t key, mod_hash_val_t *val, void *arg) { i_dld_str_state_t *statep = arg; dld_str_t *dsp = (dld_str_t *)val; mac_handle_t mh; if (statep->ds_major != dsp->ds_major) return (MH_WALK_CONTINUE); ASSERT(statep->ds_minor != 0); mh = dsp->ds_mh; if (statep->ds_minor == dsp->ds_minor) { /* * Clone: a clone minor is unique. we can terminate the * walk if we find a matching stream -- even if we fail * to obtain the devinfo. */ if (mh != NULL) statep->ds_dip = mac_devinfo_get(mh); return (MH_WALK_TERMINATE); } return (MH_WALK_CONTINUE); } static dev_info_t * dld_finddevinfo(dev_t dev) { dev_info_t *dip; i_dld_str_state_t state; if (getminor(dev) == 0) return (NULL); /* * See if it's a minor node of a link */ if ((dip = dls_link_devinfo(dev)) != NULL) return (dip); state.ds_minor = getminor(dev); state.ds_major = getmajor(dev); state.ds_dip = NULL; mod_hash_walk(str_hashp, i_dld_str_walker, &state); return (state.ds_dip); } /* * devo_getinfo: getinfo(9e) */ /*ARGSUSED*/ int dld_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **resp) { dev_info_t *devinfo; minor_t minor = getminor((dev_t)arg); int rc = DDI_FAILURE; switch (cmd) { case DDI_INFO_DEVT2DEVINFO: if ((devinfo = dld_finddevinfo((dev_t)arg)) != NULL) { *(dev_info_t **)resp = devinfo; rc = DDI_SUCCESS; } break; case DDI_INFO_DEVT2INSTANCE: if (minor > 0 && minor <= DLS_MAX_MINOR) { *resp = (void *)(uintptr_t)DLS_MINOR2INST(minor); rc = DDI_SUCCESS; } else if (minor > DLS_MAX_MINOR && (devinfo = dld_finddevinfo((dev_t)arg)) != NULL) { *resp = (void *)(uintptr_t)ddi_get_instance(devinfo); rc = DDI_SUCCESS; } break; } return (rc); } /* * qi_qopen: open(9e) */ /*ARGSUSED*/ int dld_open(queue_t *rq, dev_t *devp, int flag, int sflag, cred_t *credp) { dld_str_t *dsp; major_t major; minor_t minor; int err; if (sflag == MODOPEN) return (ENOTSUP); /* * This is a cloning driver and therefore each queue should only * ever get opened once. */ if (rq->q_ptr != NULL) return (EBUSY); major = getmajor(*devp); minor = getminor(*devp); /* * Create a new dld_str_t for the stream. This will grab a new minor * number that will be handed back in the cloned dev_t. Creation may * fail if we can't allocate the dummy mblk used for flow-control. */ dsp = dld_str_create(rq, DLD_DLPI, major, ((minor == 0) ? DL_STYLE2 : DL_STYLE1)); if (dsp == NULL) return (ENOSR); ASSERT(dsp->ds_dlstate == DL_UNATTACHED); if (minor != 0) { /* * Style 1 open */ if ((err = dld_str_attach(dsp, (t_uscalar_t)minor - 1)) != 0) goto failed; ASSERT(dsp->ds_dlstate == DL_UNBOUND); } else { (void) qassociate(rq, -1); } /* * Enable the queue srv(9e) routine. */ qprocson(rq); /* * Construct a cloned dev_t to hand back. */ *devp = makedevice(getmajor(*devp), dsp->ds_minor); return (0); failed: dld_str_destroy(dsp); return (err); } /* * qi_qclose: close(9e) */ int dld_close(queue_t *rq) { dld_str_t *dsp = rq->q_ptr; /* * All modules on top have been popped off. So there can't be any * threads from the top. */ ASSERT(dsp->ds_datathr_cnt == 0); /* * Wait until pending DLPI requests are processed. */ mutex_enter(&dsp->ds_lock); while (dsp->ds_dlpi_pending) cv_wait(&dsp->ds_dlpi_pending_cv, &dsp->ds_lock); mutex_exit(&dsp->ds_lock); /* * Disable the queue srv(9e) routine. */ qprocsoff(rq); /* * This stream was open to a provider node. Check to see * if it has been cleanly shut down. */ if (dsp->ds_dlstate != DL_UNATTACHED) { /* * The stream is either open to a style 1 provider or * this is not clean shutdown. Detach from the PPA. * (This is still ok even in the style 1 case). */ dld_str_detach(dsp); } dld_str_destroy(dsp); return (0); } /* * qi_qputp: put(9e) */ void dld_wput(queue_t *wq, mblk_t *mp) { dld_str_t *dsp = (dld_str_t *)wq->q_ptr; dld_str_mode_t mode; switch (DB_TYPE(mp)) { case M_DATA: mutex_enter(&dsp->ds_lock); if (dsp->ds_dlstate == DL_IDLE) { mode = dsp->ds_mode; if (mode == DLD_FASTPATH || mode == DLD_RAW) { DLD_DATATHR_INC(dsp); mutex_exit(&dsp->ds_lock); if (mode == DLD_FASTPATH) { (void) str_mdata_fastpath_put(dsp, mp, 0, 0); } else { str_mdata_raw_put(dsp, mp); } DLD_DATATHR_DCR(dsp); break; } } mutex_exit(&dsp->ds_lock); freemsg(mp); break; case M_PROTO: case M_PCPROTO: { t_uscalar_t prim; if (MBLKL(mp) < sizeof (t_uscalar_t)) break; prim = ((union DL_primitives *)mp->b_rptr)->dl_primitive; if (prim == DL_UNITDATA_REQ) { proto_unitdata_req(dsp, mp); } else { dld_wput_nondata(dsp, mp); } break; } case M_IOCTL: dld_wput_nondata(dsp, mp); break; case M_FLUSH: if (*mp->b_rptr & FLUSHW) { DLD_CLRQFULL(dsp); *mp->b_rptr &= ~FLUSHW; } if (*mp->b_rptr & FLUSHR) { qreply(wq, mp); } else { freemsg(mp); } break; default: freemsg(mp); break; } } /* * qi_srvp: srv(9e) */ void dld_wsrv(queue_t *wq) { dld_str_t *dsp = wq->q_ptr; DLD_CLRQFULL(dsp); } void dld_init_ops(struct dev_ops *ops, const char *name) { struct streamtab *stream; struct qinit *rq, *wq; struct module_info *modinfo; modinfo = kmem_zalloc(sizeof (struct module_info), KM_SLEEP); modinfo->mi_idname = kmem_zalloc(FMNAMESZ, KM_SLEEP); (void) snprintf(modinfo->mi_idname, FMNAMESZ, "%s", name); modinfo->mi_minpsz = 0; modinfo->mi_maxpsz = 64*1024; modinfo->mi_hiwat = 1; modinfo->mi_lowat = 0; rq = kmem_zalloc(sizeof (struct qinit), KM_SLEEP); rq->qi_qopen = dld_open; rq->qi_qclose = dld_close; rq->qi_minfo = modinfo; wq = kmem_zalloc(sizeof (struct qinit), KM_SLEEP); wq->qi_putp = (pfi_t)dld_wput; wq->qi_srvp = (pfi_t)dld_wsrv; wq->qi_minfo = modinfo; stream = kmem_zalloc(sizeof (struct streamtab), KM_SLEEP); stream->st_rdinit = rq; stream->st_wrinit = wq; ops->devo_cb_ops->cb_str = stream; if (ops->devo_getinfo == NULL) ops->devo_getinfo = &dld_getinfo; } void dld_fini_ops(struct dev_ops *ops) { struct streamtab *stream; struct qinit *rq, *wq; struct module_info *modinfo; stream = ops->devo_cb_ops->cb_str; rq = stream->st_rdinit; wq = stream->st_wrinit; modinfo = rq->qi_minfo; ASSERT(wq->qi_minfo == modinfo); kmem_free(stream, sizeof (struct streamtab)); kmem_free(wq, sizeof (struct qinit)); kmem_free(rq, sizeof (struct qinit)); kmem_free(modinfo->mi_idname, FMNAMESZ); kmem_free(modinfo, sizeof (struct module_info)); } /* * Initialize this module's data structures. */ void dld_str_init(void) { /* * Create dld_str_t object cache. */ str_cachep = kmem_cache_create("dld_str_cache", sizeof (dld_str_t), 0, str_constructor, str_destructor, NULL, NULL, NULL, 0); ASSERT(str_cachep != NULL); /* * Create a hash table for maintaining dld_str_t's. * The ds_minor field (the clone minor number) of a dld_str_t * is used as a key for this hash table because this number is * globally unique (allocated from "dls_minor_arena"). */ str_hashp = mod_hash_create_idhash("dld_str_hash", STR_HASHSZ, mod_hash_null_valdtor); mutex_init(&dld_taskq_lock, NULL, MUTEX_DRIVER, NULL); cv_init(&dld_taskq_cv, NULL, CV_DRIVER, NULL); dld_taskq_quit = B_FALSE; dld_taskq_done = B_FALSE; list_create(&dld_taskq_list, sizeof (dld_str_t), offsetof(dld_str_t, ds_tqlist)); (void) thread_create(NULL, 0, dld_taskq_dispatch, NULL, 0, &p0, TS_RUN, minclsyspri); } /* * Tear down this module's data structures. */ int dld_str_fini(void) { /* * Make sure that there are no objects in use. */ if (str_count != 0) return (EBUSY); /* * Ask the dld_taskq thread to quit and wait for it to be done */ mutex_enter(&dld_taskq_lock); dld_taskq_quit = B_TRUE; cv_signal(&dld_taskq_cv); while (!dld_taskq_done) cv_wait(&dld_taskq_cv, &dld_taskq_lock); mutex_exit(&dld_taskq_lock); list_destroy(&dld_taskq_list); /* * Destroy object cache. */ kmem_cache_destroy(str_cachep); mod_hash_destroy_idhash(str_hashp); return (0); } /* * Create a new dld_str_t object. */ dld_str_t * dld_str_create(queue_t *rq, uint_t type, major_t major, t_uscalar_t style) { dld_str_t *dsp; int err; /* * Allocate an object from the cache. */ atomic_add_32(&str_count, 1); dsp = kmem_cache_alloc(str_cachep, KM_SLEEP); /* * Allocate the dummy mblk for flow-control. */ dsp->ds_tx_flow_mp = allocb(1, BPRI_HI); if (dsp->ds_tx_flow_mp == NULL) { kmem_cache_free(str_cachep, dsp); atomic_add_32(&str_count, -1); return (NULL); } dsp->ds_type = type; dsp->ds_major = major; dsp->ds_style = style; /* * Initialize the queue pointers. */ ASSERT(RD(rq) == rq); dsp->ds_rq = rq; dsp->ds_wq = WR(rq); rq->q_ptr = WR(rq)->q_ptr = (void *)dsp; /* * We want explicit control over our write-side STREAMS queue * where the dummy mblk gets added/removed for flow-control. */ noenable(WR(rq)); err = mod_hash_insert(str_hashp, STR_HASH_KEY(dsp->ds_minor), (mod_hash_val_t)dsp); ASSERT(err == 0); return (dsp); } /* * Destroy a dld_str_t object. */ void dld_str_destroy(dld_str_t *dsp) { queue_t *rq; queue_t *wq; mod_hash_val_t val; /* * Clear the queue pointers. */ rq = dsp->ds_rq; wq = dsp->ds_wq; ASSERT(wq == WR(rq)); rq->q_ptr = wq->q_ptr = NULL; dsp->ds_rq = dsp->ds_wq = NULL; ASSERT(dsp->ds_dlstate == DL_UNATTACHED); ASSERT(dsp->ds_sap == 0); ASSERT(dsp->ds_mh == NULL); ASSERT(dsp->ds_mch == NULL); ASSERT(dsp->ds_promisc == 0); ASSERT(dsp->ds_mph == NULL); ASSERT(dsp->ds_mip == NULL); ASSERT(dsp->ds_mnh == NULL); ASSERT(dsp->ds_polling == B_FALSE); ASSERT(dsp->ds_direct == B_FALSE); ASSERT(dsp->ds_lso == B_FALSE); ASSERT(dsp->ds_lso_max == 0); /* * Reinitialize all the flags. */ dsp->ds_notifications = 0; dsp->ds_passivestate = DLD_UNINITIALIZED; dsp->ds_mode = DLD_UNITDATA; dsp->ds_native = B_FALSE; ASSERT(dsp->ds_datathr_cnt == 0); ASSERT(dsp->ds_pending_head == NULL); ASSERT(dsp->ds_pending_tail == NULL); ASSERT(!dsp->ds_dlpi_pending); ASSERT(dsp->ds_dlp == NULL); ASSERT(dsp->ds_dmap == NULL); ASSERT(dsp->ds_rx == NULL); ASSERT(dsp->ds_rx_arg == NULL); ASSERT(dsp->ds_next == NULL); ASSERT(dsp->ds_head == NULL); /* * Free the dummy mblk if exists. */ if (dsp->ds_tx_flow_mp != NULL) { freeb(dsp->ds_tx_flow_mp); dsp->ds_tx_flow_mp = NULL; } (void) mod_hash_remove(str_hashp, STR_HASH_KEY(dsp->ds_minor), &val); ASSERT(dsp == (dld_str_t *)val); /* * Free the object back to the cache. */ kmem_cache_free(str_cachep, dsp); atomic_add_32(&str_count, -1); } /* * kmem_cache contructor function: see kmem_cache_create(9f). */ /*ARGSUSED*/ static int str_constructor(void *buf, void *cdrarg, int kmflags) { dld_str_t *dsp = buf; bzero(buf, sizeof (dld_str_t)); /* * Allocate a new minor number. */ if ((dsp->ds_minor = mac_minor_hold(kmflags == KM_SLEEP)) == 0) return (-1); /* * Initialize the DLPI state machine. */ dsp->ds_dlstate = DL_UNATTACHED; mutex_init(&dsp->ds_lock, NULL, MUTEX_DRIVER, NULL); cv_init(&dsp->ds_datathr_cv, NULL, CV_DRIVER, NULL); cv_init(&dsp->ds_dlpi_pending_cv, NULL, CV_DRIVER, NULL); return (0); } /* * kmem_cache destructor function. */ /*ARGSUSED*/ static void str_destructor(void *buf, void *cdrarg) { dld_str_t *dsp = buf; /* * Release the minor number. */ mac_minor_rele(dsp->ds_minor); ASSERT(dsp->ds_tx_flow_mp == NULL); mutex_destroy(&dsp->ds_lock); cv_destroy(&dsp->ds_datathr_cv); cv_destroy(&dsp->ds_dlpi_pending_cv); } /* * Update the priority bits and VID (may need to insert tag if mp points * to an untagged packet. * If vid is VLAN_ID_NONE, use the VID encoded in the packet. */ static mblk_t * i_dld_ether_header_update_tag(mblk_t *mp, uint_t pri, uint16_t vid) { mblk_t *hmp; struct ether_vlan_header *evhp; struct ether_header *ehp; uint16_t old_tci = 0; size_t len; ASSERT(pri != 0 || vid != VLAN_ID_NONE); evhp = (struct ether_vlan_header *)mp->b_rptr; if (ntohs(evhp->ether_tpid) == ETHERTYPE_VLAN) { /* * Tagged packet, update the priority bits. */ old_tci = ntohs(evhp->ether_tci); len = sizeof (struct ether_vlan_header); if ((DB_REF(mp) > 1) || (MBLKL(mp) < len)) { /* * In case some drivers only check the db_ref * count of the first mblk, we pullup the * message into a single mblk. */ hmp = msgpullup(mp, -1); if ((hmp == NULL) || (MBLKL(hmp) < len)) { freemsg(hmp); return (NULL); } else { freemsg(mp); mp = hmp; } } evhp = (struct ether_vlan_header *)mp->b_rptr; } else { /* * Untagged packet. Insert the special priority tag. * First allocate a header mblk. */ hmp = allocb(sizeof (struct ether_vlan_header), BPRI_MED); if (hmp == NULL) return (NULL); evhp = (struct ether_vlan_header *)hmp->b_rptr; ehp = (struct ether_header *)mp->b_rptr; /* * Copy the MAC addresses and typelen */ bcopy(ehp, evhp, (ETHERADDRL * 2)); evhp->ether_type = ehp->ether_type; evhp->ether_tpid = htons(ETHERTYPE_VLAN); hmp->b_wptr += sizeof (struct ether_vlan_header); mp->b_rptr += sizeof (struct ether_header); /* * Free the original message if it's now empty. Link the * rest of the messages to the header message. */ if (MBLKL(mp) == 0) { hmp->b_cont = mp->b_cont; freeb(mp); } else { hmp->b_cont = mp; } mp = hmp; } if (pri == 0) pri = VLAN_PRI(old_tci); if (vid == VLAN_ID_NONE) vid = VLAN_ID(old_tci); evhp->ether_tci = htons(VLAN_TCI(pri, VLAN_CFI(old_tci), vid)); return (mp); } /* * M_DATA put (IP fast-path mode) */ mac_tx_cookie_t str_mdata_fastpath_put(dld_str_t *dsp, mblk_t *mp, uintptr_t f_hint, uint16_t flag) { boolean_t is_ethernet = (dsp->ds_mip->mi_media == DL_ETHER); mblk_t *newmp; uint_t pri; mac_tx_cookie_t cookie; if (is_ethernet) { /* * Update the priority bits to the assigned priority. */ pri = (VLAN_MBLKPRI(mp) == 0) ? dsp->ds_pri : VLAN_MBLKPRI(mp); if (pri != 0) { newmp = i_dld_ether_header_update_tag(mp, pri, VLAN_ID_NONE); if (newmp == NULL) goto discard; mp = newmp; } } if ((cookie = DLD_TX(dsp, mp, f_hint, flag)) != NULL) { DLD_SETQFULL(dsp); } return (cookie); discard: /* TODO: bump kstat? */ freemsg(mp); return (NULL); } /* * M_DATA put (DLIOCRAW mode) */ static void str_mdata_raw_put(dld_str_t *dsp, mblk_t *mp) { boolean_t is_ethernet = (dsp->ds_mip->mi_media == DL_ETHER); mblk_t *bp, *newmp; size_t size; mac_header_info_t mhi; uint_t pri, vid, dvid; uint_t max_sdu; /* * Certain MAC type plugins provide an illusion for raw DLPI * consumers. They pretend that the MAC layer is something that * it's not for the benefit of observability tools. For example, * mac_wifi pretends that it's Ethernet for such consumers. * Here, unless native mode is enabled, we call into the MAC layer so * that this illusion can be maintained. The plugin will optionally * transform the MAC header here into something that can be passed * down. The header goes from raw mode to "cooked" mode. */ if (!dsp->ds_native) { if ((newmp = mac_header_cook(dsp->ds_mh, mp)) == NULL) goto discard; mp = newmp; } size = MBLKL(mp); /* * Check the packet is not too big and that any remaining * fragment list is composed entirely of M_DATA messages. (We * know the first fragment was M_DATA otherwise we could not * have got here). */ for (bp = mp->b_cont; bp != NULL; bp = bp->b_cont) { if (DB_TYPE(bp) != M_DATA) goto discard; size += MBLKL(bp); } if (dls_link_header_info(dsp->ds_dlp, mp, &mhi) != 0) goto discard; mac_sdu_get(dsp->ds_mh, NULL, &max_sdu); /* * If LSO is enabled, check the size against lso_max. Otherwise, * compare the packet size with max_sdu. */ max_sdu = dsp->ds_lso ? dsp->ds_lso_max : max_sdu; if (size > max_sdu + mhi.mhi_hdrsize) goto discard; if (is_ethernet) { dvid = mac_client_vid(dsp->ds_mch); /* * Discard the packet if this is a VLAN stream but the VID in * the packet is not correct. */ vid = VLAN_ID(mhi.mhi_tci); if ((dvid != VLAN_ID_NONE) && (vid != VLAN_ID_NONE)) goto discard; /* * Discard the packet if this packet is a tagged packet * but both pri and VID are 0. */ pri = VLAN_PRI(mhi.mhi_tci); if (mhi.mhi_istagged && (pri == 0) && (vid == VLAN_ID_NONE)) goto discard; /* * Update the priority bits to the per-stream priority if * priority is not set in the packet. Update the VID for * packets on a VLAN stream. */ pri = (pri == 0) ? dsp->ds_pri : 0; if ((pri != 0) || (dvid != VLAN_ID_NONE)) { if ((newmp = i_dld_ether_header_update_tag(mp, pri, dvid)) == NULL) { goto discard; } mp = newmp; } } if (DLD_TX(dsp, mp, 0, 0) != NULL) { /* Turn on flow-control for dld */ DLD_SETQFULL(dsp); } return; discard: /* TODO: bump kstat? */ freemsg(mp); } /* * Process DL_ATTACH_REQ (style 2) or open(2) (style 1). */ int dld_str_attach(dld_str_t *dsp, t_uscalar_t ppa) { dev_t dev; int err; const char *drvname; mac_perim_handle_t mph; boolean_t qassociated = B_FALSE; dls_link_t *dlp = NULL; dls_dl_handle_t ddp = NULL; boolean_t entered_perim = B_FALSE; if ((drvname = ddi_major_to_name(dsp->ds_major)) == NULL) return (EINVAL); if (dsp->ds_style == DL_STYLE2 && ppa > DLS_MAX_PPA) return (ENOTSUP); /* * /dev node access. This will still be supported for backward * compatibility reason. */ if ((dsp->ds_style == DL_STYLE2) && (strcmp(drvname, "aggr") != 0) && (strcmp(drvname, "vnic") != 0)) { if (qassociate(dsp->ds_wq, DLS_PPA2INST(ppa)) != 0) return (EINVAL); qassociated = B_TRUE; } dev = makedevice(dsp->ds_major, (minor_t)ppa + 1); if ((err = dls_devnet_hold_by_dev(dev, &ddp)) != 0) goto failed; if ((err = mac_perim_enter_by_macname(dls_devnet_mac(ddp), &mph)) != 0) goto failed; entered_perim = B_TRUE; /* * Open a channel. */ if ((err = dls_link_hold(dls_devnet_mac(ddp), &dlp)) != 0) goto failed; if ((err = dls_open(dlp, ddp, dsp)) != 0) goto failed; /* * Set the default packet priority. */ dsp->ds_pri = 0; /* * Add a notify function so that the we get updates from the MAC. */ dsp->ds_mnh = mac_notify_add(dsp->ds_mh, str_notify, dsp); dsp->ds_dlstate = DL_UNBOUND; mac_perim_exit(mph); return (0); failed: if (dlp != NULL) dls_link_rele(dlp); if (entered_perim) mac_perim_exit(mph); if (ddp != NULL) dls_devnet_rele(ddp); if (qassociated) (void) qassociate(dsp->ds_wq, -1); return (err); } /* * Process DL_DETACH_REQ (style 2) or close(2) (style 1). Can also be called * from close(2) for style 2. */ void dld_str_detach(dld_str_t *dsp) { mac_perim_handle_t mph; int err; ASSERT(dsp->ds_datathr_cnt == 0); mac_perim_enter_by_mh(dsp->ds_mh, &mph); /* * Remove the notify function. * * Note that we cannot wait for the notification callback to be removed * since it could cause the deadlock with str_notify() since they both * need the mac perimeter. Continue if we cannot remove the * notification callback right now and wait after we leave the * perimeter. */ err = mac_notify_remove(dsp->ds_mnh, B_FALSE); dsp->ds_mnh = NULL; /* * Disable the capabilities */ dld_capabilities_disable(dsp); /* * Clear LSO flags. */ dsp->ds_lso = B_FALSE; dsp->ds_lso_max = 0; dls_close(dsp); mac_perim_exit(mph); /* * Now we leave the mac perimeter. If mac_notify_remove() failed * because the notification callback was in progress, wait for * it to finish before we proceed. */ if (err != 0) mac_notify_remove_wait(dsp->ds_mh); /* * An unreferenced tagged (non-persistent) vlan gets destroyed * automatically in the call to dls_devnet_rele. */ dls_devnet_rele(dsp->ds_ddh); dsp->ds_sap = 0; dsp->ds_mh = NULL; dsp->ds_mch = NULL; dsp->ds_mip = NULL; if (dsp->ds_style == DL_STYLE2) (void) qassociate(dsp->ds_wq, -1); /* * Re-initialize the DLPI state machine. */ dsp->ds_dlstate = DL_UNATTACHED; } /* * This function is only called for VLAN streams. In raw mode, we strip VLAN * tags before sending packets up to the DLS clients, with the exception of * special priority tagged packets, in that case, we set the VID to 0. * mp must be a VLAN tagged packet. */ static mblk_t * i_dld_ether_header_strip_tag(mblk_t *mp) { mblk_t *newmp; struct ether_vlan_header *evhp; uint16_t tci, new_tci; ASSERT(MBLKL(mp) >= sizeof (struct ether_vlan_header)); if (DB_REF(mp) > 1) { newmp = copymsg(mp); if (newmp == NULL) return (NULL); freemsg(mp); mp = newmp; } evhp = (struct ether_vlan_header *)mp->b_rptr; tci = ntohs(evhp->ether_tci); if (VLAN_PRI(tci) == 0) { /* * Priority is 0, strip the tag. */ ovbcopy(mp->b_rptr, mp->b_rptr + VLAN_TAGSZ, 2 * ETHERADDRL); mp->b_rptr += VLAN_TAGSZ; } else { /* * Priority is not 0, update the VID to 0. */ new_tci = VLAN_TCI(VLAN_PRI(tci), VLAN_CFI(tci), VLAN_ID_NONE); evhp->ether_tci = htons(new_tci); } return (mp); } /* * Raw mode receive function. */ /*ARGSUSED*/ void dld_str_rx_raw(void *arg, mac_resource_handle_t mrh, mblk_t *mp, mac_header_info_t *mhip) { dld_str_t *dsp = (dld_str_t *)arg; boolean_t is_ethernet = (dsp->ds_mip->mi_media == DL_ETHER); mblk_t *next, *newmp; ASSERT(mp != NULL); do { /* * Get the pointer to the next packet in the chain and then * clear b_next before the packet gets passed on. */ next = mp->b_next; mp->b_next = NULL; /* * Wind back b_rptr to point at the MAC header. */ ASSERT(mp->b_rptr >= DB_BASE(mp) + mhip->mhi_hdrsize); mp->b_rptr -= mhip->mhi_hdrsize; /* * Certain MAC type plugins provide an illusion for raw * DLPI consumers. They pretend that the MAC layer is * something that it's not for the benefit of observability * tools. For example, mac_wifi pretends that it's Ethernet * for such consumers. Here, unless native mode is enabled, * we call into the MAC layer so that this illusion can be * maintained. The plugin will optionally transform the MAC * header here into something that can be passed up to raw * consumers. The header goes from "cooked" mode to raw mode. */ if (!dsp->ds_native) { newmp = mac_header_uncook(dsp->ds_mh, mp); if (newmp == NULL) { freemsg(mp); goto next; } mp = newmp; } /* * Strip the VLAN tag for VLAN streams. */ if (is_ethernet && mac_client_vid(dsp->ds_mch) != VLAN_ID_NONE) { newmp = i_dld_ether_header_strip_tag(mp); if (newmp == NULL) { freemsg(mp); goto next; } mp = newmp; } /* * Pass the packet on. */ if (canputnext(dsp->ds_rq)) putnext(dsp->ds_rq, mp); else freemsg(mp); next: /* * Move on to the next packet in the chain. */ mp = next; } while (mp != NULL); } /* * Fast-path receive function. */ /*ARGSUSED*/ void dld_str_rx_fastpath(void *arg, mac_resource_handle_t mrh, mblk_t *mp, mac_header_info_t *mhip) { dld_str_t *dsp = (dld_str_t *)arg; mblk_t *next; size_t offset = 0; /* * MAC header stripping rules: * - Tagged packets: * a. VLAN streams. Strip the whole VLAN header including the tag. * b. Physical streams * - VLAN packets (non-zero VID). The stream must be either a * DL_PROMISC_SAP listener or a ETHERTYPE_VLAN listener. * Strip the Ethernet header but keep the VLAN header. * - Special tagged packets (zero VID) * * The stream is either a DL_PROMISC_SAP listener or a * ETHERTYPE_VLAN listener, strip the Ethernet header but * keep the VLAN header. * * Otherwise, strip the whole VLAN header. * - Untagged packets. Strip the whole MAC header. */ if (mhip->mhi_istagged && (mac_client_vid(dsp->ds_mch) == VLAN_ID_NONE) && ((dsp->ds_sap == ETHERTYPE_VLAN) || (dsp->ds_promisc & DLS_PROMISC_SAP))) { offset = VLAN_TAGSZ; } ASSERT(mp != NULL); do { /* * Get the pointer to the next packet in the chain and then * clear b_next before the packet gets passed on. */ next = mp->b_next; mp->b_next = NULL; /* * Wind back b_rptr to point at the VLAN header. */ ASSERT(mp->b_rptr >= DB_BASE(mp) + offset); mp->b_rptr -= offset; /* * Pass the packet on. */ if (canputnext(dsp->ds_rq)) putnext(dsp->ds_rq, mp); else freemsg(mp); /* * Move on to the next packet in the chain. */ mp = next; } while (mp != NULL); } /* * Default receive function (send DL_UNITDATA_IND messages). */ /*ARGSUSED*/ void dld_str_rx_unitdata(void *arg, mac_resource_handle_t mrh, mblk_t *mp, mac_header_info_t *mhip) { dld_str_t *dsp = (dld_str_t *)arg; mblk_t *ud_mp; mblk_t *next; size_t offset = 0; boolean_t strip_vlan = B_TRUE; /* * See MAC header stripping rules in the dld_str_rx_fastpath() function. */ if (mhip->mhi_istagged && (mac_client_vid(dsp->ds_mch) == VLAN_ID_NONE) && ((dsp->ds_sap == ETHERTYPE_VLAN) || (dsp->ds_promisc & DLS_PROMISC_SAP))) { offset = VLAN_TAGSZ; strip_vlan = B_FALSE; } ASSERT(mp != NULL); do { /* * Get the pointer to the next packet in the chain and then * clear b_next before the packet gets passed on. */ next = mp->b_next; mp->b_next = NULL; /* * Wind back b_rptr to point at the MAC header. */ ASSERT(mp->b_rptr >= DB_BASE(mp) + mhip->mhi_hdrsize); mp->b_rptr -= mhip->mhi_hdrsize; /* * Create the DL_UNITDATA_IND M_PROTO. */ if ((ud_mp = str_unitdata_ind(dsp, mp, strip_vlan)) == NULL) { freemsgchain(mp); return; } /* * Advance b_rptr to point at the payload (or the VLAN header). */ mp->b_rptr += (mhip->mhi_hdrsize - offset); /* * Prepend the DL_UNITDATA_IND. */ ud_mp->b_cont = mp; /* * Send the message. */ if (canputnext(dsp->ds_rq)) putnext(dsp->ds_rq, ud_mp); else freemsg(ud_mp); /* * Move on to the next packet in the chain. */ mp = next; } while (mp != NULL); } /* * DL_NOTIFY_IND: DL_NOTE_SDU_SIZE */ static void str_notify_sdu_size(dld_str_t *dsp, uint_t max_sdu) { mblk_t *mp; dl_notify_ind_t *dlip; if (!(dsp->ds_notifications & DL_NOTE_SDU_SIZE)) return; if ((mp = mexchange(dsp->ds_wq, NULL, sizeof (dl_notify_ind_t), M_PROTO, 0)) == NULL) return; bzero(mp->b_rptr, sizeof (dl_notify_ind_t)); dlip = (dl_notify_ind_t *)mp->b_rptr; dlip->dl_primitive = DL_NOTIFY_IND; dlip->dl_notification = DL_NOTE_SDU_SIZE; dlip->dl_data = max_sdu; qreply(dsp->ds_wq, mp); } /* * Generate DL_NOTIFY_IND messages to notify the DLPI consumer of the * current state of the interface. */ void dld_str_notify_ind(dld_str_t *dsp) { mac_notify_type_t type; for (type = 0; type < MAC_NNOTE; type++) str_notify(dsp, type); } typedef struct dl_unitdata_ind_wrapper { dl_unitdata_ind_t dl_unitdata; uint8_t dl_dest_addr[MAXMACADDRLEN + sizeof (uint16_t)]; uint8_t dl_src_addr[MAXMACADDRLEN + sizeof (uint16_t)]; } dl_unitdata_ind_wrapper_t; /* * Create a DL_UNITDATA_IND M_PROTO message. */ static mblk_t * str_unitdata_ind(dld_str_t *dsp, mblk_t *mp, boolean_t strip_vlan) { mblk_t *nmp; dl_unitdata_ind_wrapper_t *dlwp; dl_unitdata_ind_t *dlp; mac_header_info_t mhi; uint_t addr_length; uint8_t *daddr; uint8_t *saddr; /* * Get the packet header information. */ if (dls_link_header_info(dsp->ds_dlp, mp, &mhi) != 0) return (NULL); /* * Allocate a message large enough to contain the wrapper structure * defined above. */ if ((nmp = mexchange(dsp->ds_wq, NULL, sizeof (dl_unitdata_ind_wrapper_t), M_PROTO, DL_UNITDATA_IND)) == NULL) return (NULL); dlwp = (dl_unitdata_ind_wrapper_t *)nmp->b_rptr; dlp = &(dlwp->dl_unitdata); ASSERT(dlp == (dl_unitdata_ind_t *)nmp->b_rptr); ASSERT(dlp->dl_primitive == DL_UNITDATA_IND); /* * Copy in the destination address. */ addr_length = dsp->ds_mip->mi_addr_length; daddr = dlwp->dl_dest_addr; dlp->dl_dest_addr_offset = (uintptr_t)daddr - (uintptr_t)dlp; bcopy(mhi.mhi_daddr, daddr, addr_length); /* * Set the destination DLSAP to the SAP value encoded in the packet. */ if (mhi.mhi_istagged && !strip_vlan) *(uint16_t *)(daddr + addr_length) = ETHERTYPE_VLAN; else *(uint16_t *)(daddr + addr_length) = mhi.mhi_bindsap; dlp->dl_dest_addr_length = addr_length + sizeof (uint16_t); /* * If the destination address was multicast or broadcast then the * dl_group_address field should be non-zero. */ dlp->dl_group_address = (mhi.mhi_dsttype == MAC_ADDRTYPE_MULTICAST) || (mhi.mhi_dsttype == MAC_ADDRTYPE_BROADCAST); /* * Copy in the source address if one exists. Some MAC types (DL_IB * for example) may not have access to source information. */ if (mhi.mhi_saddr == NULL) { dlp->dl_src_addr_offset = dlp->dl_src_addr_length = 0; } else { saddr = dlwp->dl_src_addr; dlp->dl_src_addr_offset = (uintptr_t)saddr - (uintptr_t)dlp; bcopy(mhi.mhi_saddr, saddr, addr_length); /* * Set the source DLSAP to the packet ethertype. */ *(uint16_t *)(saddr + addr_length) = mhi.mhi_origsap; dlp->dl_src_addr_length = addr_length + sizeof (uint16_t); } return (nmp); } /* * DL_NOTIFY_IND: DL_NOTE_PROMISC_ON_PHYS */ static void str_notify_promisc_on_phys(dld_str_t *dsp) { mblk_t *mp; dl_notify_ind_t *dlip; if (!(dsp->ds_notifications & DL_NOTE_PROMISC_ON_PHYS)) return; if ((mp = mexchange(dsp->ds_wq, NULL, sizeof (dl_notify_ind_t), M_PROTO, 0)) == NULL) return; bzero(mp->b_rptr, sizeof (dl_notify_ind_t)); dlip = (dl_notify_ind_t *)mp->b_rptr; dlip->dl_primitive = DL_NOTIFY_IND; dlip->dl_notification = DL_NOTE_PROMISC_ON_PHYS; qreply(dsp->ds_wq, mp); } /* * DL_NOTIFY_IND: DL_NOTE_PROMISC_OFF_PHYS */ static void str_notify_promisc_off_phys(dld_str_t *dsp) { mblk_t *mp; dl_notify_ind_t *dlip; if (!(dsp->ds_notifications & DL_NOTE_PROMISC_OFF_PHYS)) return; if ((mp = mexchange(dsp->ds_wq, NULL, sizeof (dl_notify_ind_t), M_PROTO, 0)) == NULL) return; bzero(mp->b_rptr, sizeof (dl_notify_ind_t)); dlip = (dl_notify_ind_t *)mp->b_rptr; dlip->dl_primitive = DL_NOTIFY_IND; dlip->dl_notification = DL_NOTE_PROMISC_OFF_PHYS; qreply(dsp->ds_wq, mp); } /* * DL_NOTIFY_IND: DL_NOTE_PHYS_ADDR */ static void str_notify_phys_addr(dld_str_t *dsp, const uint8_t *addr) { mblk_t *mp; dl_notify_ind_t *dlip; uint_t addr_length; uint16_t ethertype; if (!(dsp->ds_notifications & DL_NOTE_PHYS_ADDR)) return; addr_length = dsp->ds_mip->mi_addr_length; if ((mp = mexchange(dsp->ds_wq, NULL, sizeof (dl_notify_ind_t) + addr_length + sizeof (uint16_t), M_PROTO, 0)) == NULL) return; bzero(mp->b_rptr, sizeof (dl_notify_ind_t)); dlip = (dl_notify_ind_t *)mp->b_rptr; dlip->dl_primitive = DL_NOTIFY_IND; dlip->dl_notification = DL_NOTE_PHYS_ADDR; dlip->dl_data = DL_CURR_PHYS_ADDR; dlip->dl_addr_offset = sizeof (dl_notify_ind_t); dlip->dl_addr_length = addr_length + sizeof (uint16_t); bcopy(addr, &dlip[1], addr_length); ethertype = (dsp->ds_sap < ETHERTYPE_802_MIN) ? 0 : dsp->ds_sap; *(uint16_t *)((uchar_t *)(dlip + 1) + addr_length) = ethertype; qreply(dsp->ds_wq, mp); } /* * DL_NOTIFY_IND: DL_NOTE_LINK_UP */ static void str_notify_link_up(dld_str_t *dsp) { mblk_t *mp; dl_notify_ind_t *dlip; if (!(dsp->ds_notifications & DL_NOTE_LINK_UP)) return; if ((mp = mexchange(dsp->ds_wq, NULL, sizeof (dl_notify_ind_t), M_PROTO, 0)) == NULL) return; bzero(mp->b_rptr, sizeof (dl_notify_ind_t)); dlip = (dl_notify_ind_t *)mp->b_rptr; dlip->dl_primitive = DL_NOTIFY_IND; dlip->dl_notification = DL_NOTE_LINK_UP; qreply(dsp->ds_wq, mp); } /* * DL_NOTIFY_IND: DL_NOTE_LINK_DOWN */ static void str_notify_link_down(dld_str_t *dsp) { mblk_t *mp; dl_notify_ind_t *dlip; if (!(dsp->ds_notifications & DL_NOTE_LINK_DOWN)) return; if ((mp = mexchange(dsp->ds_wq, NULL, sizeof (dl_notify_ind_t), M_PROTO, 0)) == NULL) return; bzero(mp->b_rptr, sizeof (dl_notify_ind_t)); dlip = (dl_notify_ind_t *)mp->b_rptr; dlip->dl_primitive = DL_NOTIFY_IND; dlip->dl_notification = DL_NOTE_LINK_DOWN; qreply(dsp->ds_wq, mp); } /* * DL_NOTIFY_IND: DL_NOTE_SPEED */ static void str_notify_speed(dld_str_t *dsp, uint32_t speed) { mblk_t *mp; dl_notify_ind_t *dlip; if (!(dsp->ds_notifications & DL_NOTE_SPEED)) return; if ((mp = mexchange(dsp->ds_wq, NULL, sizeof (dl_notify_ind_t), M_PROTO, 0)) == NULL) return; bzero(mp->b_rptr, sizeof (dl_notify_ind_t)); dlip = (dl_notify_ind_t *)mp->b_rptr; dlip->dl_primitive = DL_NOTIFY_IND; dlip->dl_notification = DL_NOTE_SPEED; dlip->dl_data = speed; qreply(dsp->ds_wq, mp); } /* * DL_NOTIFY_IND: DL_NOTE_CAPAB_RENEG */ static void str_notify_capab_reneg(dld_str_t *dsp) { mblk_t *mp; dl_notify_ind_t *dlip; if (!(dsp->ds_notifications & DL_NOTE_CAPAB_RENEG)) return; if ((mp = mexchange(dsp->ds_wq, NULL, sizeof (dl_notify_ind_t), M_PROTO, 0)) == NULL) return; bzero(mp->b_rptr, sizeof (dl_notify_ind_t)); dlip = (dl_notify_ind_t *)mp->b_rptr; dlip->dl_primitive = DL_NOTIFY_IND; dlip->dl_notification = DL_NOTE_CAPAB_RENEG; qreply(dsp->ds_wq, mp); } /* * DL_NOTIFY_IND: DL_NOTE_FASTPATH_FLUSH */ static void str_notify_fastpath_flush(dld_str_t *dsp) { mblk_t *mp; dl_notify_ind_t *dlip; if (!(dsp->ds_notifications & DL_NOTE_FASTPATH_FLUSH)) return; if ((mp = mexchange(dsp->ds_wq, NULL, sizeof (dl_notify_ind_t), M_PROTO, 0)) == NULL) return; bzero(mp->b_rptr, sizeof (dl_notify_ind_t)); dlip = (dl_notify_ind_t *)mp->b_rptr; dlip->dl_primitive = DL_NOTIFY_IND; dlip->dl_notification = DL_NOTE_FASTPATH_FLUSH; qreply(dsp->ds_wq, mp); } /* * MAC notification callback. */ void str_notify(void *arg, mac_notify_type_t type) { dld_str_t *dsp = (dld_str_t *)arg; queue_t *q = dsp->ds_wq; mac_handle_t mh = dsp->ds_mh; mac_client_handle_t mch = dsp->ds_mch; uint8_t addr[MAXMACADDRLEN]; switch (type) { case MAC_NOTE_TX: qenable(q); break; case MAC_NOTE_DEVPROMISC: /* * Send the appropriate DL_NOTIFY_IND. */ if (mac_promisc_get(mh, MAC_DEVPROMISC)) str_notify_promisc_on_phys(dsp); else str_notify_promisc_off_phys(dsp); break; case MAC_NOTE_UNICST: /* * This notification is sent whenever the MAC unicast * address changes. */ mac_unicast_primary_get(mh, addr); /* * Send the appropriate DL_NOTIFY_IND. */ str_notify_phys_addr(dsp, addr); break; case MAC_NOTE_LINK: /* * This notification is sent every time the MAC driver * updates the link state. */ switch (mac_client_stat_get(mch, MAC_STAT_LINK_STATE)) { case LINK_STATE_UP: { uint64_t speed; /* * The link is up so send the appropriate * DL_NOTIFY_IND. */ str_notify_link_up(dsp); speed = mac_stat_get(mh, MAC_STAT_IFSPEED); str_notify_speed(dsp, (uint32_t)(speed / 1000ull)); break; } case LINK_STATE_DOWN: /* * The link is down so send the appropriate * DL_NOTIFY_IND. */ str_notify_link_down(dsp); break; default: break; } break; case MAC_NOTE_RESOURCE: case MAC_NOTE_CAPAB_CHG: /* * This notification is sent whenever the MAC resources * change or capabilities change. We need to renegotiate * the capabilities. Send the appropriate DL_NOTIFY_IND. */ str_notify_capab_reneg(dsp); break; case MAC_NOTE_SDU_SIZE: { uint_t max_sdu; mac_sdu_get(dsp->ds_mh, NULL, &max_sdu); str_notify_sdu_size(dsp, max_sdu); break; } case MAC_NOTE_FASTPATH_FLUSH: str_notify_fastpath_flush(dsp); break; case MAC_NOTE_MARGIN: break; case MAC_NOTE_PROMISC: break; default: ASSERT(B_FALSE); break; } } /* * This function is called via a taskq mechansim to process all control * messages on a per 'dsp' end point. */ static void dld_wput_nondata_task(void *arg) { dld_str_t *dsp = arg; mblk_t *mp; mutex_enter(&dsp->ds_lock); while (dsp->ds_pending_head != NULL) { mp = dsp->ds_pending_head; dsp->ds_pending_head = mp->b_next; mp->b_next = NULL; if (dsp->ds_pending_head == NULL) dsp->ds_pending_tail = NULL; mutex_exit(&dsp->ds_lock); switch (DB_TYPE(mp)) { case M_PROTO: case M_PCPROTO: dld_proto(dsp, mp); break; case M_IOCTL: dld_ioc(dsp, mp); break; default: ASSERT(0); } mutex_enter(&dsp->ds_lock); } ASSERT(dsp->ds_pending_tail == NULL); dsp->ds_dlpi_pending = 0; cv_broadcast(&dsp->ds_dlpi_pending_cv); mutex_exit(&dsp->ds_lock); } /* * Kernel thread to handle taskq dispatch failures in dld_wput_data. This * thread is started at boot time. */ static void dld_taskq_dispatch(void) { callb_cpr_t cprinfo; dld_str_t *dsp; CALLB_CPR_INIT(&cprinfo, &dld_taskq_lock, callb_generic_cpr, "dld_taskq_dispatch"); mutex_enter(&dld_taskq_lock); while (!dld_taskq_quit) { dsp = list_head(&dld_taskq_list); while (dsp != NULL) { list_remove(&dld_taskq_list, dsp); mutex_exit(&dld_taskq_lock); VERIFY(taskq_dispatch(dld_taskq, dld_wput_nondata_task, dsp, TQ_SLEEP) != 0); mutex_enter(&dld_taskq_lock); dsp = list_head(&dld_taskq_list); } CALLB_CPR_SAFE_BEGIN(&cprinfo); cv_wait(&dld_taskq_cv, &dld_taskq_lock); CALLB_CPR_SAFE_END(&cprinfo, &dld_taskq_lock); } dld_taskq_done = B_TRUE; cv_signal(&dld_taskq_cv); CALLB_CPR_EXIT(&cprinfo); thread_exit(); } /* * All control operations are serialized on the 'dsp' and are also funneled * through a taskq mechanism to ensure that subsequent processing has kernel * context and can safely use cv_wait. * * Mechanisms to handle taskq dispatch failures * * The only way to be sure that taskq dispatch does not fail is to either * specify TQ_SLEEP or to use a static taskq and prepopulate it with * some number of entries and make sure that the number of outstanding requests * are less than that number. We can't use TQ_SLEEP since we don't know the * context. Nor can we bound the total number of 'dsp' end points. So we are * unable to use either of the above schemes, and are forced to deal with * taskq dispatch failures. Note that even dynamic taskq could fail in * dispatch if TQ_NOSLEEP is specified, since this flag is translated * eventually to KM_NOSLEEP and kmem allocations could fail in the taskq * framework. * * We maintain a queue of 'dsp's that encountered taskq dispatch failure. * We also have a single global thread to retry the taskq dispatch. This * thread loops in 'dld_taskq_dispatch' and retries the taskq dispatch, but * uses TQ_SLEEP to ensure eventual success of the dispatch operation. */ static void dld_wput_nondata(dld_str_t *dsp, mblk_t *mp) { ASSERT(mp->b_next == NULL); mutex_enter(&dsp->ds_lock); if (dsp->ds_pending_head != NULL) { ASSERT(dsp->ds_dlpi_pending); dsp->ds_pending_tail->b_next = mp; dsp->ds_pending_tail = mp; mutex_exit(&dsp->ds_lock); return; } ASSERT(dsp->ds_pending_tail == NULL); dsp->ds_pending_head = dsp->ds_pending_tail = mp; /* * At this point if ds_dlpi_pending is set, it implies that the taskq * thread is still active and is processing the last message, though * the pending queue has been emptied. */ if (dsp->ds_dlpi_pending) { mutex_exit(&dsp->ds_lock); return; } dsp->ds_dlpi_pending = 1; mutex_exit(&dsp->ds_lock); if (taskq_dispatch(dld_taskq, dld_wput_nondata_task, dsp, TQ_NOSLEEP) != 0) return; mutex_enter(&dld_taskq_lock); list_insert_tail(&dld_taskq_list, dsp); cv_signal(&dld_taskq_cv); mutex_exit(&dld_taskq_lock); } /* * Process an M_IOCTL message. */ static void dld_ioc(dld_str_t *dsp, mblk_t *mp) { uint_t cmd; cmd = ((struct iocblk *)mp->b_rptr)->ioc_cmd; ASSERT(dsp->ds_type == DLD_DLPI); switch (cmd) { case DLIOCNATIVE: ioc_native(dsp, mp); break; case DLIOCMARGININFO: ioc_margin(dsp, mp); break; case DLIOCRAW: ioc_raw(dsp, mp); break; case DLIOCHDRINFO: ioc_fast(dsp, mp); break; default: ioc(dsp, mp); } } /* * DLIOCNATIVE */ static void ioc_native(dld_str_t *dsp, mblk_t *mp) { queue_t *q = dsp->ds_wq; const mac_info_t *mip = dsp->ds_mip; /* * Native mode can be enabled if it's disabled and if the * native media type is different. */ if (!dsp->ds_native && mip->mi_media != mip->mi_nativemedia) dsp->ds_native = B_TRUE; if (dsp->ds_native) miocack(q, mp, 0, mip->mi_nativemedia); else miocnak(q, mp, 0, ENOTSUP); } /* * DLIOCMARGININFO */ static void ioc_margin(dld_str_t *dsp, mblk_t *mp) { queue_t *q = dsp->ds_wq; uint32_t margin; int err; if (dsp->ds_dlstate == DL_UNATTACHED) { err = EINVAL; goto failed; } if ((err = miocpullup(mp, sizeof (uint32_t))) != 0) goto failed; mac_margin_get(dsp->ds_mh, &margin); *((uint32_t *)mp->b_cont->b_rptr) = margin; miocack(q, mp, sizeof (uint32_t), 0); return; failed: miocnak(q, mp, 0, err); } /* * DLIOCRAW */ static void ioc_raw(dld_str_t *dsp, mblk_t *mp) { queue_t *q = dsp->ds_wq; mac_perim_handle_t mph; if (dsp->ds_mh == NULL) { dsp->ds_mode = DLD_RAW; miocack(q, mp, 0, 0); return; } mac_perim_enter_by_mh(dsp->ds_mh, &mph); if (dsp->ds_polling || dsp->ds_direct) { mac_perim_exit(mph); miocnak(q, mp, 0, EPROTO); return; } if (dsp->ds_mode != DLD_RAW && dsp->ds_dlstate == DL_IDLE) { /* * Set the receive callback. */ dls_rx_set(dsp, dld_str_rx_raw, dsp); } /* * Note that raw mode is enabled. */ dsp->ds_mode = DLD_RAW; mac_perim_exit(mph); miocack(q, mp, 0, 0); } /* * DLIOCHDRINFO */ static void ioc_fast(dld_str_t *dsp, mblk_t *mp) { dl_unitdata_req_t *dlp; off_t off; size_t len; const uint8_t *addr; uint16_t sap; mblk_t *nmp; mblk_t *hmp; uint_t addr_length; queue_t *q = dsp->ds_wq; int err; mac_perim_handle_t mph; if (dld_opt & DLD_OPT_NO_FASTPATH) { err = ENOTSUP; goto failed; } /* * DLIOCHDRINFO should only come from IP. The one initiated from * user-land should not be allowed. */ if (((struct iocblk *)mp->b_rptr)->ioc_cr != kcred) { err = EINVAL; goto failed; } nmp = mp->b_cont; if (nmp == NULL || MBLKL(nmp) < sizeof (dl_unitdata_req_t) || (dlp = (dl_unitdata_req_t *)nmp->b_rptr, dlp->dl_primitive != DL_UNITDATA_REQ)) { err = EINVAL; goto failed; } off = dlp->dl_dest_addr_offset; len = dlp->dl_dest_addr_length; if (!MBLKIN(nmp, off, len)) { err = EINVAL; goto failed; } if (dsp->ds_dlstate != DL_IDLE) { err = ENOTSUP; goto failed; } addr_length = dsp->ds_mip->mi_addr_length; if (len != addr_length + sizeof (uint16_t)) { err = EINVAL; goto failed; } addr = nmp->b_rptr + off; sap = *(uint16_t *)(nmp->b_rptr + off + addr_length); if ((hmp = dls_header(dsp, addr, sap, 0, NULL)) == NULL) { err = ENOMEM; goto failed; } /* * This ioctl might happen concurrently with a direct call to dld_capab * that tries to enable direct and/or poll capabilities. Since the * stack does not serialize them, we do so here to avoid mixing * the callbacks. */ mac_perim_enter_by_mh(dsp->ds_mh, &mph); if (dsp->ds_mode != DLD_FASTPATH) { /* * Set the receive callback (unless polling is enabled). */ if (!dsp->ds_polling && !dsp->ds_direct) dls_rx_set(dsp, dld_str_rx_fastpath, dsp); /* * Note that fast-path mode is enabled. */ dsp->ds_mode = DLD_FASTPATH; } mac_perim_exit(mph); freemsg(nmp->b_cont); nmp->b_cont = hmp; miocack(q, mp, MBLKL(nmp) + MBLKL(hmp), 0); return; failed: miocnak(q, mp, 0, err); } /* * Catch-all handler. */ static void ioc(dld_str_t *dsp, mblk_t *mp) { queue_t *q = dsp->ds_wq; if (dsp->ds_dlstate == DL_UNATTACHED) { miocnak(q, mp, 0, EINVAL); return; } mac_ioctl(dsp->ds_mh, q, mp); }