xref: /illumos-gate/usr/src/uts/common/io/mac/mac_util.c (revision fe231ea6f3cdffee825d2e92e1a4639b3bc796b7)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright 2019 Joyent, Inc.
 * Copyright 2026 Oxide Computer Company
 */

/*
 * MAC Services Module - misc utilities
 */

#include <sys/types.h>
#include <sys/mac.h>
#include <sys/mac_impl.h>
#include <sys/mac_client_priv.h>
#include <sys/mac_client_impl.h>
#include <sys/mac_soft_ring.h>
#include <sys/strsubr.h>
#include <sys/strsun.h>
#include <sys/vlan.h>
#include <sys/pattr.h>
#include <sys/pci_tools.h>
#include <inet/ip.h>
#include <inet/ip_impl.h>
#include <inet/ip6.h>
#include <sys/vtrace.h>
#include <sys/dlpi.h>
#include <sys/sunndi.h>
#include <inet/ipsec_impl.h>
#include <inet/sadb.h>
#include <inet/ipsecesp.h>
#include <inet/ipsecah.h>
#include <inet/tcp.h>
#include <inet/sctp_ip.h>

/*
 * The next two functions are used for dropping packets or chains of
 * packets, respectively. We could use one function for both but
 * separating the use cases allows us to specify intent and prevent
 * dropping more data than intended.
 *
 * The purpose of these functions is to aid the debugging effort,
 * especially in production. Rather than use freemsg()/freemsgchain(),
 * it's preferable to use these functions when dropping a packet in
 * the MAC layer. These functions should only be used during
 * unexpected conditions. That is, any time a packet is dropped
 * outside of the regular, successful datapath. Consolidating all
 * drops on these functions allows the user to trace one location and
 * determine why the packet was dropped based on the msg. It also
 * allows the user to inspect the packet before it is freed. Finally,
 * it allows the user to avoid tracing freemsg()/freemsgchain() thus
 * keeping the hot path running as efficiently as possible.
 *
 * NOTE: At this time not all MAC drops are aggregated on these
 * functions; but that is the plan. This comment should be erased once
 * completed.
 */

/*PRINTFLIKE2*/
void
mac_drop_pkt(mblk_t *mp, const char *fmt, ...)
{
	va_list adx;
	char msg[128];
	char *msgp = msg;

	ASSERT3P(mp->b_next, ==, NULL);

	va_start(adx, fmt);
	(void) vsnprintf(msgp, sizeof (msg), fmt, adx);
	va_end(adx);

	DTRACE_PROBE2(mac__drop, mblk_t *, mp, char *, msgp);
	freemsg(mp);
}

/*PRINTFLIKE2*/
void
mac_drop_chain(mblk_t *chain, const char *fmt, ...)
{
	va_list adx;
	char msg[128];
	char *msgp = msg;

	va_start(adx, fmt);
	(void) vsnprintf(msgp, sizeof (msg), fmt, adx);
	va_end(adx);

	/*
	 * We could use freemsgchain() for the actual freeing but
	 * since we are already walking the chain to fire the dtrace
	 * probe we might as well free the msg here too.
	 */
	for (mblk_t *mp = chain, *next; mp != NULL; ) {
		next = mp->b_next;
		DTRACE_PROBE2(mac__drop, mblk_t *, mp, char *, msgp);
		mp->b_next = NULL;
		freemsg(mp);
		mp = next;
	}
}

/*
 * Copy an mblk, preserving its hardware checksum flags.
 */
static mblk_t *
mac_copymsg_cksum(mblk_t *mp)
{
	mblk_t *mp1;

	mp1 = copymsg(mp);
	if (mp1 == NULL)
		return (NULL);

	mac_hcksum_clone(mp, mp1);

	return (mp1);
}

/*
 * Copy an mblk chain, presenting the hardware checksum flags of the
 * individual mblks.
 */
mblk_t *
mac_copymsgchain_cksum(mblk_t *mp)
{
	mblk_t *nmp = NULL;
	mblk_t **nmpp = &nmp;

	for (; mp != NULL; mp = mp->b_next) {
		if ((*nmpp = mac_copymsg_cksum(mp)) == NULL) {
			freemsgchain(nmp);
			return (NULL);
		}

		nmpp = &((*nmpp)->b_next);
	}

	return (nmp);
}

/*
 * Perform software checksum on a single message, if needed. The emulation
 * performed is determined by an intersection of the mblk's flags and the emul
 * flags requested. The emul flags are documented in mac.h.
 */
static mblk_t *
mac_sw_cksum(mblk_t *mp, mac_emul_t emul)
{
	mac_ether_offload_info_t meoi = { 0 };
	const char *err = "";

	/*
	 * The only current caller is mac_hw_emul(), which handles any chaining
	 * of mblks prior to now.
	 */
	VERIFY3P(mp->b_next, ==, NULL);

	uint32_t flags = DB_CKSUMFLAGS(mp);

	/* Why call this if checksum emulation isn't needed? */
	ASSERT3U(flags & (HCK_FLAGS), !=, 0);
	/* But also, requesting both ULP cksum types is improper */
	if ((flags & HCK_FULLCKSUM) != 0 && (flags & HCK_PARTIALCKSUM) != 0) {
		err = "full and partial ULP cksum requested";
		goto bail;
	}

	const boolean_t do_v4_cksum = (emul & MAC_IPCKSUM_EMUL) != 0 &&
	    (flags & HCK_IPV4_HDRCKSUM) != 0;
	const boolean_t do_ulp_cksum = (emul & MAC_HWCKSUM_EMUL) != 0 &&
	    (flags & (HCK_FULLCKSUM | HCK_PARTIALCKSUM)) != 0;
	const boolean_t ulp_prefer_partial = (flags & HCK_PARTIALCKSUM) != 0;

	mac_ether_offload_info(mp, &meoi);
	if ((meoi.meoi_flags & MEOI_L2INFO_SET) == 0 ||
	    (meoi.meoi_l3proto != ETHERTYPE_IP &&
	    meoi.meoi_l3proto != ETHERTYPE_IPV6)) {
		/* Non-IP traffic (like ARP) is left alone */
		return (mp);
	}

	/*
	 * Ensure that requested checksum type(s) are supported by the
	 * protocols encoded in the packet headers.
	 */
	if (do_v4_cksum) {
		if (meoi.meoi_l3proto != ETHERTYPE_IP) {
			err = "IPv4 csum requested on non-IPv4 packet";
			goto bail;
		}
	}
	if (do_ulp_cksum) {
		if ((meoi.meoi_flags & MEOI_L4INFO_SET) == 0) {
			err = "missing ULP header";
			goto bail;
		}
		switch (meoi.meoi_l4proto) {
		case IPPROTO_TCP:
		case IPPROTO_UDP:
		case IPPROTO_ICMP:
		case IPPROTO_ICMPV6:
		case IPPROTO_SCTP:
			break;
		default:
			err = "unexpected ULP";
			goto bail;
		}
	}

	/*
	 * If the first mblk of this packet contains only the Ethernet header,
	 * skip past it for now. Packets with their data contained in only a
	 * single mblk can then use the fastpaths tuned to that possibility.
	 */
	mblk_t *skipped_hdr = NULL;
	if (MBLKL(mp) == meoi.meoi_l2hlen) {
		meoi.meoi_len -= meoi.meoi_l2hlen;
		meoi.meoi_l2hlen = 0;
		skipped_hdr = mp;
		mp = mp->b_cont;

		ASSERT(mp != NULL);
	}

	/*
	 * Ensure that all of the headers we need to access are:
	 * 1. Collected in the first mblk
	 * 2. Held in a data-block which is safe for us to modify
	 *    (It must have a refcount of 1)
	 * 3. IP headers are 4-byte aligned. IP header size is always a multiple
	 *    of 4 bytes, thus L4 headers will also be safe to access.
	 */
	const size_t hdr_len_reqd = (meoi.meoi_l2hlen + meoi.meoi_l3hlen) +
	    (do_ulp_cksum ? meoi.meoi_l4hlen : 0);
	if (MBLKL(mp) < hdr_len_reqd || DB_REF(mp) > 1 ||
	    !OK_32PTR(mp->b_rptr + meoi.meoi_l2hlen)) {
		const size_t pad_by = (4 - (meoi.meoi_l2hlen % 4)) % 4;
		mblk_t *hdrmp = msgpullup_pad(mp, hdr_len_reqd, pad_by);

		if (hdrmp == NULL) {
			err = "could not pullup msg headers";
			goto bail;
		}

		mac_hcksum_clone(mp, hdrmp);
		if (skipped_hdr != NULL) {
			ASSERT3P(skipped_hdr->b_cont, ==, mp);
			skipped_hdr->b_cont = hdrmp;
		}
		freemsg(mp);
		mp = hdrmp;
	}

	/* Calculate IPv4 header checksum, if requested */
	if (do_v4_cksum) {
		/*
		 * While unlikely, it's possible to write code that might end up
		 * calling mac_sw_cksum() twice on the same mblk (performing
		 * both LSO and checksum emulation in a single mblk chain loop
		 * -- the LSO emulation inserts a new chain into the existing
		 * chain and then the loop iterates back over the new segments
		 * and emulates the checksum a second time).  Normally this
		 * wouldn't be a problem, because the HCK_*_OK flags are
		 * supposed to indicate that we don't need to do peform the
		 * work. But HCK_IPV4_HDRCKSUM and HCK_IPV4_HDRCKSUM_OK have the
		 * same value; so we cannot use these flags to determine if the
		 * IP header checksum has already been calculated or not. For
		 * this reason, we zero out the the checksum first. In the
		 * future, we should fix the HCK_* flags.
		 */
		ipha_t *ipha = (ipha_t *)(mp->b_rptr + meoi.meoi_l2hlen);
		ipha->ipha_hdr_checksum = 0;
		ipha->ipha_hdr_checksum = (uint16_t)ip_csum_hdr(ipha);
		flags &= ~HCK_IPV4_HDRCKSUM;
		flags |= HCK_IPV4_HDRCKSUM_OK;
	}

	/*
	 * The SCTP is different from all the other protocols in that it uses
	 * CRC32 for its checksum, rather than ones' complement.
	 */
	if (do_ulp_cksum && meoi.meoi_l4proto == IPPROTO_SCTP) {
		if (ulp_prefer_partial) {
			err = "SCTP does not support partial checksum";
			goto bail;
		}

		const uint_t ulp_off = meoi.meoi_l2hlen + meoi.meoi_l3hlen;
		sctp_hdr_t *sctph = (sctp_hdr_t *)(mp->b_rptr + ulp_off);

		sctph->sh_chksum = 0;
		sctph->sh_chksum = sctp_cksum(mp, ulp_off);

		flags &= ~HCK_FULLCKSUM;
		flags |= HCK_FULLCKSUM_OK;
		goto success;
	}

	/* Calculate full ULP checksum, if requested */
	if (do_ulp_cksum && !ulp_prefer_partial) {
		/*
		 * Calculate address and length portions of pseudo-header csum
		 */
		uint32_t cksum = 0;
		if (meoi.meoi_l3proto == ETHERTYPE_IP) {
			const ipha_t *ipha =
			    (const ipha_t *)(mp->b_rptr + meoi.meoi_l2hlen);
			const uint16_t *ipp =
			    (const uint16_t *)(&ipha->ipha_src);

			cksum += ipp[0] + ipp[1] + ipp[2] + ipp[3];

			/*
			 * While it is tempting to calculate the payload length
			 * solely from `meoi`, like as done below for IPv6,
			 * doing so is a trap.  Packets shorter than 60 bytes
			 * will get padded out to that length in order to meet
			 * the minimums for Ethernet.  Instead, we pull the
			 * length from the IP header.
			 */
			const uint16_t payload_len =
			    ntohs(ipha->ipha_length) - meoi.meoi_l3hlen;
			cksum += htons(payload_len);
		} else if (meoi.meoi_l3proto == ETHERTYPE_IPV6) {
			const ip6_t *ip6h =
			    (const ip6_t *)(mp->b_rptr + meoi.meoi_l2hlen);
			const uint16_t *ipp =
			    (const uint16_t *)(&ip6h->ip6_src);

			cksum += ipp[0] + ipp[1] + ipp[2] + ipp[3] +
			    ipp[4] + ipp[5] + ipp[6] + ipp[7];
			cksum += ipp[8] + ipp[9] + ipp[10] + ipp[11] +
			    ipp[12] + ipp[13] + ipp[14] + ipp[15];

			const uint16_t payload_len = meoi.meoi_len -
			    ((uint16_t)meoi.meoi_l2hlen + meoi.meoi_l3hlen);
			cksum += htons(payload_len);
		} else {
			/*
			 * Since we already checked for recognized L3 protocols
			 * earlier, this should not be reachable.
			 */
			panic("L3 protocol unexpectedly changed");
		}

		/* protocol portion of pseudo-header */
		uint_t cksum_off;
		switch (meoi.meoi_l4proto) {
		case IPPROTO_TCP:
			cksum += IP_TCP_CSUM_COMP;
			cksum_off = TCP_CHECKSUM_OFFSET;
			break;
		case IPPROTO_UDP:
			cksum += IP_UDP_CSUM_COMP;
			cksum_off = UDP_CHECKSUM_OFFSET;
			break;
		case IPPROTO_ICMP:
			/* ICMP cksum does not include pseudo-header contents */
			cksum = 0;
			cksum_off = ICMP_CHECKSUM_OFFSET;
			break;
		case IPPROTO_ICMPV6:
			cksum += IP_ICMPV6_CSUM_COMP;
			cksum_off = ICMPV6_CHECKSUM_OFFSET;
			break;
		default:
			err = "unrecognized L4 protocol";
			goto bail;
		}

		/*
		 * With IP_CSUM() taking into account the pseudo-header
		 * checksum, make sure the ULP checksum field is zeroed before
		 * computing the rest;
		 */
		const uint_t l4_off = meoi.meoi_l3hlen + meoi.meoi_l2hlen;
		uint16_t *up = (uint16_t *)(mp->b_rptr + l4_off + cksum_off);
		*up = 0;
		cksum = IP_CSUM(mp, l4_off, cksum);

		if (meoi.meoi_l4proto == IPPROTO_UDP && cksum == 0) {
			/*
			 * A zero checksum is not allowed on UDPv6, and on UDPv4
			 * implies no checksum.  In either case, invert to a
			 * values of all-1s.
			 */
			*up = 0xffff;
		} else {
			*up = cksum;
		}

		flags &= ~HCK_FULLCKSUM;
		flags |= HCK_FULLCKSUM_OK;
		goto success;
	}

	/* Calculate partial ULP checksum, if requested */
	if (do_ulp_cksum && ulp_prefer_partial) {
		uint32_t start, stuff, end, value;
		mac_hcksum_get(mp, &start, &stuff, &end, &value, NULL);

		ASSERT3S(end, >, start);

		/*
		 * The prior size checks against the header length data ensure
		 * that the mblk contains everything through at least the ULP
		 * header, but if the partial checksum (unexpectedly) requests
		 * its result be stored past that, we cannot continue.
		 */
		if (stuff + sizeof (uint16_t) > MBLKL(mp)) {
			err = "partial csum request is out of bounds";
			goto bail;
		}

		uchar_t *ipp = (uchar_t *)(mp->b_rptr + meoi.meoi_l2hlen);
		uint16_t *up = (uint16_t *)(ipp + stuff);

		const uint16_t partial = *up;
		*up = 0;
		const uint16_t cksum =
		    ~IP_CSUM_PARTIAL(mp, start + meoi.meoi_l2hlen, partial);
		*up = cksum != 0 ? cksum : ~cksum;

		flags &= ~HCK_PARTIALCKSUM;
		flags |= HCK_FULLCKSUM_OK;
	}

success:
	/*
	 * With the checksum(s) calculated, store the updated flags to reflect
	 * the current status, and zero out any of the partial-checksum fields
	 * which would be irrelevant now.
	 */
	mac_hcksum_set(mp, 0, 0, 0, 0, flags);

	/* Don't forget to reattach the header. */
	if (skipped_hdr != NULL) {
		ASSERT3P(skipped_hdr->b_cont, ==, mp);

		/*
		 * Duplicate the HCKSUM data into the header mblk.
		 *
		 * This mimics mac_add_vlan_tag() which ensures that both the
		 * first mblk _and_ the first data bearing mblk possess the
		 * HCKSUM information. Consumers like IP will end up discarding
		 * the ether_header mblk, so for now, it is important that the
		 * data be available in both places.
		 */
		mac_hcksum_clone(mp, skipped_hdr);
		mp = skipped_hdr;
	}
	return (mp);

bail:
	if (skipped_hdr != NULL) {
		ASSERT3P(skipped_hdr->b_cont, ==, mp);
		mp = skipped_hdr;
	}

	mac_drop_pkt(mp, err);
	return (NULL);
}

/*
 * Build a single data segment from an LSO packet. The mblk chain
 * returned, seg_head, represents the data segment and is always
 * exactly seg_len bytes long. The lso_mp and offset input/output
 * parameters track our position in the LSO packet. This function
 * exists solely as a helper to mac_sw_lso().
 *
 * Case A
 *
 *     The current lso_mp is larger than the requested seg_len. The
 *     beginning of seg_head may start at the beginning of lso_mp or
 *     offset into it. In either case, a single mblk is returned, and
 *     *offset is updated to reflect our new position in the current
 *     lso_mp.
 *
 *          +----------------------------+
 *          |  in *lso_mp / out *lso_mp  |
 *          +----------------------------+
 *          ^                        ^
 *          |                        |
 *          |                        |
 *          |                        |
 *          +------------------------+
 *          |        seg_head        |
 *          +------------------------+
 *          ^                        ^
 *          |                        |
 *   in *offset = 0        out *offset = seg_len
 *
 *          |------   seg_len    ----|
 *
 *
 *       +------------------------------+
 *       |   in *lso_mp / out *lso_mp   |
 *       +------------------------------+
 *          ^                        ^
 *          |                        |
 *          |                        |
 *          |                        |
 *          +------------------------+
 *          |        seg_head        |
 *          +------------------------+
 *          ^                        ^
 *          |                        |
 *   in *offset = N        out *offset = N + seg_len
 *
 *          |------   seg_len    ----|
 *
 *
 *
 * Case B
 *
 *    The requested seg_len consumes exactly the rest of the lso_mp.
 *    I.e., the seg_head's b_wptr is equivalent to lso_mp's b_wptr.
 *    The seg_head may start at the beginning of the lso_mp or at some
 *    offset into it. In either case we return a single mblk, reset
 *    *offset to zero, and walk to the next lso_mp.
 *
 *          +------------------------+           +------------------------+
 *          |       in *lso_mp       |---------->|      out *lso_mp       |
 *          +------------------------+           +------------------------+
 *          ^                        ^           ^
 *          |                        |           |
 *          |                        |    out *offset = 0
 *          |                        |
 *          +------------------------+
 *          |        seg_head        |
 *          +------------------------+
 *          ^
 *          |
 *   in *offset = 0
 *
 *          |------   seg_len    ----|
 *
 *
 *
 *      +----------------------------+           +------------------------+
 *      |         in *lso_mp         |---------->|      out *lso_mp       |
 *      +----------------------------+           +------------------------+
 *          ^                        ^           ^
 *          |                        |           |
 *          |                        |    out *offset = 0
 *          |                        |
 *          +------------------------+
 *          |        seg_head        |
 *          +------------------------+
 *          ^
 *          |
 *   in *offset = N
 *
 *          |------   seg_len    ----|
 *
 *
 * Case C
 *
 *    The requested seg_len is greater than the current lso_mp. In
 *    this case we must consume LSO mblks until we have enough data to
 *    satisfy either case (A) or (B) above. We will return multiple
 *    mblks linked via b_cont, offset will be set based on the cases
 *    above, and lso_mp will walk forward at least one mblk, but maybe
 *    more.
 *
 *    N.B. This digram is not exhaustive. The seg_head may start on
 *    the beginning of an lso_mp. The seg_tail may end exactly on the
 *    boundary of an lso_mp. And there may be two (in this case the
 *    middle block wouldn't exist), three, or more mblks in the
 *    seg_head chain. This is meant as one example of what might
 *    happen. The main thing to remember is that the seg_tail mblk
 *    must be one of case (A) or (B) above.
 *
 *  +------------------+    +----------------+    +------------------+
 *  |    in *lso_mp    |--->|    *lso_mp     |--->|   out *lso_mp    |
 *  +------------------+    +----------------+    +------------------+
 *        ^            ^    ^                ^    ^            ^
 *        |            |    |                |    |            |
 *        |            |    |                |    |            |
 *        |            |    |                |    |            |
 *        |            |    |                |    |            |
 *        +------------+    +----------------+    +------------+
 *        |  seg_head  |--->|                |--->|  seg_tail  |
 *        +------------+    +----------------+    +------------+
 *        ^                                                    ^
 *        |                                                    |
 *  in *offset = N                          out *offset = MBLKL(seg_tail)
 *
 *        |-------------------   seg_len    -------------------|
 *
 */
static mblk_t *
build_data_seg(mblk_t **lso_mp, uint32_t *offset, uint32_t seg_len)
{
	mblk_t *seg_head, *seg_tail, *seg_mp;

	ASSERT3P(*lso_mp, !=, NULL);
	ASSERT3U((*lso_mp)->b_rptr + *offset, <, (*lso_mp)->b_wptr);

	seg_mp = dupb(*lso_mp);
	if (seg_mp == NULL)
		return (NULL);

	seg_head = seg_mp;
	seg_tail = seg_mp;

	/* Continue where we left off from in the lso_mp. */
	seg_mp->b_rptr += *offset;

last_mblk:
	/* Case (A) */
	if ((seg_mp->b_rptr + seg_len) < seg_mp->b_wptr) {
		*offset += seg_len;
		seg_mp->b_wptr = seg_mp->b_rptr + seg_len;
		return (seg_head);
	}

	/* Case (B) */
	if ((seg_mp->b_rptr + seg_len) == seg_mp->b_wptr) {
		*offset = 0;
		*lso_mp = (*lso_mp)->b_cont;
		return (seg_head);
	}

	/* Case (C) */
	ASSERT3U(seg_mp->b_rptr + seg_len, >, seg_mp->b_wptr);

	/*
	 * The current LSO mblk doesn't have enough data to satisfy
	 * seg_len -- continue peeling off LSO mblks to build the new
	 * segment message. If allocation fails we free the previously
	 * allocated segment mblks and return NULL.
	 */
	while ((seg_mp->b_rptr + seg_len) > seg_mp->b_wptr) {
		ASSERT3U(MBLKL(seg_mp), <=, seg_len);
		seg_len -= MBLKL(seg_mp);
		*offset = 0;
		*lso_mp = (*lso_mp)->b_cont;
		seg_mp = dupb(*lso_mp);

		if (seg_mp == NULL) {
			freemsgchain(seg_head);
			return (NULL);
		}

		seg_tail->b_cont = seg_mp;
		seg_tail = seg_mp;
	}

	/*
	 * We've walked enough LSO mblks that we can now satisfy the
	 * remaining seg_len. At this point we need to jump back to
	 * determine if we have arrived at case (A) or (B).
	 */

	/* Just to be paranoid that we didn't underflow. */
	ASSERT3U(seg_len, <, IP_MAXPACKET);
	ASSERT3U(seg_len, >, 0);
	goto last_mblk;
}

/*
 * Perform software segmentation of a single LSO message. Take an LSO
 * message as input and return head/tail pointers as output. This
 * function should not be invoked directly but instead through
 * mac_hw_emul().
 *
 * The resulting chain is comprised of multiple (nsegs) MSS sized
 * segments. Each segment will consist of two or more mblks joined by
 * b_cont: a header and one or more data mblks. The header mblk is
 * allocated anew for each message. The first segment's header is used
 * as a template for the rest with adjustments made for things such as
 * ID, sequence, length, TCP flags, etc. The data mblks reference into
 * the existing LSO mblk (passed in as omp) by way of dupb(). Their
 * b_rptr/b_wptr values are adjusted to reference only the fraction of
 * the LSO message they are responsible for. At the successful
 * completion of this function the original mblk (omp) is freed,
 * leaving the newely created segment chain as the only remaining
 * reference to the data.
 */
static void
mac_sw_lso(mblk_t *omp, mac_emul_t emul, mblk_t **head, mblk_t **tail,
    uint_t *count)
{
	uint32_t ocsum_flags, ocsum_start, ocsum_stuff;
	uint32_t mss;
	uint32_t oehlen, oiphlen, otcphlen, ohdrslen, opktlen;
	uint32_t odatalen, oleft;
	uint_t nsegs, seg;
	int len;

	const void *oiph;
	const tcph_t *otcph;
	ipha_t *niph;
	tcph_t *ntcph;
	uint16_t ip_id;
	uint32_t tcp_seq, tcp_sum, otcp_sum;

	boolean_t is_v6 = B_FALSE;
	ip6_t *niph6;

	uint32_t offset = 0;
	mblk_t *odatamp;
	mblk_t *seg_chain, *prev_nhdrmp, *next_nhdrmp, *nhdrmp, *ndatamp;
	mblk_t *tmptail;

	mac_ether_offload_info_t meoi = { 0 };

	ASSERT3P(head, !=, NULL);
	ASSERT3P(tail, !=, NULL);
	ASSERT3P(count, !=, NULL);
	ASSERT3U((DB_CKSUMFLAGS(omp) & HW_LSO), !=, 0);

	/* Assume we are dealing with a single LSO message. */
	ASSERT3P(omp->b_next, ==, NULL);

	mac_ether_offload_info(omp, &meoi);
	opktlen = meoi.meoi_len;
	oehlen = meoi.meoi_l2hlen;
	oiphlen = meoi.meoi_l3hlen;
	otcphlen = meoi.meoi_l4hlen;
	ohdrslen = oehlen + oiphlen + otcphlen;

	/* Performing LSO requires that we successfully read fully up to L4 */
	if ((MEOI_L4INFO_SET & meoi.meoi_flags) == 0) {
		mac_drop_pkt(omp, "unable to fully parse packet to L4");
		goto fail;
	}

	if (meoi.meoi_l3proto != ETHERTYPE_IP &&
	    meoi.meoi_l3proto != ETHERTYPE_IPV6) {
		mac_drop_pkt(omp, "LSO'd packet has non-IP L3 header: %x",
		    meoi.meoi_l3proto);
		goto fail;
	}

	if (meoi.meoi_l4proto != IPPROTO_TCP) {
		mac_drop_pkt(omp, "LSO unsupported protocol: %x",
		    meoi.meoi_l4proto);
		goto fail;
	}

	is_v6 = meoi.meoi_l3proto == ETHERTYPE_IPV6;

	mss = DB_LSOMSS(omp);
	if (mss == 0) {
		mac_drop_pkt(omp, "packet misconfigured for LSO (MSS == 0)");
		goto fail;
	}
	ASSERT3U(opktlen, <=, IP_MAXPACKET + oehlen);

	/*
	 * Ensure the headers are contiguous and that L3 and L4 headers are 4B
	 * aligned. The IP header is used only for the benefit of DTrace SDTs,
	 * whereas the TCP header is actively read. This small pullup should
	 * only practically happen when mac_add_vlan_tag is in play, which
	 * prepends a new mblk in front containing the amended Ethernet header.
	 */
	const size_t pad_by = (4 - (meoi.meoi_l2hlen % 4)) % 4;
	if (MBLKL(omp) < ohdrslen || !OK_32PTR(omp->b_rptr + oehlen)) {
		mblk_t *tmp = msgpullup_pad(omp, ohdrslen, pad_by);

		if (tmp == NULL) {
			mac_drop_pkt(omp, "failed to pull up");
			goto fail;
		}

		mac_hcksum_clone(omp, tmp);
		freemsg(omp);
		omp = tmp;
	}

	oiph = (void *)(omp->b_rptr + oehlen);
	otcph = (tcph_t *)(omp->b_rptr + oehlen + oiphlen);

	if (otcph->th_flags[0] & (TH_SYN | TH_RST | TH_URG)) {
		mac_drop_pkt(omp, "LSO packet has SYN|RST|URG set");
		goto fail;
	}

	len = MBLKL(omp);

	/*
	 * Either we have data in the first mblk or it's just the
	 * header. In either case, we need to set rptr to the start of
	 * the TCP data.
	 */
	if (len > ohdrslen) {
		odatamp = omp;
		offset = ohdrslen;
	} else {
		ASSERT3U(len, ==, ohdrslen);
		odatamp = omp->b_cont;
		offset = 0;
	}

	/* Make sure we still have enough data. */
	odatalen = opktlen - ohdrslen;
	ASSERT3U(msgsize(odatamp), >=, odatalen);

	/*
	 * If a MAC negotiated LSO then it must negotiate both
	 * HCKSUM_IPHDRCKSUM and either HCKSUM_INET_FULL_V4 or
	 * HCKSUM_INET_PARTIAL; because both the IP and TCP headers
	 * change during LSO segmentation (only the 3 fields of the
	 * pseudo header checksum don't change: src, dst, proto). Thus
	 * we would expect these flags (HCK_IPV4_HDRCKSUM |
	 * HCK_PARTIALCKSUM | HCK_FULLCKSUM) to be set and for this
	 * function to emulate those checksums in software. However,
	 * that assumes a world where we only expose LSO if the
	 * underlying hardware exposes LSO. Moving forward the plan is
	 * to assume LSO in the upper layers and have MAC perform
	 * software LSO when the underlying provider doesn't support
	 * it. In such a world, if the provider doesn't support LSO
	 * but does support hardware checksum offload, then we could
	 * simply perform the segmentation and allow the hardware to
	 * calculate the checksums. To the hardware it's just another
	 * chain of non-LSO packets.
	 */
	ASSERT3S(DB_TYPE(omp), ==, M_DATA);
	ocsum_flags = DB_CKSUMFLAGS(omp);
	ASSERT3U(ocsum_flags & (HCK_PARTIALCKSUM | HCK_FULLCKSUM), !=, 0);

	/*
	 * If hardware only provides partial checksum then software
	 * must supply the pseudo-header checksum. In the case of LSO
	 * we leave the TCP length at zero to be filled in by
	 * hardware. This function must handle two scenarios.
	 *
	 * 1. Being called by a MAC client on the Rx path to segment
	 *    an LSO packet and calculate the checksum.
	 *
	 * 2. Being called by a MAC provider to segment an LSO packet.
	 *    In this case the LSO segmentation is performed in
	 *    software (by this routine) but the MAC provider should
	 *    still calculate the TCP/IP checksums in hardware.
	 *
	 *  To elaborate on the second case: we cannot have the
	 *  scenario where IP sends LSO packets but the underlying HW
	 *  doesn't support checksum offload -- because in that case
	 *  TCP/IP would calculate the checksum in software (for the
	 *  LSO packet) but then MAC would segment the packet and have
	 *  to redo all the checksum work. So IP should never do LSO
	 *  if HW doesn't support both IP and TCP checksum.
	 */
	if (ocsum_flags & HCK_PARTIALCKSUM) {
		ocsum_start = (uint32_t)DB_CKSUMSTART(omp);
		ocsum_stuff = (uint32_t)DB_CKSUMSTUFF(omp);
	}

	/*
	 * Subtract one to account for the case where the data length
	 * is evenly divisble by the MSS. Add one to account for the
	 * fact that the division will always result in one less
	 * segment than needed.
	 */
	nsegs = ((odatalen - 1) / mss) + 1;
	if (nsegs < 2) {
		mac_drop_pkt(omp, "LSO not enough segs: %u", nsegs);
		goto fail;
	}

	DTRACE_PROBE6(sw__lso__start, mblk_t *, omp, void_ip_t *, oiph,
	    __dtrace_tcp_tcph_t *, otcph, uint_t, odatalen, uint_t, mss,
	    uint_t, nsegs);

	seg_chain = NULL;
	tmptail = seg_chain;
	oleft = odatalen;

	for (uint_t i = 0; i < nsegs; i++) {
		boolean_t last_seg = ((i + 1) == nsegs);
		uint32_t seg_len;

		/*
		 * Ensure that we have 4B L3/L4 alignment for any output frames.
		 * If we fail to allocate, then drop the partially
		 * allocated chain as well as the LSO packet. Let the
		 * sender deal with the fallout.
		 */
		if ((nhdrmp = allocb(pad_by + ohdrslen, 0)) == NULL) {
			freemsgchain(seg_chain);
			mac_drop_pkt(omp, "failed to alloc segment header");
			goto fail;
		}
		ASSERT3P(nhdrmp->b_cont, ==, NULL);

		/* Copy over the header stack. */
		nhdrmp->b_rptr += pad_by;
		nhdrmp->b_wptr = nhdrmp->b_rptr + ohdrslen;
		bcopy(omp->b_rptr, nhdrmp->b_rptr, ohdrslen);

		if (seg_chain == NULL) {
			seg_chain = nhdrmp;
		} else {
			ASSERT3P(tmptail, !=, NULL);
			tmptail->b_next = nhdrmp;
		}

		tmptail = nhdrmp;

		/*
		 * Calculate this segment's length. It's either the MSS
		 * or whatever remains for the last segment.
		 */
		seg_len = last_seg ? oleft : mss;
		ASSERT3U(seg_len, <=, mss);
		ndatamp = build_data_seg(&odatamp, &offset, seg_len);

		if (ndatamp == NULL) {
			freemsgchain(seg_chain);
			mac_drop_pkt(omp, "LSO failed to segment data");
			goto fail;
		}

		/* Attach data mblk to header mblk. */
		nhdrmp->b_cont = ndatamp;
		DB_CKSUMFLAGS(ndatamp) &= ~HW_LSO;
		ASSERT3U(seg_len, <=, oleft);
		oleft -= seg_len;

		/* Setup partial checksum offsets. */
		if (ocsum_flags & HCK_PARTIALCKSUM) {
			DB_CKSUMSTART(nhdrmp) = ocsum_start;
			DB_CKSUMEND(nhdrmp) = oiphlen + otcphlen + seg_len;
			DB_CKSUMSTUFF(nhdrmp) = ocsum_stuff;
		}
	}

	/* We should have consumed entire LSO msg. */
	ASSERT3S(oleft, ==, 0);
	ASSERT3P(odatamp, ==, NULL);

	/*
	 * All seg data mblks are referenced by the header mblks, null
	 * out this pointer to catch any bad derefs.
	 */
	ndatamp = NULL;

	/*
	 * Set headers and checksum for first segment.
	 */
	nhdrmp = seg_chain;
	ASSERT3U(msgsize(nhdrmp->b_cont), ==, mss);

	if (is_v6) {
		niph6 = (ip6_t *)(nhdrmp->b_rptr + oehlen);
		niph6->ip6_plen = htons(
		    (oiphlen - IPV6_HDR_LEN) + otcphlen + mss);
	} else {
		niph = (ipha_t *)(nhdrmp->b_rptr + oehlen);
		niph->ipha_length = htons(oiphlen + otcphlen + mss);
		/*
		 * If the v4 checksum was filled, we won't have a v4 offload
		 * flag. We can't write zero checksums without inserting said
		 * flag, but our output frames won't necessarily be rechecked by
		 * the caller! As a compromise, we need to force emulation to
		 * uphold the same contracts the packet already agreed to.
		 */
		if (niph->ipha_hdr_checksum != 0) {
			emul |= MAC_IPCKSUM_EMUL;
			ocsum_flags |= HCK_IPV4_HDRCKSUM;
		}
		niph->ipha_hdr_checksum = 0;
		ip_id = ntohs(niph->ipha_ident);
	}

	ntcph = (tcph_t *)(nhdrmp->b_rptr + oehlen + oiphlen);
	tcp_seq = BE32_TO_U32(ntcph->th_seq);
	tcp_seq += mss;

	/*
	 * The first segment shouldn't:
	 *
	 *	o indicate end of data transmission (FIN),
	 *	o indicate immediate handling of the data (PUSH).
	 */
	ntcph->th_flags[0] &= ~(TH_FIN | TH_PUSH);
	DB_CKSUMFLAGS(nhdrmp) = (uint16_t)(ocsum_flags & ~HW_LSO);

	/*
	 * If the underlying HW provides partial checksum, then make
	 * sure to correct the pseudo header checksum before calling
	 * mac_sw_cksum(). The native TCP stack doesn't include the
	 * length field in the pseudo header when LSO is in play -- so
	 * we need to calculate it here.
	 */
	if (ocsum_flags & HCK_PARTIALCKSUM) {
		tcp_sum = BE16_TO_U16(ntcph->th_sum);
		otcp_sum = tcp_sum;
		tcp_sum += mss + otcphlen;
		tcp_sum = (tcp_sum >> 16) + (tcp_sum & 0xFFFF);
		U16_TO_BE16(tcp_sum, ntcph->th_sum);
	}

	if ((ocsum_flags & HCK_TX_FLAGS) && (emul & MAC_HWCKSUM_EMULS)) {
		next_nhdrmp = nhdrmp->b_next;
		nhdrmp->b_next = NULL;
		nhdrmp = mac_sw_cksum(nhdrmp, emul);
		/*
		 * The mblk could be replaced (via pull-up) or freed (due to
		 * failure) during mac_sw_cksum(), so we must take care with the
		 * result here.
		 */
		if (nhdrmp != NULL) {
			nhdrmp->b_next = next_nhdrmp;
			next_nhdrmp = NULL;
			seg_chain = nhdrmp;
		} else {
			freemsgchain(next_nhdrmp);
			/*
			 * nhdrmp referenced the head of seg_chain when it was
			 * freed, so further clean-up there is unnecessary
			 */
			seg_chain = NULL;
			mac_drop_pkt(omp, "LSO cksum emulation failed");
			goto fail;
		}
	}

	ASSERT3P(nhdrmp, !=, NULL);

	seg = 1;
	DTRACE_PROBE5(sw__lso__seg, mblk_t *, nhdrmp, void_ip_t *,
	    (is_v6 ? (void *)niph6 : (void *)niph),
	    __dtrace_tcp_tcph_t *, ntcph, uint_t, mss, int_t, seg);
	seg++;

	/* There better be at least 2 segs. */
	ASSERT3P(nhdrmp->b_next, !=, NULL);
	prev_nhdrmp = nhdrmp;
	nhdrmp = nhdrmp->b_next;

	/*
	 * Now adjust the headers of the middle segments. For each
	 * header we need to adjust the following.
	 *
	 *	o IP ID
	 *	o IP length
	 *	o TCP sequence
	 *	o TCP flags
	 *	o cksum flags
	 *	o cksum values (if MAC_HWCKSUM_EMUL is set)
	 */
	for (; seg < nsegs; seg++) {
		/*
		 * We use seg_chain as a reference to the first seg
		 * header mblk -- this first header is a template for
		 * the rest of the segments. This copy will include
		 * the now updated checksum values from the first
		 * header. We must reset these checksum values to
		 * their original to make sure we produce the correct
		 * value.
		 */
		ASSERT3P(msgsize(nhdrmp->b_cont), ==, mss);
		if (is_v6) {
			niph6 = (ip6_t *)(nhdrmp->b_rptr + oehlen);
			niph6->ip6_plen = htons(
			    (oiphlen - IPV6_HDR_LEN) + otcphlen + mss);
		} else {
			niph = (ipha_t *)(nhdrmp->b_rptr + oehlen);
			niph->ipha_ident = htons(++ip_id);
			niph->ipha_length = htons(oiphlen + otcphlen + mss);
			niph->ipha_hdr_checksum = 0;
		}
		ntcph = (tcph_t *)(nhdrmp->b_rptr + oehlen + oiphlen);
		U32_TO_BE32(tcp_seq, ntcph->th_seq);
		tcp_seq += mss;
		/*
		 * Just like the first segment, the middle segments
		 * shouldn't have these flags set.
		 */
		ntcph->th_flags[0] &= ~(TH_FIN | TH_PUSH);
		DB_CKSUMFLAGS(nhdrmp) = (uint16_t)(ocsum_flags & ~HW_LSO);

		/*
		 * First and middle segs have same
		 * pseudo-header checksum.
		 */
		if (ocsum_flags & HCK_PARTIALCKSUM)
			U16_TO_BE16(tcp_sum, ntcph->th_sum);

		if ((ocsum_flags & HCK_TX_FLAGS) &&
		    (emul & MAC_HWCKSUM_EMULS)) {
			next_nhdrmp = nhdrmp->b_next;
			nhdrmp->b_next = NULL;
			nhdrmp = mac_sw_cksum(nhdrmp, emul);
			/*
			 * Like above, handle cases where mac_sw_cksum() does a
			 * pull-up or drop of the mblk.
			 */
			if (nhdrmp != NULL) {
				nhdrmp->b_next = next_nhdrmp;
				next_nhdrmp = NULL;
				prev_nhdrmp->b_next = nhdrmp;
			} else {
				freemsgchain(next_nhdrmp);
				/*
				 * Critical to de-link the now-freed nhdrmp
				 * before freeing the rest of the preceding
				 * chain.
				 */
				prev_nhdrmp->b_next = NULL;
				freemsgchain(seg_chain);
				seg_chain = NULL;
				mac_drop_pkt(omp, "LSO cksum emulation failed");
				goto fail;
			}
		}

		DTRACE_PROBE5(sw__lso__seg, mblk_t *, nhdrmp, void_ip_t *,
		    (is_v6 ? (void *)niph6 : (void *)niph),
		    __dtrace_tcp_tcph_t *, ntcph, uint_t, mss, uint_t, seg);

		ASSERT3P(nhdrmp->b_next, !=, NULL);
		prev_nhdrmp = nhdrmp;
		nhdrmp = nhdrmp->b_next;
	}

	/* Make sure we are on the last segment. */
	ASSERT3U(seg, ==, nsegs);
	ASSERT3P(nhdrmp->b_next, ==, NULL);

	/*
	 * Now we set the last segment header. The difference being
	 * that FIN/PSH/RST flags are allowed.
	 */
	len = msgsize(nhdrmp->b_cont);
	ASSERT3S(len, >, 0);
	if (is_v6) {
		niph6 = (ip6_t *)(nhdrmp->b_rptr + oehlen);
		niph6->ip6_plen = htons(
		    (oiphlen - IPV6_HDR_LEN) + otcphlen + len);
	} else {
		niph = (ipha_t *)(nhdrmp->b_rptr + oehlen);
		niph->ipha_ident = htons(++ip_id);
		niph->ipha_length = htons(oiphlen + otcphlen + len);
		niph->ipha_hdr_checksum = 0;
	}
	ntcph = (tcph_t *)(nhdrmp->b_rptr + oehlen + oiphlen);
	U32_TO_BE32(tcp_seq, ntcph->th_seq);

	DB_CKSUMFLAGS(nhdrmp) = (uint16_t)(ocsum_flags & ~HW_LSO);
	if (ocsum_flags & HCK_PARTIALCKSUM) {
		tcp_sum = otcp_sum;
		tcp_sum += len + otcphlen;
		tcp_sum = (tcp_sum >> 16) + (tcp_sum & 0xFFFF);
		U16_TO_BE16(tcp_sum, ntcph->th_sum);
	}

	if ((ocsum_flags & HCK_TX_FLAGS) && (emul & MAC_HWCKSUM_EMULS)) {
		/* This should be the last mblk. */
		ASSERT3P(nhdrmp->b_next, ==, NULL);
		nhdrmp = mac_sw_cksum(nhdrmp, emul);
		/*
		 * If the final mblk happens to be dropped as part of
		 * mac_sw_cksum(), that is unfortunate, but it need not be a
		 * show-stopper at this point.  We can just pretend that final
		 * packet was dropped in transit.
		 */
		prev_nhdrmp->b_next = nhdrmp;
	}

	DTRACE_PROBE5(sw__lso__seg, mblk_t *, nhdrmp, void_ip_t *,
	    (is_v6 ? (void *)niph6 : (void *)niph),
	    __dtrace_tcp_tcph_t *, ntcph, uint_t, len, uint_t, seg);

	/*
	 * Free the reference to the original LSO message as it is
	 * being replaced by seg_cahin.
	 */
	freemsg(omp);
	*head = seg_chain;
	*tail = nhdrmp;
	*count = nsegs;
	return;

fail:
	*head = NULL;
	*tail = NULL;
	*count = 0;
}

#define	HCK_NEEDED	(HCK_IPV4_HDRCKSUM | HCK_PARTIALCKSUM | HCK_FULLCKSUM)

/*
 * Emulate various hardware offload features in software. Take a chain
 * of packets as input and emulate the hardware features specified in
 * 'emul'. The resulting chain's head pointer replaces the 'mp_chain'
 * pointer given as input, and its tail pointer is written to
 * '*otail'. The number of packets in the new chain is written to
 * '*ocount'. The 'otail' and 'ocount' arguments are optional and thus
 * may be NULL. The 'mp_chain' argument may point to a NULL chain; in
 * which case 'mp_chain' will simply stay a NULL chain.
 *
 * While unlikely, it is technically possible that this function could
 * receive a non-NULL chain as input and return a NULL chain as output
 * ('*mp_chain' and '*otail' would be NULL and '*ocount' would be
 * zero). This could happen if all the packets in the chain are
 * dropped or if we fail to allocate new mblks. In this case, there is
 * nothing for the caller to free. In any event, the caller shouldn't
 * assume that '*mp_chain' is non-NULL on return.
 *
 * This function was written with three main use cases in mind.
 *
 * 1. To emulate hardware offloads when traveling mac-loopback (two
 *    clients on the same mac). This is wired up in mac_tx_send().
 *
 * 2. To provide hardware offloads to the client when the underlying
 *    provider cannot. This is currently wired up in mac_tx() but we
 *    still only negotiate offloads when the underlying provider
 *    supports them.
 *
 * 3. To emulate real hardware in simnet.
 */
void
mac_hw_emul(mblk_t **mp_chain, mblk_t **otail, uint_t *ocount, mac_emul_t emul)
{
	mblk_t *head = NULL, *tail = NULL;
	uint_t count = 0;

	ASSERT3S(~(MAC_HWCKSUM_EMULS | MAC_LSO_EMUL) & emul, ==, 0);
	ASSERT3P(mp_chain, !=, NULL);

	for (mblk_t *mp = *mp_chain; mp != NULL; ) {
		mblk_t *tmp, *next, *tmphead, *tmptail;
		struct ether_header *ehp;
		uint32_t flags;
		uint_t len = MBLKL(mp), l2len;

		/* Perform LSO/cksum one message at a time. */
		next = mp->b_next;
		mp->b_next = NULL;

		/*
		 * For our sanity the first mblk should contain at
		 * least the full L2 header.
		 */
		if (len < sizeof (struct ether_header)) {
			mac_drop_pkt(mp, "packet too short (A): %u", len);
			mp = next;
			continue;
		}

		ehp = (struct ether_header *)mp->b_rptr;
		if (ntohs(ehp->ether_type) == VLAN_TPID)
			l2len = sizeof (struct ether_vlan_header);
		else
			l2len = sizeof (struct ether_header);

		/*
		 * If the first mblk is solely the L2 header, then
		 * there better be more data.
		 */
		if (len < l2len || (len == l2len && mp->b_cont == NULL)) {
			mac_drop_pkt(mp, "packet too short (C): %u", len);
			mp = next;
			continue;
		}

		DTRACE_PROBE2(mac__emul, mblk_t *, mp, mac_emul_t, emul);

		/*
		 * We use DB_CKSUMFLAGS (instead of mac_hcksum_get())
		 * because we don't want to mask-out the LSO flag.
		 */
		flags = DB_CKSUMFLAGS(mp);

		if ((flags & HW_LSO) && (emul & MAC_LSO_EMUL)) {
			uint_t tmpcount = 0;

			/*
			 * LSO fix-up handles checksum emulation
			 * inline (if requested). It also frees mp.
			 */
			mac_sw_lso(mp, emul, &tmphead, &tmptail,
			    &tmpcount);
			if (tmphead == NULL) {
				/* mac_sw_lso() freed the mp. */
				mp = next;
				continue;
			}
			count += tmpcount;
		} else if ((flags & HCK_NEEDED) && (emul & MAC_HWCKSUM_EMULS)) {
			tmp = mac_sw_cksum(mp, emul);
			if (tmp == NULL) {
				/* mac_sw_cksum() freed the mp. */
				mp = next;
				continue;
			}
			tmphead = tmp;
			tmptail = tmp;
			count++;
		} else {
			/* There is nothing to emulate. */
			tmp = mp;
			tmphead = tmp;
			tmptail = tmp;
			count++;
		}

		/*
		 * The tmp mblk chain is either the start of the new
		 * chain or added to the tail of the new chain.
		 */
		if (head == NULL) {
			head = tmphead;
			tail = tmptail;
		} else {
			/* Attach the new mblk to the end of the new chain. */
			tail->b_next = tmphead;
			tail = tmptail;
		}

		mp = next;
	}

	*mp_chain = head;

	if (otail != NULL)
		*otail = tail;

	if (ocount != NULL)
		*ocount = count;
}

/*
 * Add VLAN tag to the specified mblk.
 */
mblk_t *
mac_add_vlan_tag(mblk_t *mp, uint_t pri, uint16_t vid)
{
	mblk_t *hmp;
	struct ether_vlan_header *evhp;
	struct ether_header *ehp;

	ASSERT(pri != 0 || vid != 0);

	/*
	 * Allocate an mblk for the new tagged ethernet header,
	 * and copy the MAC addresses and ethertype from the
	 * original header.
	 */

	hmp = allocb(sizeof (struct ether_vlan_header), BPRI_MED);
	if (hmp == NULL) {
		freemsg(mp);
		return (NULL);
	}

	evhp = (struct ether_vlan_header *)hmp->b_rptr;
	ehp = (struct ether_header *)mp->b_rptr;

	bcopy(ehp, evhp, (ETHERADDRL * 2));
	evhp->ether_type = ehp->ether_type;
	evhp->ether_tpid = htons(ETHERTYPE_VLAN);

	hmp->b_wptr += sizeof (struct ether_vlan_header);
	mp->b_rptr += sizeof (struct ether_header);

	/*
	 * Free the original message if it's now empty. Link the
	 * rest of messages to the header message.
	 */
	mac_hcksum_clone(mp, hmp);
	if (MBLKL(mp) == 0) {
		hmp->b_cont = mp->b_cont;
		freeb(mp);
	} else {
		hmp->b_cont = mp;
	}
	ASSERT(MBLKL(hmp) >= sizeof (struct ether_vlan_header));

	/*
	 * Initialize the new TCI (Tag Control Information).
	 */
	evhp->ether_tci = htons(VLAN_TCI(pri, 0, vid));

	return (hmp);
}

/*
 * Adds a VLAN tag with the specified VID and priority to each mblk of
 * the specified chain.
 */
mblk_t *
mac_add_vlan_tag_chain(mblk_t *mp_chain, uint_t pri, uint16_t vid)
{
	mblk_t *next_mp, **prev, *mp;

	mp = mp_chain;
	prev = &mp_chain;

	while (mp != NULL) {
		next_mp = mp->b_next;
		mp->b_next = NULL;
		if ((mp = mac_add_vlan_tag(mp, pri, vid)) == NULL) {
			freemsgchain(next_mp);
			break;
		}
		*prev = mp;
		prev = &mp->b_next;
		mp = mp->b_next = next_mp;
	}

	return (mp_chain);
}

/*
 * Strip VLAN tag
 */
mblk_t *
mac_strip_vlan_tag(mblk_t *mp)
{
	mblk_t *newmp;
	struct ether_vlan_header *evhp;

	evhp = (struct ether_vlan_header *)mp->b_rptr;
	if (ntohs(evhp->ether_tpid) == ETHERTYPE_VLAN) {
		ASSERT(MBLKL(mp) >= sizeof (struct ether_vlan_header));

		if (DB_REF(mp) > 1) {
			newmp = copymsg(mp);
			if (newmp == NULL)
				return (NULL);
			freemsg(mp);
			mp = newmp;
		}

		evhp = (struct ether_vlan_header *)mp->b_rptr;

		ovbcopy(mp->b_rptr, mp->b_rptr + VLAN_TAGSZ, 2 * ETHERADDRL);
		mp->b_rptr += VLAN_TAGSZ;
	}
	return (mp);
}

/*
 * Strip VLAN tag from each mblk of the chain.
 */
mblk_t *
mac_strip_vlan_tag_chain(mblk_t *mp_chain)
{
	mblk_t *mp, *next_mp, **prev;

	mp = mp_chain;
	prev = &mp_chain;

	while (mp != NULL) {
		next_mp = mp->b_next;
		mp->b_next = NULL;
		if ((mp = mac_strip_vlan_tag(mp)) == NULL) {
			freemsgchain(next_mp);
			break;
		}
		*prev = mp;
		prev = &mp->b_next;
		mp = mp->b_next = next_mp;
	}

	return (mp_chain);
}

/*
 * Default callback function. Used when the datapath is not yet initialized.
 */
/* ARGSUSED */
void
mac_rx_def(void *arg, mac_resource_handle_t resource, mblk_t *mp_chain,
    boolean_t loopback)
{
	freemsgchain(mp_chain);
}

/*
 * Determines the IPv6 header length accounting for all the optional IPv6
 * headers (hop-by-hop, destination, routing and fragment). The header length
 * and next header value (a transport header) is captured.
 *
 * Returns B_FALSE if all the IP headers are not in the same mblk otherwise
 * returns B_TRUE.
 */
boolean_t
mac_ip_hdr_length_v6(ip6_t *ip6h, uint8_t *endptr, uint16_t *hdr_length,
    uint8_t *next_hdr, ip6_frag_t **fragp)
{
	uint16_t length;
	uint_t	ehdrlen;
	uint8_t *whereptr;
	uint8_t *nexthdrp;
	ip6_dest_t *desthdr;
	ip6_rthdr_t *rthdr;
	ip6_frag_t *fraghdr;

	if (((uchar_t *)ip6h + IPV6_HDR_LEN) > endptr)
		return (B_FALSE);
	ASSERT(IPH_HDR_VERSION(ip6h) == IPV6_VERSION);
	length = IPV6_HDR_LEN;
	whereptr = ((uint8_t *)&ip6h[1]); /* point to next hdr */

	if (fragp != NULL)
		*fragp = NULL;

	nexthdrp = &ip6h->ip6_nxt;
	while (whereptr < endptr) {
		/* Is there enough left for len + nexthdr? */
		if (whereptr + MIN_EHDR_LEN > endptr)
			break;

		switch (*nexthdrp) {
		case IPPROTO_HOPOPTS:
		case IPPROTO_DSTOPTS:
			/* Assumes the headers are identical for hbh and dst */
			desthdr = (ip6_dest_t *)whereptr;
			ehdrlen = 8 * (desthdr->ip6d_len + 1);
			if ((uchar_t *)desthdr +  ehdrlen > endptr)
				return (B_FALSE);
			nexthdrp = &desthdr->ip6d_nxt;
			break;
		case IPPROTO_ROUTING:
			rthdr = (ip6_rthdr_t *)whereptr;
			ehdrlen =  8 * (rthdr->ip6r_len + 1);
			if ((uchar_t *)rthdr +  ehdrlen > endptr)
				return (B_FALSE);
			nexthdrp = &rthdr->ip6r_nxt;
			break;
		case IPPROTO_FRAGMENT:
			fraghdr = (ip6_frag_t *)whereptr;
			ehdrlen = sizeof (ip6_frag_t);
			if ((uchar_t *)&fraghdr[1] > endptr)
				return (B_FALSE);
			nexthdrp = &fraghdr->ip6f_nxt;
			if (fragp != NULL)
				*fragp = fraghdr;
			break;
		case IPPROTO_NONE:
			/* No next header means we're finished */
		default:
			*hdr_length = length;
			*next_hdr = *nexthdrp;
			return (B_TRUE);
		}
		length += ehdrlen;
		whereptr += ehdrlen;
		*hdr_length = length;
		*next_hdr = *nexthdrp;
	}
	switch (*nexthdrp) {
	case IPPROTO_HOPOPTS:
	case IPPROTO_DSTOPTS:
	case IPPROTO_ROUTING:
	case IPPROTO_FRAGMENT:
		/*
		 * If any know extension headers are still to be processed,
		 * the packet's malformed (or at least all the IP header(s) are
		 * not in the same mblk - and that should never happen.
		 */
		return (B_FALSE);

	default:
		/*
		 * If we get here, we know that all of the IP headers were in
		 * the same mblk, even if the ULP header is in the next mblk.
		 */
		*hdr_length = length;
		*next_hdr = *nexthdrp;
		return (B_TRUE);
	}
}

/*
 * The following set of routines are there to take care of interrupt
 * re-targeting for legacy (fixed) interrupts. Some older versions
 * of the popular NICs like e1000g do not support MSI-X interrupts
 * and they reserve fixed interrupts for RX/TX rings. To re-target
 * these interrupts, PCITOOL ioctls need to be used.
 */
typedef struct mac_dladm_intr {
	int	ino;
	int	cpu_id;
	char	driver_path[MAXPATHLEN];
	char	nexus_path[MAXPATHLEN];
} mac_dladm_intr_t;

/* Bind the interrupt to cpu_num */
static int
mac_set_intr(ldi_handle_t lh, processorid_t cpu_num, int oldcpuid, int ino)
{
	pcitool_intr_set_t	iset;
	int			err;

	iset.old_cpu = oldcpuid;
	iset.ino = ino;
	iset.cpu_id = cpu_num;
	iset.user_version = PCITOOL_VERSION;
	err = ldi_ioctl(lh, PCITOOL_DEVICE_SET_INTR, (intptr_t)&iset, FKIOCTL,
	    kcred, NULL);

	return (err);
}

/*
 * Search interrupt information. iget is filled in with the info to search
 */
static boolean_t
mac_search_intrinfo(pcitool_intr_get_t *iget_p, mac_dladm_intr_t *dln)
{
	int	i;
	char	driver_path[2 * MAXPATHLEN];

	for (i = 0; i < iget_p->num_devs; i++) {
		(void) strlcpy(driver_path, iget_p->dev[i].path, MAXPATHLEN);
		(void) snprintf(&driver_path[strlen(driver_path)], MAXPATHLEN,
		    ":%s%d", iget_p->dev[i].driver_name,
		    iget_p->dev[i].dev_inst);
		/* Match the device path for the device path */
		if (strcmp(driver_path, dln->driver_path) == 0) {
			dln->ino = iget_p->ino;
			dln->cpu_id = iget_p->cpu_id;
			return (B_TRUE);
		}
	}
	return (B_FALSE);
}

/*
 * Get information about ino, i.e. if this is the interrupt for our
 * device and where it is bound etc.
 */
static boolean_t
mac_get_single_intr(ldi_handle_t lh, int oldcpuid, int ino,
    mac_dladm_intr_t *dln)
{
	pcitool_intr_get_t	*iget_p;
	int			ipsz;
	int			nipsz;
	int			err;
	uint8_t			inum;

	/*
	 * Check if SLEEP is OK, i.e if could come here in response to
	 * changing the fanout due to some callback from the driver, say
	 * link speed changes.
	 */
	ipsz = PCITOOL_IGET_SIZE(0);
	iget_p = kmem_zalloc(ipsz, KM_SLEEP);

	iget_p->num_devs_ret = 0;
	iget_p->user_version = PCITOOL_VERSION;
	iget_p->cpu_id = oldcpuid;
	iget_p->ino = ino;

	err = ldi_ioctl(lh, PCITOOL_DEVICE_GET_INTR, (intptr_t)iget_p,
	    FKIOCTL, kcred, NULL);
	if (err != 0) {
		kmem_free(iget_p, ipsz);
		return (B_FALSE);
	}
	if (iget_p->num_devs == 0) {
		kmem_free(iget_p, ipsz);
		return (B_FALSE);
	}
	inum = iget_p->num_devs;
	if (iget_p->num_devs_ret < iget_p->num_devs) {
		/* Reallocate */
		nipsz = PCITOOL_IGET_SIZE(iget_p->num_devs);

		kmem_free(iget_p, ipsz);
		ipsz = nipsz;
		iget_p = kmem_zalloc(ipsz, KM_SLEEP);

		iget_p->num_devs_ret = inum;
		iget_p->cpu_id = oldcpuid;
		iget_p->ino = ino;
		iget_p->user_version = PCITOOL_VERSION;
		err = ldi_ioctl(lh, PCITOOL_DEVICE_GET_INTR, (intptr_t)iget_p,
		    FKIOCTL, kcred, NULL);
		if (err != 0) {
			kmem_free(iget_p, ipsz);
			return (B_FALSE);
		}
		/* defensive */
		if (iget_p->num_devs != iget_p->num_devs_ret) {
			kmem_free(iget_p, ipsz);
			return (B_FALSE);
		}
	}

	if (mac_search_intrinfo(iget_p, dln)) {
		kmem_free(iget_p, ipsz);
		return (B_TRUE);
	}
	kmem_free(iget_p, ipsz);
	return (B_FALSE);
}

/*
 * Get the interrupts and check each one to see if it is for our device.
 */
static int
mac_validate_intr(ldi_handle_t lh, mac_dladm_intr_t *dln, processorid_t cpuid)
{
	pcitool_intr_info_t	intr_info;
	int			err;
	int			ino;
	int			oldcpuid;

	err = ldi_ioctl(lh, PCITOOL_SYSTEM_INTR_INFO, (intptr_t)&intr_info,
	    FKIOCTL, kcred, NULL);
	if (err != 0)
		return (-1);

	for (oldcpuid = 0; oldcpuid < intr_info.num_cpu; oldcpuid++) {
		for (ino = 0; ino < intr_info.num_intr; ino++) {
			if (mac_get_single_intr(lh, oldcpuid, ino, dln)) {
				if (dln->cpu_id == cpuid)
					return (0);
				return (1);
			}
		}
	}
	return (-1);
}

/*
 * Obtain the nexus parent node info. for mdip.
 */
static dev_info_t *
mac_get_nexus_node(dev_info_t *mdip, mac_dladm_intr_t *dln)
{
	struct dev_info		*tdip = (struct dev_info *)mdip;
	struct ddi_minor_data	*minordata;
	dev_info_t		*pdip;
	char			pathname[MAXPATHLEN];

	while (tdip != NULL) {
		/*
		 * The netboot code could call this function while walking the
		 * device tree so we need to use ndi_devi_tryenter() here to
		 * avoid deadlock.
		 */
		if (ndi_devi_tryenter((dev_info_t *)tdip) == 0)
			break;

		for (minordata = tdip->devi_minor; minordata != NULL;
		    minordata = minordata->next) {
			if (strncmp(minordata->ddm_node_type, DDI_NT_INTRCTL,
			    strlen(DDI_NT_INTRCTL)) == 0) {
				pdip = minordata->dip;
				(void) ddi_pathname(pdip, pathname);
				(void) snprintf(dln->nexus_path, MAXPATHLEN,
				    "/devices%s:intr", pathname);
				(void) ddi_pathname_minor(minordata, pathname);
				ndi_devi_exit((dev_info_t *)tdip);
				return (pdip);
			}
		}
		ndi_devi_exit((dev_info_t *)tdip);
		tdip = tdip->devi_parent;
	}
	return (NULL);
}

/*
 * For a primary MAC client, if the user has set a list or CPUs or
 * we have obtained it implicitly, we try to retarget the interrupt
 * for that device on one of the CPUs in the list.
 * We assign the interrupt to the same CPU as the poll thread.
 */
static boolean_t
mac_check_interrupt_binding(dev_info_t *mdip, int32_t cpuid)
{
	ldi_handle_t		lh = NULL;
	ldi_ident_t		li = NULL;
	int			err;
	int			ret;
	mac_dladm_intr_t	dln;
	dev_info_t		*dip;
	struct ddi_minor_data	*minordata;

	dln.nexus_path[0] = '\0';
	dln.driver_path[0] = '\0';

	minordata = ((struct dev_info *)mdip)->devi_minor;
	while (minordata != NULL) {
		if (minordata->type == DDM_MINOR)
			break;
		minordata = minordata->next;
	}
	if (minordata == NULL)
		return (B_FALSE);

	(void) ddi_pathname_minor(minordata, dln.driver_path);

	dip = mac_get_nexus_node(mdip, &dln);
	/* defensive */
	if (dip == NULL)
		return (B_FALSE);

	err = ldi_ident_from_major(ddi_driver_major(dip), &li);
	if (err != 0)
		return (B_FALSE);

	err = ldi_open_by_name(dln.nexus_path, FREAD|FWRITE, kcred, &lh, li);
	if (err != 0)
		return (B_FALSE);

	ret = mac_validate_intr(lh, &dln, cpuid);
	if (ret < 0) {
		(void) ldi_close(lh, FREAD|FWRITE, kcred);
		return (B_FALSE);
	}
	/* cmn_note? */
	if (ret != 0)
		if ((err = (mac_set_intr(lh, cpuid, dln.cpu_id, dln.ino)))
		    != 0) {
			(void) ldi_close(lh, FREAD|FWRITE, kcred);
			return (B_FALSE);
		}
	(void) ldi_close(lh, FREAD|FWRITE, kcred);
	return (B_TRUE);
}

void
mac_client_set_intr_cpu(void *arg, mac_client_handle_t mch, int32_t cpuid)
{
	dev_info_t		*mdip = (dev_info_t *)arg;
	mac_client_impl_t	*mcip = (mac_client_impl_t *)mch;
	mac_resource_props_t	*mrp;
	mac_perim_handle_t	mph;
	flow_entry_t		*flent = mcip->mci_flent;
	mac_soft_ring_set_t	*rx_srs;
	mac_cpus_t		*srs_cpu;

	if (!mac_check_interrupt_binding(mdip, cpuid))
		cpuid = -1;
	mac_perim_enter_by_mh((mac_handle_t)mcip->mci_mip, &mph);
	mrp = MCIP_RESOURCE_PROPS(mcip);
	mrp->mrp_rx_intr_cpu = cpuid;
	if (flent != NULL && flent->fe_rx_srs_cnt == 2) {
		rx_srs = flent->fe_rx_srs[1];
		srs_cpu = &rx_srs->srs_cpu;
		srs_cpu->mc_rx_intr_cpu = cpuid;
	}
	mac_perim_exit(mph);
}

int32_t
mac_client_intr_cpu(mac_client_handle_t mch)
{
	mac_client_impl_t	*mcip = (mac_client_impl_t *)mch;
	mac_cpus_t		*srs_cpu;
	mac_soft_ring_set_t	*rx_srs;
	flow_entry_t		*flent = mcip->mci_flent;
	mac_resource_props_t	*mrp = MCIP_RESOURCE_PROPS(mcip);
	mac_ring_t		*ring;
	mac_intr_t		*mintr;

	/*
	 * Check if we need to retarget the interrupt. We do this only
	 * for the primary MAC client. We do this if we have the only
	 * exclusive ring in the group.
	 */
	if (mac_is_primary_client(mcip) && flent->fe_rx_srs_cnt == 2) {
		rx_srs = flent->fe_rx_srs[1];
		srs_cpu = &rx_srs->srs_cpu;
		ring = rx_srs->srs_ring;
		mintr = &ring->mr_info.mri_intr;
		/*
		 * If ddi_handle is present or the poll CPU is
		 * already bound to the interrupt CPU, return -1.
		 */
		if (mintr->mi_ddi_handle != NULL ||
		    ((mrp->mrp_ncpus != 0) &&
		    (mrp->mrp_rx_intr_cpu == srs_cpu->mc_rx_pollid))) {
			return (-1);
		}
		return (srs_cpu->mc_rx_pollid);
	}
	return (-1);
}

void *
mac_get_devinfo(mac_handle_t mh)
{
	mac_impl_t	*mip = (mac_impl_t *)mh;

	return ((void *)mip->mi_dip);
}

#define	PKT_HASH_2BYTES(x) ((x)[0] ^ (x)[1])
#define	PKT_HASH_4BYTES(x) ((x)[0] ^ (x)[1] ^ (x)[2] ^ (x)[3])
#define	PKT_HASH_MAC(x) ((x)[0] ^ (x)[1] ^ (x)[2] ^ (x)[3] ^ (x)[4] ^ (x)[5])

uint64_t
mac_pkt_hash(uint_t media, mblk_t *mp, uint8_t policy, boolean_t is_outbound)
{
	struct ether_header *ehp;
	uint64_t hash = 0;
	uint16_t sap;
	uint_t skip_len;
	uint8_t proto;
	boolean_t ip_fragmented;

	/*
	 * We may want to have one of these per MAC type plugin in the
	 * future. For now supports only ethernet.
	 */
	if (media != DL_ETHER)
		return (0L);

	/* for now we support only outbound packets */
	ASSERT(is_outbound);
	ASSERT(IS_P2ALIGNED(mp->b_rptr, sizeof (uint16_t)));
	ASSERT(MBLKL(mp) >= sizeof (struct ether_header));

	/* compute L2 hash */

	ehp = (struct ether_header *)mp->b_rptr;

	if ((policy & MAC_PKT_HASH_L2) != 0) {
		uchar_t *mac_src = ehp->ether_shost.ether_addr_octet;
		uchar_t *mac_dst = ehp->ether_dhost.ether_addr_octet;
		hash = PKT_HASH_MAC(mac_src) ^ PKT_HASH_MAC(mac_dst);
		policy &= ~MAC_PKT_HASH_L2;
	}

	if (policy == 0)
		goto done;

	/* skip ethernet header */

	sap = ntohs(ehp->ether_type);
	if (sap == ETHERTYPE_VLAN) {
		struct ether_vlan_header *evhp;
		mblk_t *newmp = NULL;

		skip_len = sizeof (struct ether_vlan_header);
		if (MBLKL(mp) < skip_len) {
			/* the vlan tag is the payload, pull up first */
			newmp = msgpullup(mp, -1);
			if ((newmp == NULL) || (MBLKL(newmp) < skip_len)) {
				goto done;
			}
			evhp = (struct ether_vlan_header *)newmp->b_rptr;
		} else {
			evhp = (struct ether_vlan_header *)mp->b_rptr;
		}

		sap = ntohs(evhp->ether_type);
		freemsg(newmp);
	} else {
		skip_len = sizeof (struct ether_header);
	}

	/* if ethernet header is in its own mblk, skip it */
	if (MBLKL(mp) <= skip_len) {
		skip_len -= MBLKL(mp);
		mp = mp->b_cont;
		if (mp == NULL)
			goto done;
	}

	sap = (sap < ETHERTYPE_802_MIN) ? 0 : sap;

	/* compute IP src/dst addresses hash and skip IPv{4,6} header */

	switch (sap) {
	case ETHERTYPE_IP: {
		ipha_t *iphp;

		/*
		 * If the header is not aligned or the header doesn't fit
		 * in the mblk, bail now. Note that this may cause packets
		 * reordering.
		 */
		iphp = (ipha_t *)(mp->b_rptr + skip_len);
		if (((unsigned char *)iphp + sizeof (ipha_t) > mp->b_wptr) ||
		    !OK_32PTR((char *)iphp))
			goto done;

		proto = iphp->ipha_protocol;
		skip_len += IPH_HDR_LENGTH(iphp);

		/* Check if the packet is fragmented. */
		ip_fragmented = ntohs(iphp->ipha_fragment_offset_and_flags) &
		    IPH_OFFSET;

		/*
		 * For fragmented packets, use addresses in addition to
		 * the frag_id to generate the hash inorder to get
		 * better distribution.
		 */
		if (ip_fragmented || (policy & MAC_PKT_HASH_L3) != 0) {
			uint8_t *ip_src = (uint8_t *)&(iphp->ipha_src);
			uint8_t *ip_dst = (uint8_t *)&(iphp->ipha_dst);

			hash ^= (PKT_HASH_4BYTES(ip_src) ^
			    PKT_HASH_4BYTES(ip_dst));
			policy &= ~MAC_PKT_HASH_L3;
		}

		if (ip_fragmented) {
			uint8_t *identp = (uint8_t *)&iphp->ipha_ident;
			hash ^= PKT_HASH_2BYTES(identp);
			goto done;
		}
		break;
	}
	case ETHERTYPE_IPV6: {
		ip6_t *ip6hp;
		ip6_frag_t *frag = NULL;
		uint16_t hdr_length;

		/*
		 * If the header is not aligned or the header doesn't fit
		 * in the mblk, bail now. Note that this may cause packets
		 * reordering.
		 */

		ip6hp = (ip6_t *)(mp->b_rptr + skip_len);
		if (((unsigned char *)ip6hp + IPV6_HDR_LEN > mp->b_wptr) ||
		    !OK_32PTR((char *)ip6hp))
			goto done;

		if (!mac_ip_hdr_length_v6(ip6hp, mp->b_wptr, &hdr_length,
		    &proto, &frag))
			goto done;
		skip_len += hdr_length;

		/*
		 * For fragmented packets, use addresses in addition to
		 * the frag_id to generate the hash inorder to get
		 * better distribution.
		 */
		if (frag != NULL || (policy & MAC_PKT_HASH_L3) != 0) {
			uint8_t *ip_src = &(ip6hp->ip6_src.s6_addr8[12]);
			uint8_t *ip_dst = &(ip6hp->ip6_dst.s6_addr8[12]);

			hash ^= (PKT_HASH_4BYTES(ip_src) ^
			    PKT_HASH_4BYTES(ip_dst));
			policy &= ~MAC_PKT_HASH_L3;
		}

		if (frag != NULL) {
			uint8_t *identp = (uint8_t *)&frag->ip6f_ident;
			hash ^= PKT_HASH_4BYTES(identp);
			goto done;
		}
		break;
	}
	default:
		goto done;
	}

	if (policy == 0)
		goto done;

	/* if ip header is in its own mblk, skip it */
	if (MBLKL(mp) <= skip_len) {
		skip_len -= MBLKL(mp);
		mp = mp->b_cont;
		if (mp == NULL)
			goto done;
	}

	/* parse ULP header */
again:
	switch (proto) {
	case IPPROTO_TCP:
	case IPPROTO_UDP:
	case IPPROTO_ESP:
	case IPPROTO_SCTP:
		/*
		 * These Internet Protocols are intentionally designed
		 * for hashing from the git-go.  Port numbers are in the first
		 * word for transports, SPI is first for ESP.
		 */
		if (mp->b_rptr + skip_len + 4 > mp->b_wptr)
			goto done;
		hash ^= PKT_HASH_4BYTES((mp->b_rptr + skip_len));
		break;

	case IPPROTO_AH: {
		ah_t *ah = (ah_t *)(mp->b_rptr + skip_len);
		uint_t ah_length = AH_TOTAL_LEN(ah);

		if ((unsigned char *)ah + sizeof (ah_t) > mp->b_wptr)
			goto done;

		proto = ah->ah_nexthdr;
		skip_len += ah_length;

		/* if AH header is in its own mblk, skip it */
		if (MBLKL(mp) <= skip_len) {
			skip_len -= MBLKL(mp);
			mp = mp->b_cont;
			if (mp == NULL)
				goto done;
		}

		goto again;
	}
	}

done:
	return (hash);
}