/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* Copyright (c) 1990 Mentat Inc. */ #include <sys/types.h> #include <sys/inttypes.h> #include <sys/systm.h> #include <sys/stream.h> #include <sys/strsun.h> #include <sys/debug.h> #include <sys/ddi.h> #include <sys/vtrace.h> #include <inet/sctp_crc32.h> #include <inet/ip.h> #include <sys/multidata.h> #include <sys/multidata_impl.h> extern unsigned int ip_ocsum(ushort_t *address, int halfword_count, unsigned int sum); /* * Checksum routine for Internet Protocol family headers. * This routine is very heavily used in the network * code and should be modified for each CPU to be as fast as possible. */ #define mp_len(mp) ((mp)->b_wptr - (mp)->b_rptr) /* * Even/Odd checks. Usually it is performed on pointers but may be * used on integers as well. uintptr_t is long enough to hold both * integer and pointer. */ #define is_odd(p) (((uintptr_t)(p) & 0x1) != 0) #define is_even(p) (!is_odd(p)) #ifdef ZC_TEST /* * Disable the TCP s/w cksum. * XXX - This is just a hack for testing purpose. Don't use it for * anything else! */ int noswcksum = 0; #endif /* * Note: this does not ones-complement the result since it is used * when computing partial checksums. * For nonSTRUIO_IP mblks, assumes mp->b_rptr+offset is 16 bit aligned. * For STRUIO_IP mblks, assumes mp->b_datap->db_struiobase is 16 bit aligned. * * Note: for STRUIO_IP special mblks some data may have been previously * checksumed, this routine will handle additional data prefixed within * an mblk or b_cont (chained) mblk(s). This routine will also handle * suffixed b_cont mblk(s) and data suffixed within an mblk. */ unsigned int ip_cksum(mblk_t *mp, int offset, uint_t sum) { ushort_t *w; ssize_t mlen; int pmlen; mblk_t *pmp; dblk_t *dp = mp->b_datap; ushort_t psum = 0; #ifdef ZC_TEST if (noswcksum) return (0xffff); #endif ASSERT(dp); if (mp->b_cont == NULL) { /* * May be fast-path, only one mblk. */ w = (ushort_t *)(mp->b_rptr + offset); if (dp->db_struioflag & STRUIO_IP) { /* * Checksum any data not already done by * the caller and add in any partial checksum. */ if ((offset > dp->db_cksumstart) || mp->b_wptr != (uchar_t *)(mp->b_rptr + dp->db_cksumend)) { /* * Mblk data pointers aren't inclusive * of uio data, so disregard checksum. * * not using all of data in dblk make sure * not use to use the precalculated checksum * in this case. */ dp->db_struioflag &= ~STRUIO_IP; goto norm; } ASSERT(mp->b_wptr == (mp->b_rptr + dp->db_cksumend)); psum = *(ushort_t *)dp->db_struioun.data; if ((mlen = dp->db_cksumstart - offset) < 0) mlen = 0; if (is_odd(mlen)) goto slow; if (mlen && dp->db_cksumstart != dp->db_cksumstuff && dp->db_cksumend != dp->db_cksumstuff) { /* * There is prefix data to do and some uio * data has already been checksumed and there * is more uio data to do, so do the prefix * data first, then do the remainder of the * uio data. */ sum = ip_ocsum(w, mlen >> 1, sum); w = (ushort_t *)(mp->b_rptr + dp->db_cksumstuff); if (is_odd(w)) { pmp = mp; goto slow1; } mlen = dp->db_cksumend - dp->db_cksumstuff; } else if (dp->db_cksumend != dp->db_cksumstuff) { /* * There may be uio data to do, if there is * prefix data to do then add in all of the * uio data (if any) to do, else just do any * uio data. */ if (mlen) mlen += dp->db_cksumend - dp->db_cksumstuff; else { w = (ushort_t *)(mp->b_rptr + dp->db_cksumstuff); if (is_odd(w)) goto slow; mlen = dp->db_cksumend - dp->db_cksumstuff; } } else if (mlen == 0) return (psum); if (is_odd(mlen)) goto slow; sum += psum; } else { /* * Checksum all data not already done by the caller. */ norm: mlen = mp->b_wptr - (uchar_t *)w; if (is_odd(mlen)) goto slow; } ASSERT(is_even(w)); ASSERT(is_even(mlen)); return (ip_ocsum(w, mlen >> 1, sum)); } if (dp->db_struioflag & STRUIO_IP) psum = *(ushort_t *)dp->db_struioun.data; slow: pmp = 0; slow1: mlen = 0; pmlen = 0; for (; ; ) { /* * Each trip around loop adds in word(s) from one mbuf segment * (except for when pmp == mp, then its two partial trips). */ w = (ushort_t *)(mp->b_rptr + offset); if (pmp) { /* * This is the second trip around for this mblk. */ pmp = 0; mlen = 0; goto douio; } else if (dp->db_struioflag & STRUIO_IP) { /* * Checksum any data not already done by the * caller and add in any partial checksum. */ if ((offset > dp->db_cksumstart) || mp->b_wptr != (uchar_t *)(mp->b_rptr + dp->db_cksumend)) { /* * Mblk data pointers aren't inclusive * of uio data, so disregard checksum. * * not using all of data in dblk make sure * not use to use the precalculated checksum * in this case. */ dp->db_struioflag &= ~STRUIO_IP; goto snorm; } ASSERT(mp->b_wptr == (mp->b_rptr + dp->db_cksumend)); if ((mlen = dp->db_cksumstart - offset) < 0) mlen = 0; if (mlen && dp->db_cksumstart != dp->db_cksumstuff) { /* * There is prefix data too do and some * uio data has already been checksumed, * so do the prefix data only this trip. */ pmp = mp; } else { /* * Add in any partial cksum (if any) and * do the remainder of the uio data. */ int odd; douio: odd = is_odd(dp->db_cksumstuff - dp->db_cksumstart); if (pmlen == -1) { /* * Previous mlen was odd, so swap * the partial checksum bytes. */ sum += ((psum << 8) & 0xffff) | (psum >> 8); if (odd) pmlen = 0; } else { sum += psum; if (odd) pmlen = -1; } if (dp->db_cksumend != dp->db_cksumstuff) { /* * If prefix data to do and then all * the uio data nees to be checksumed, * else just do any uio data. */ if (mlen) mlen += dp->db_cksumend - dp->db_cksumstuff; else { w = (ushort_t *)(mp->b_rptr + dp->db_cksumstuff); mlen = dp->db_cksumend - dp->db_cksumstuff; } } } } else { /* * Checksum all of the mblk data. */ snorm: mlen = mp->b_wptr - (uchar_t *)w; } mp = mp->b_cont; if (mlen > 0 && pmlen == -1) { /* * There is a byte left from the last * segment; add it into the checksum. * Don't have to worry about a carry- * out here because we make sure that * high part of (32 bit) sum is small * below. */ #ifdef _LITTLE_ENDIAN sum += *(uchar_t *)w << 8; #else sum += *(uchar_t *)w; #endif w = (ushort_t *)((char *)w + 1); mlen--; pmlen = 0; } if (mlen > 0) { if (is_even(w)) { sum = ip_ocsum(w, mlen>>1, sum); w += mlen>>1; /* * If we had an odd number of bytes, * then the last byte goes in the high * part of the sum, and we take the * first byte to the low part of the sum * the next time around the loop. */ if (is_odd(mlen)) { #ifdef _LITTLE_ENDIAN sum += *(uchar_t *)w; #else sum += *(uchar_t *)w << 8; #endif pmlen = -1; } } else { ushort_t swsum; #ifdef _LITTLE_ENDIAN sum += *(uchar_t *)w; #else sum += *(uchar_t *)w << 8; #endif mlen--; w = (ushort_t *)(1 + (uintptr_t)w); /* Do a separate checksum and copy operation */ swsum = ip_ocsum(w, mlen>>1, 0); sum += ((swsum << 8) & 0xffff) | (swsum >> 8); w += mlen>>1; /* * If we had an even number of bytes, * then the last byte goes in the low * part of the sum. Otherwise we had an * odd number of bytes and we take the first * byte to the low part of the sum the * next time around the loop. */ if (is_odd(mlen)) { #ifdef _LITTLE_ENDIAN sum += *(uchar_t *)w << 8; #else sum += *(uchar_t *)w; #endif } else pmlen = -1; } } /* * Locate the next block with some data. * If there is a word split across a boundary we * will wrap to the top with mlen == -1 and * then add it in shifted appropriately. */ offset = 0; if (! pmp) { for (; ; ) { if (mp == 0) { goto done; } if (mp_len(mp)) break; mp = mp->b_cont; } dp = mp->b_datap; if (dp->db_struioflag & STRUIO_IP) psum = *(ushort_t *)dp->db_struioun.data; } else mp = pmp; } done: /* * Add together high and low parts of sum * and carry to get cksum. * Have to be careful to not drop the last * carry here. */ sum = (sum & 0xFFFF) + (sum >> 16); sum = (sum & 0xFFFF) + (sum >> 16); TRACE_3(TR_FAC_IP, TR_IP_CKSUM_END, "ip_cksum_end:(%S) type %d (%X)", "ip_cksum", 1, sum); return (sum); } uint32_t sctp_cksum(mblk_t *mp, int offset) { uint32_t crc32; uchar_t *p = NULL; crc32 = 0xFFFFFFFF; p = mp->b_rptr + offset; crc32 = sctp_crc32(crc32, p, mp->b_wptr - p); for (mp = mp->b_cont; mp != NULL; mp = mp->b_cont) { crc32 = sctp_crc32(crc32, mp->b_rptr, MBLKL(mp)); } /* Complement the result */ crc32 = ~crc32; return (crc32); } /* * Routine to compute Internet checksum (16-bit 1's complement) of a given * Multidata packet descriptor. As in the non-Multidata routine, this doesn't * 1's complement the result, such that it may be used to compute partial * checksums. Since it works on buffer spans rather than mblks, this routine * does not handle existing partial checksum value as in the STRUIO_IP special * mblk case (supporting this is rather trivial, but is perhaps of no use at * the moment unless synchronous streams and delayed checksum calculation are * revived.) * * Note also here that the given Multidata packet descriptor must refer to * a header buffer, i.e. it must have a header fragment. In addition, the * offset must lie within the boundary of the header fragment. For the * outbound tcp (MDT) case, this will not be an issue because the stack * ensures that such conditions are met, and that there is no need whatsoever * to compute partial checksums on an arbitrary offset that is not part of * the header fragment. We may need to revisit this routine to handle all * cases of the inbound (MDR) case, especially when we need to perform partial * checksum calculation due to padded bytes (non-zeroes) in the frame. */ uint_t ip_md_cksum(pdesc_t *pd, int offset, uint_t sum) { pdescinfo_t *pdi = &pd->pd_pdi; uchar_t *reg_start, *reg_end; ssize_t mlen, i; ushort_t *w; boolean_t byteleft = B_FALSE; ASSERT((pdi->flags & PDESC_HAS_REF) != 0); ASSERT(pdi->hdr_rptr != NULL && pdi->hdr_wptr != NULL); ASSERT(offset <= PDESC_HDRL(pdi)); for (i = 0; i < pdi->pld_cnt + 1; i++) { if (i == 0) { reg_start = pdi->hdr_rptr; reg_end = pdi->hdr_wptr; } else { reg_start = pdi->pld_ary[i - 1].pld_rptr; reg_end = pdi->pld_ary[i - 1].pld_wptr; offset = 0; } w = (ushort_t *)(reg_start + offset); mlen = reg_end - (uchar_t *)w; if (mlen > 0 && byteleft) { /* * There is a byte left from the last * segment; add it into the checksum. * Don't have to worry about a carry- * out here because we make sure that * high part of (32 bit) sum is small * below. */ #ifdef _LITTLE_ENDIAN sum += *(uchar_t *)w << 8; #else sum += *(uchar_t *)w; #endif w = (ushort_t *)((char *)w + 1); mlen--; byteleft = B_FALSE; } if (mlen == 0) continue; if (is_even(w)) { sum = ip_ocsum(w, mlen >> 1, sum); w += mlen >> 1; /* * If we had an odd number of bytes, * then the last byte goes in the high * part of the sum, and we take the * first byte to the low part of the sum * the next time around the loop. */ if (is_odd(mlen)) { #ifdef _LITTLE_ENDIAN sum += *(uchar_t *)w; #else sum += *(uchar_t *)w << 8; #endif byteleft = B_TRUE; } } else { ushort_t swsum; #ifdef _LITTLE_ENDIAN sum += *(uchar_t *)w; #else sum += *(uchar_t *)w << 8; #endif mlen--; w = (ushort_t *)(1 + (uintptr_t)w); /* Do a separate checksum and copy operation */ swsum = ip_ocsum(w, mlen >> 1, 0); sum += ((swsum << 8) & 0xffff) | (swsum >> 8); w += mlen >> 1; /* * If we had an even number of bytes, * then the last byte goes in the low * part of the sum. Otherwise we had an * odd number of bytes and we take the first * byte to the low part of the sum the * next time around the loop. */ if (is_odd(mlen)) { #ifdef _LITTLE_ENDIAN sum += *(uchar_t *)w << 8; #else sum += *(uchar_t *)w; #endif } else { byteleft = B_TRUE; } } } /* * Add together high and low parts of sum and carry to get cksum. * Have to be careful to not drop the last carry here. */ sum = (sum & 0xffff) + (sum >> 16); sum = (sum & 0xffff) + (sum >> 16); return (sum); } /* Return the IP checksum for the IP header at "iph". */ uint16_t ip_csum_hdr(ipha_t *ipha) { uint16_t *uph; uint32_t sum; int opt_len; opt_len = (ipha->ipha_version_and_hdr_length & 0xF) - IP_SIMPLE_HDR_LENGTH_IN_WORDS; uph = (uint16_t *)ipha; sum = uph[0] + uph[1] + uph[2] + uph[3] + uph[4] + uph[5] + uph[6] + uph[7] + uph[8] + uph[9]; if (opt_len > 0) { do { sum += uph[10]; sum += uph[11]; uph += 2; } while (--opt_len); } sum = (sum & 0xFFFF) + (sum >> 16); sum = ~(sum + (sum >> 16)) & 0xFFFF; if (sum == 0xffff) sum = 0; return ((uint16_t)sum); }