xref: /titanic_51/usr/src/uts/common/os/ip_cksum.c (revision bbaa8b60dd95d714741fc474adad3cf710ef4efd)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 /* Copyright (c) 1990 Mentat Inc. */
26 
27 #include <sys/types.h>
28 #include <sys/inttypes.h>
29 #include <sys/systm.h>
30 #include <sys/stream.h>
31 #include <sys/strsun.h>
32 #include <sys/debug.h>
33 #include <sys/ddi.h>
34 #include <sys/vtrace.h>
35 #include <inet/sctp_crc32.h>
36 #include <inet/ip.h>
37 
38 #include <sys/multidata.h>
39 #include <sys/multidata_impl.h>
40 
41 extern unsigned int 	ip_ocsum(ushort_t *address, int halfword_count,
42     unsigned int sum);
43 
44 /*
45  * Checksum routine for Internet Protocol family headers.
46  * This routine is very heavily used in the network
47  * code and should be modified for each CPU to be as fast as possible.
48  */
49 
50 #define	mp_len(mp) ((mp)->b_wptr - (mp)->b_rptr)
51 
52 /*
53  * Even/Odd checks. Usually it is performed on pointers but may be
54  * used on integers as well. uintptr_t is long enough to hold both
55  * integer and pointer.
56  */
57 #define	is_odd(p) (((uintptr_t)(p) & 0x1) != 0)
58 #define	is_even(p) (!is_odd(p))
59 
60 
61 #ifdef ZC_TEST
62 /*
63  * Disable the TCP s/w cksum.
64  * XXX - This is just a hack for testing purpose. Don't use it for
65  * anything else!
66  */
67 int noswcksum = 0;
68 #endif
69 /*
70  * Note: this does not ones-complement the result since it is used
71  * when computing partial checksums.
72  * For nonSTRUIO_IP mblks, assumes mp->b_rptr+offset is 16 bit aligned.
73  * For STRUIO_IP mblks, assumes mp->b_datap->db_struiobase is 16 bit aligned.
74  *
75  * Note: for STRUIO_IP special mblks some data may have been previously
76  *	 checksumed, this routine will handle additional data prefixed within
77  *	 an mblk or b_cont (chained) mblk(s). This routine will also handle
78  *	 suffixed b_cont mblk(s) and data suffixed within an mblk.
79  */
80 unsigned int
81 ip_cksum(mblk_t *mp, int offset, uint_t sum)
82 {
83 	ushort_t *w;
84 	ssize_t	mlen;
85 	int pmlen;
86 	mblk_t *pmp;
87 	dblk_t *dp = mp->b_datap;
88 	ushort_t psum = 0;
89 
90 #ifdef ZC_TEST
91 	if (noswcksum)
92 		return (0xffff);
93 #endif
94 	ASSERT(dp);
95 
96 	if (mp->b_cont == NULL) {
97 		/*
98 		 * May be fast-path, only one mblk.
99 		 */
100 		w = (ushort_t *)(mp->b_rptr + offset);
101 		if (dp->db_struioflag & STRUIO_IP) {
102 			/*
103 			 * Checksum any data not already done by
104 			 * the caller and add in any partial checksum.
105 			 */
106 			if ((offset > dp->db_cksumstart) ||
107 			    mp->b_wptr != (uchar_t *)(mp->b_rptr +
108 			    dp->db_cksumend)) {
109 				/*
110 				 * Mblk data pointers aren't inclusive
111 				 * of uio data, so disregard checksum.
112 				 *
113 				 * not using all of data in dblk make sure
114 				 * not use to use the precalculated checksum
115 				 * in this case.
116 				 */
117 				dp->db_struioflag &= ~STRUIO_IP;
118 				goto norm;
119 			}
120 			ASSERT(mp->b_wptr == (mp->b_rptr + dp->db_cksumend));
121 			psum = *(ushort_t *)dp->db_struioun.data;
122 			if ((mlen = dp->db_cksumstart - offset) < 0)
123 				mlen = 0;
124 			if (is_odd(mlen))
125 				goto slow;
126 			if (mlen && dp->db_cksumstart != dp->db_cksumstuff &&
127 			    dp->db_cksumend != dp->db_cksumstuff) {
128 				/*
129 				 * There is prefix data to do and some uio
130 				 * data has already been checksumed and there
131 				 * is more uio data to do, so do the prefix
132 				 * data first, then do the remainder of the
133 				 * uio data.
134 				 */
135 				sum = ip_ocsum(w, mlen >> 1, sum);
136 				w = (ushort_t *)(mp->b_rptr +
137 				    dp->db_cksumstuff);
138 				if (is_odd(w)) {
139 					pmp = mp;
140 					goto slow1;
141 				}
142 				mlen = dp->db_cksumend - dp->db_cksumstuff;
143 			} else if (dp->db_cksumend != dp->db_cksumstuff) {
144 				/*
145 				 * There may be uio data to do, if there is
146 				 * prefix data to do then add in all of the
147 				 * uio data (if any) to do, else just do any
148 				 * uio data.
149 				 */
150 				if (mlen)
151 					mlen += dp->db_cksumend
152 					    - dp->db_cksumstuff;
153 				else {
154 					w = (ushort_t *)(mp->b_rptr +
155 					    dp->db_cksumstuff);
156 					if (is_odd(w))
157 						goto slow;
158 					mlen = dp->db_cksumend
159 					    - dp->db_cksumstuff;
160 				}
161 			} else if (mlen == 0)
162 				return (psum);
163 
164 			if (is_odd(mlen))
165 				goto slow;
166 			sum += psum;
167 		} else {
168 			/*
169 			 * Checksum all data not already done by the caller.
170 			 */
171 		norm:
172 			mlen = mp->b_wptr - (uchar_t *)w;
173 			if (is_odd(mlen))
174 				goto slow;
175 		}
176 		ASSERT(is_even(w));
177 		ASSERT(is_even(mlen));
178 		return (ip_ocsum(w, mlen >> 1, sum));
179 	}
180 	if (dp->db_struioflag & STRUIO_IP)
181 		psum = *(ushort_t *)dp->db_struioun.data;
182 slow:
183 	pmp = 0;
184 slow1:
185 	mlen = 0;
186 	pmlen = 0;
187 	for (; ; ) {
188 		/*
189 		 * Each trip around loop adds in word(s) from one mbuf segment
190 		 * (except for when pmp == mp, then its two partial trips).
191 		 */
192 		w = (ushort_t *)(mp->b_rptr + offset);
193 		if (pmp) {
194 			/*
195 			 * This is the second trip around for this mblk.
196 			 */
197 			pmp = 0;
198 			mlen = 0;
199 			goto douio;
200 		} else if (dp->db_struioflag & STRUIO_IP) {
201 			/*
202 			 * Checksum any data not already done by the
203 			 * caller and add in any partial checksum.
204 			 */
205 			if ((offset > dp->db_cksumstart) ||
206 			    mp->b_wptr != (uchar_t *)(mp->b_rptr +
207 			    dp->db_cksumend)) {
208 				/*
209 				 * Mblk data pointers aren't inclusive
210 				 * of uio data, so disregard checksum.
211 				 *
212 				 * not using all of data in dblk make sure
213 				 * not use to use the precalculated checksum
214 				 * in this case.
215 				 */
216 				dp->db_struioflag &= ~STRUIO_IP;
217 				goto snorm;
218 			}
219 			ASSERT(mp->b_wptr == (mp->b_rptr + dp->db_cksumend));
220 			if ((mlen = dp->db_cksumstart - offset) < 0)
221 				mlen = 0;
222 			if (mlen && dp->db_cksumstart != dp->db_cksumstuff) {
223 				/*
224 				 * There is prefix data too do and some
225 				 * uio data has already been checksumed,
226 				 * so do the prefix data only this trip.
227 				 */
228 				pmp = mp;
229 			} else {
230 				/*
231 				 * Add in any partial cksum (if any) and
232 				 * do the remainder of the uio data.
233 				 */
234 				int odd;
235 			douio:
236 				odd = is_odd(dp->db_cksumstuff -
237 				    dp->db_cksumstart);
238 				if (pmlen == -1) {
239 					/*
240 					 * Previous mlen was odd, so swap
241 					 * the partial checksum bytes.
242 					 */
243 					sum += ((psum << 8) & 0xffff)
244 					    | (psum >> 8);
245 					if (odd)
246 						pmlen = 0;
247 				} else {
248 					sum += psum;
249 					if (odd)
250 						pmlen = -1;
251 				}
252 				if (dp->db_cksumend != dp->db_cksumstuff) {
253 					/*
254 					 * If prefix data to do and then all
255 					 * the uio data nees to be checksumed,
256 					 * else just do any uio data.
257 					 */
258 					if (mlen)
259 						mlen += dp->db_cksumend
260 						    - dp->db_cksumstuff;
261 					else {
262 						w = (ushort_t *)(mp->b_rptr +
263 						    dp->db_cksumstuff);
264 						mlen = dp->db_cksumend -
265 						    dp->db_cksumstuff;
266 					}
267 				}
268 			}
269 		} else {
270 			/*
271 			 * Checksum all of the mblk data.
272 			 */
273 		snorm:
274 			mlen = mp->b_wptr - (uchar_t *)w;
275 		}
276 
277 		mp = mp->b_cont;
278 		if (mlen > 0 && pmlen == -1) {
279 			/*
280 			 * There is a byte left from the last
281 			 * segment; add it into the checksum.
282 			 * Don't have to worry about a carry-
283 			 * out here because we make sure that
284 			 * high part of (32 bit) sum is small
285 			 * below.
286 			 */
287 #ifdef _LITTLE_ENDIAN
288 			sum += *(uchar_t *)w << 8;
289 #else
290 			sum += *(uchar_t *)w;
291 #endif
292 			w = (ushort_t *)((char *)w + 1);
293 			mlen--;
294 			pmlen = 0;
295 		}
296 		if (mlen > 0) {
297 			if (is_even(w)) {
298 				sum = ip_ocsum(w, mlen>>1, sum);
299 				w += mlen>>1;
300 				/*
301 				 * If we had an odd number of bytes,
302 				 * then the last byte goes in the high
303 				 * part of the sum, and we take the
304 				 * first byte to the low part of the sum
305 				 * the next time around the loop.
306 				 */
307 				if (is_odd(mlen)) {
308 #ifdef _LITTLE_ENDIAN
309 					sum += *(uchar_t *)w;
310 #else
311 					sum += *(uchar_t *)w << 8;
312 #endif
313 					pmlen = -1;
314 				}
315 			} else {
316 				ushort_t swsum;
317 #ifdef _LITTLE_ENDIAN
318 				sum += *(uchar_t *)w;
319 #else
320 				sum += *(uchar_t *)w << 8;
321 #endif
322 				mlen--;
323 				w = (ushort_t *)(1 + (uintptr_t)w);
324 
325 				/* Do a separate checksum and copy operation */
326 				swsum = ip_ocsum(w, mlen>>1, 0);
327 				sum += ((swsum << 8) & 0xffff) | (swsum >> 8);
328 				w += mlen>>1;
329 				/*
330 				 * If we had an even number of bytes,
331 				 * then the last byte goes in the low
332 				 * part of the sum.  Otherwise we had an
333 				 * odd number of bytes and we take the first
334 				 * byte to the low part of the sum the
335 				 * next time around the loop.
336 				 */
337 				if (is_odd(mlen)) {
338 #ifdef _LITTLE_ENDIAN
339 					sum += *(uchar_t *)w << 8;
340 #else
341 					sum += *(uchar_t *)w;
342 #endif
343 				}
344 				else
345 					pmlen = -1;
346 			}
347 		}
348 		/*
349 		 * Locate the next block with some data.
350 		 * If there is a word split across a boundary we
351 		 * will wrap to the top with mlen == -1 and
352 		 * then add it in shifted appropriately.
353 		 */
354 		offset = 0;
355 		if (! pmp) {
356 			for (; ; ) {
357 				if (mp == 0) {
358 					goto done;
359 				}
360 				if (mp_len(mp))
361 					break;
362 				mp = mp->b_cont;
363 			}
364 			dp = mp->b_datap;
365 			if (dp->db_struioflag & STRUIO_IP)
366 				psum = *(ushort_t *)dp->db_struioun.data;
367 		} else
368 			mp = pmp;
369 	}
370 done:
371 	/*
372 	 * Add together high and low parts of sum
373 	 * and carry to get cksum.
374 	 * Have to be careful to not drop the last
375 	 * carry here.
376 	 */
377 	sum = (sum & 0xFFFF) + (sum >> 16);
378 	sum = (sum & 0xFFFF) + (sum >> 16);
379 	TRACE_3(TR_FAC_IP, TR_IP_CKSUM_END,
380 	    "ip_cksum_end:(%S) type %d (%X)", "ip_cksum", 1, sum);
381 	return (sum);
382 }
383 
384 uint32_t
385 sctp_cksum(mblk_t *mp, int offset)
386 {
387 	uint32_t crc32;
388 	uchar_t *p = NULL;
389 
390 	crc32 = 0xFFFFFFFF;
391 	p = mp->b_rptr + offset;
392 	crc32 = sctp_crc32(crc32, p, mp->b_wptr - p);
393 	for (mp = mp->b_cont; mp != NULL; mp = mp->b_cont) {
394 		crc32 = sctp_crc32(crc32, mp->b_rptr, MBLKL(mp));
395 	}
396 
397 	/* Complement the result */
398 	crc32 = ~crc32;
399 
400 	return (crc32);
401 }
402 
403 /*
404  * Routine to compute Internet checksum (16-bit 1's complement) of a given
405  * Multidata packet descriptor.  As in the non-Multidata routine, this doesn't
406  * 1's complement the result, such that it may be used to compute partial
407  * checksums.  Since it works on buffer spans rather than mblks, this routine
408  * does not handle existing partial checksum value as in the STRUIO_IP special
409  * mblk case (supporting this is rather trivial, but is perhaps of no use at
410  * the moment unless synchronous streams and delayed checksum calculation are
411  * revived.)
412  *
413  * Note also here that the given Multidata packet descriptor must refer to
414  * a header buffer, i.e. it must have a header fragment.  In addition, the
415  * offset must lie within the boundary of the header fragment.  For the
416  * outbound tcp (MDT) case, this will not be an issue because the stack
417  * ensures that such conditions are met, and that there is no need whatsoever
418  * to compute partial checksums on an arbitrary offset that is not part of
419  * the header fragment.  We may need to revisit this routine to handle all
420  * cases of the inbound (MDR) case, especially when we need to perform partial
421  * checksum calculation due to padded bytes (non-zeroes) in the frame.
422  */
423 uint_t
424 ip_md_cksum(pdesc_t *pd, int offset, uint_t sum)
425 {
426 	pdescinfo_t	*pdi = &pd->pd_pdi;
427 	uchar_t		*reg_start, *reg_end;
428 	ssize_t		mlen, i;
429 	ushort_t	*w;
430 	boolean_t	byteleft = B_FALSE;
431 
432 	ASSERT((pdi->flags & PDESC_HAS_REF) != 0);
433 	ASSERT(pdi->hdr_rptr != NULL && pdi->hdr_wptr != NULL);
434 	ASSERT(offset <= PDESC_HDRL(pdi));
435 
436 	for (i = 0; i < pdi->pld_cnt + 1; i++) {
437 		if (i == 0) {
438 			reg_start = pdi->hdr_rptr;
439 			reg_end = pdi->hdr_wptr;
440 		} else {
441 			reg_start = pdi->pld_ary[i - 1].pld_rptr;
442 			reg_end = pdi->pld_ary[i - 1].pld_wptr;
443 			offset = 0;
444 		}
445 
446 		w = (ushort_t *)(reg_start + offset);
447 		mlen = reg_end - (uchar_t *)w;
448 
449 		if (mlen > 0 && byteleft) {
450 			/*
451 			 * There is a byte left from the last
452 			 * segment; add it into the checksum.
453 			 * Don't have to worry about a carry-
454 			 * out here because we make sure that
455 			 * high part of (32 bit) sum is small
456 			 * below.
457 			 */
458 #ifdef _LITTLE_ENDIAN
459 			sum += *(uchar_t *)w << 8;
460 #else
461 			sum += *(uchar_t *)w;
462 #endif
463 			w = (ushort_t *)((char *)w + 1);
464 			mlen--;
465 			byteleft = B_FALSE;
466 		}
467 
468 		if (mlen == 0)
469 			continue;
470 
471 		if (is_even(w)) {
472 			sum = ip_ocsum(w, mlen >> 1, sum);
473 			w += mlen >> 1;
474 			/*
475 			 * If we had an odd number of bytes,
476 			 * then the last byte goes in the high
477 			 * part of the sum, and we take the
478 			 * first byte to the low part of the sum
479 			 * the next time around the loop.
480 			 */
481 			if (is_odd(mlen)) {
482 #ifdef _LITTLE_ENDIAN
483 				sum += *(uchar_t *)w;
484 #else
485 				sum += *(uchar_t *)w << 8;
486 #endif
487 				byteleft = B_TRUE;
488 			}
489 		} else {
490 			ushort_t swsum;
491 #ifdef _LITTLE_ENDIAN
492 			sum += *(uchar_t *)w;
493 #else
494 			sum += *(uchar_t *)w << 8;
495 #endif
496 			mlen--;
497 			w = (ushort_t *)(1 + (uintptr_t)w);
498 
499 			/* Do a separate checksum and copy operation */
500 			swsum = ip_ocsum(w, mlen >> 1, 0);
501 			sum += ((swsum << 8) & 0xffff) | (swsum >> 8);
502 			w += mlen >> 1;
503 			/*
504 			 * If we had an even number of bytes,
505 			 * then the last byte goes in the low
506 			 * part of the sum.  Otherwise we had an
507 			 * odd number of bytes and we take the first
508 			 * byte to the low part of the sum the
509 			 * next time around the loop.
510 			 */
511 			if (is_odd(mlen)) {
512 #ifdef _LITTLE_ENDIAN
513 				sum += *(uchar_t *)w << 8;
514 #else
515 				sum += *(uchar_t *)w;
516 #endif
517 			} else {
518 				byteleft = B_TRUE;
519 			}
520 		}
521 	}
522 
523 	/*
524 	 * Add together high and low parts of sum and carry to get cksum.
525 	 * Have to be careful to not drop the last carry here.
526 	 */
527 	sum = (sum & 0xffff) + (sum >> 16);
528 	sum = (sum & 0xffff) + (sum >> 16);
529 
530 	return (sum);
531 }
532 
533 /* Return the IP checksum for the IP header at "iph". */
534 uint16_t
535 ip_csum_hdr(ipha_t *ipha)
536 {
537 	uint16_t	*uph;
538 	uint32_t	sum;
539 	int		opt_len;
540 
541 	opt_len = (ipha->ipha_version_and_hdr_length & 0xF) -
542 	    IP_SIMPLE_HDR_LENGTH_IN_WORDS;
543 	uph = (uint16_t *)ipha;
544 	sum = uph[0] + uph[1] + uph[2] + uph[3] + uph[4] +
545 	    uph[5] + uph[6] + uph[7] + uph[8] + uph[9];
546 	if (opt_len > 0) {
547 		do {
548 			sum += uph[10];
549 			sum += uph[11];
550 			uph += 2;
551 		} while (--opt_len);
552 	}
553 	sum = (sum & 0xFFFF) + (sum >> 16);
554 	sum = ~(sum + (sum >> 16)) & 0xFFFF;
555 	if (sum == 0xffff)
556 		sum = 0;
557 	return ((uint16_t)sum);
558 }
559