1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 /* Copyright (c) 1990 Mentat Inc. */ 26 27 #include <sys/types.h> 28 #include <sys/inttypes.h> 29 #include <sys/systm.h> 30 #include <sys/stream.h> 31 #include <sys/strsun.h> 32 #include <sys/debug.h> 33 #include <sys/ddi.h> 34 #include <sys/vtrace.h> 35 #include <inet/sctp_crc32.h> 36 #include <inet/ip.h> 37 38 #include <sys/multidata.h> 39 #include <sys/multidata_impl.h> 40 41 extern unsigned int ip_ocsum(ushort_t *address, int halfword_count, 42 unsigned int sum); 43 44 /* 45 * Checksum routine for Internet Protocol family headers. 46 * This routine is very heavily used in the network 47 * code and should be modified for each CPU to be as fast as possible. 48 */ 49 50 #define mp_len(mp) ((mp)->b_wptr - (mp)->b_rptr) 51 52 /* 53 * Even/Odd checks. Usually it is performed on pointers but may be 54 * used on integers as well. uintptr_t is long enough to hold both 55 * integer and pointer. 56 */ 57 #define is_odd(p) (((uintptr_t)(p) & 0x1) != 0) 58 #define is_even(p) (!is_odd(p)) 59 60 61 #ifdef ZC_TEST 62 /* 63 * Disable the TCP s/w cksum. 64 * XXX - This is just a hack for testing purpose. Don't use it for 65 * anything else! 66 */ 67 int noswcksum = 0; 68 #endif 69 /* 70 * Note: this does not ones-complement the result since it is used 71 * when computing partial checksums. 72 * For nonSTRUIO_IP mblks, assumes mp->b_rptr+offset is 16 bit aligned. 73 * For STRUIO_IP mblks, assumes mp->b_datap->db_struiobase is 16 bit aligned. 74 * 75 * Note: for STRUIO_IP special mblks some data may have been previously 76 * checksumed, this routine will handle additional data prefixed within 77 * an mblk or b_cont (chained) mblk(s). This routine will also handle 78 * suffixed b_cont mblk(s) and data suffixed within an mblk. 79 */ 80 unsigned int 81 ip_cksum(mblk_t *mp, int offset, uint_t sum) 82 { 83 ushort_t *w; 84 ssize_t mlen; 85 int pmlen; 86 mblk_t *pmp; 87 dblk_t *dp = mp->b_datap; 88 ushort_t psum = 0; 89 90 #ifdef ZC_TEST 91 if (noswcksum) 92 return (0xffff); 93 #endif 94 ASSERT(dp); 95 96 if (mp->b_cont == NULL) { 97 /* 98 * May be fast-path, only one mblk. 99 */ 100 w = (ushort_t *)(mp->b_rptr + offset); 101 if (dp->db_struioflag & STRUIO_IP) { 102 /* 103 * Checksum any data not already done by 104 * the caller and add in any partial checksum. 105 */ 106 if ((offset > dp->db_cksumstart) || 107 mp->b_wptr != (uchar_t *)(mp->b_rptr + 108 dp->db_cksumend)) { 109 /* 110 * Mblk data pointers aren't inclusive 111 * of uio data, so disregard checksum. 112 * 113 * not using all of data in dblk make sure 114 * not use to use the precalculated checksum 115 * in this case. 116 */ 117 dp->db_struioflag &= ~STRUIO_IP; 118 goto norm; 119 } 120 ASSERT(mp->b_wptr == (mp->b_rptr + dp->db_cksumend)); 121 psum = *(ushort_t *)dp->db_struioun.data; 122 if ((mlen = dp->db_cksumstart - offset) < 0) 123 mlen = 0; 124 if (is_odd(mlen)) 125 goto slow; 126 if (mlen && dp->db_cksumstart != dp->db_cksumstuff && 127 dp->db_cksumend != dp->db_cksumstuff) { 128 /* 129 * There is prefix data to do and some uio 130 * data has already been checksumed and there 131 * is more uio data to do, so do the prefix 132 * data first, then do the remainder of the 133 * uio data. 134 */ 135 sum = ip_ocsum(w, mlen >> 1, sum); 136 w = (ushort_t *)(mp->b_rptr + 137 dp->db_cksumstuff); 138 if (is_odd(w)) { 139 pmp = mp; 140 goto slow1; 141 } 142 mlen = dp->db_cksumend - dp->db_cksumstuff; 143 } else if (dp->db_cksumend != dp->db_cksumstuff) { 144 /* 145 * There may be uio data to do, if there is 146 * prefix data to do then add in all of the 147 * uio data (if any) to do, else just do any 148 * uio data. 149 */ 150 if (mlen) 151 mlen += dp->db_cksumend 152 - dp->db_cksumstuff; 153 else { 154 w = (ushort_t *)(mp->b_rptr + 155 dp->db_cksumstuff); 156 if (is_odd(w)) 157 goto slow; 158 mlen = dp->db_cksumend 159 - dp->db_cksumstuff; 160 } 161 } else if (mlen == 0) 162 return (psum); 163 164 if (is_odd(mlen)) 165 goto slow; 166 sum += psum; 167 } else { 168 /* 169 * Checksum all data not already done by the caller. 170 */ 171 norm: 172 mlen = mp->b_wptr - (uchar_t *)w; 173 if (is_odd(mlen)) 174 goto slow; 175 } 176 ASSERT(is_even(w)); 177 ASSERT(is_even(mlen)); 178 return (ip_ocsum(w, mlen >> 1, sum)); 179 } 180 if (dp->db_struioflag & STRUIO_IP) 181 psum = *(ushort_t *)dp->db_struioun.data; 182 slow: 183 pmp = 0; 184 slow1: 185 mlen = 0; 186 pmlen = 0; 187 for (; ; ) { 188 /* 189 * Each trip around loop adds in word(s) from one mbuf segment 190 * (except for when pmp == mp, then its two partial trips). 191 */ 192 w = (ushort_t *)(mp->b_rptr + offset); 193 if (pmp) { 194 /* 195 * This is the second trip around for this mblk. 196 */ 197 pmp = 0; 198 mlen = 0; 199 goto douio; 200 } else if (dp->db_struioflag & STRUIO_IP) { 201 /* 202 * Checksum any data not already done by the 203 * caller and add in any partial checksum. 204 */ 205 if ((offset > dp->db_cksumstart) || 206 mp->b_wptr != (uchar_t *)(mp->b_rptr + 207 dp->db_cksumend)) { 208 /* 209 * Mblk data pointers aren't inclusive 210 * of uio data, so disregard checksum. 211 * 212 * not using all of data in dblk make sure 213 * not use to use the precalculated checksum 214 * in this case. 215 */ 216 dp->db_struioflag &= ~STRUIO_IP; 217 goto snorm; 218 } 219 ASSERT(mp->b_wptr == (mp->b_rptr + dp->db_cksumend)); 220 if ((mlen = dp->db_cksumstart - offset) < 0) 221 mlen = 0; 222 if (mlen && dp->db_cksumstart != dp->db_cksumstuff) { 223 /* 224 * There is prefix data too do and some 225 * uio data has already been checksumed, 226 * so do the prefix data only this trip. 227 */ 228 pmp = mp; 229 } else { 230 /* 231 * Add in any partial cksum (if any) and 232 * do the remainder of the uio data. 233 */ 234 int odd; 235 douio: 236 odd = is_odd(dp->db_cksumstuff - 237 dp->db_cksumstart); 238 if (pmlen == -1) { 239 /* 240 * Previous mlen was odd, so swap 241 * the partial checksum bytes. 242 */ 243 sum += ((psum << 8) & 0xffff) 244 | (psum >> 8); 245 if (odd) 246 pmlen = 0; 247 } else { 248 sum += psum; 249 if (odd) 250 pmlen = -1; 251 } 252 if (dp->db_cksumend != dp->db_cksumstuff) { 253 /* 254 * If prefix data to do and then all 255 * the uio data nees to be checksumed, 256 * else just do any uio data. 257 */ 258 if (mlen) 259 mlen += dp->db_cksumend 260 - dp->db_cksumstuff; 261 else { 262 w = (ushort_t *)(mp->b_rptr + 263 dp->db_cksumstuff); 264 mlen = dp->db_cksumend - 265 dp->db_cksumstuff; 266 } 267 } 268 } 269 } else { 270 /* 271 * Checksum all of the mblk data. 272 */ 273 snorm: 274 mlen = mp->b_wptr - (uchar_t *)w; 275 } 276 277 mp = mp->b_cont; 278 if (mlen > 0 && pmlen == -1) { 279 /* 280 * There is a byte left from the last 281 * segment; add it into the checksum. 282 * Don't have to worry about a carry- 283 * out here because we make sure that 284 * high part of (32 bit) sum is small 285 * below. 286 */ 287 #ifdef _LITTLE_ENDIAN 288 sum += *(uchar_t *)w << 8; 289 #else 290 sum += *(uchar_t *)w; 291 #endif 292 w = (ushort_t *)((char *)w + 1); 293 mlen--; 294 pmlen = 0; 295 } 296 if (mlen > 0) { 297 if (is_even(w)) { 298 sum = ip_ocsum(w, mlen>>1, sum); 299 w += mlen>>1; 300 /* 301 * If we had an odd number of bytes, 302 * then the last byte goes in the high 303 * part of the sum, and we take the 304 * first byte to the low part of the sum 305 * the next time around the loop. 306 */ 307 if (is_odd(mlen)) { 308 #ifdef _LITTLE_ENDIAN 309 sum += *(uchar_t *)w; 310 #else 311 sum += *(uchar_t *)w << 8; 312 #endif 313 pmlen = -1; 314 } 315 } else { 316 ushort_t swsum; 317 #ifdef _LITTLE_ENDIAN 318 sum += *(uchar_t *)w; 319 #else 320 sum += *(uchar_t *)w << 8; 321 #endif 322 mlen--; 323 w = (ushort_t *)(1 + (uintptr_t)w); 324 325 /* Do a separate checksum and copy operation */ 326 swsum = ip_ocsum(w, mlen>>1, 0); 327 sum += ((swsum << 8) & 0xffff) | (swsum >> 8); 328 w += mlen>>1; 329 /* 330 * If we had an even number of bytes, 331 * then the last byte goes in the low 332 * part of the sum. Otherwise we had an 333 * odd number of bytes and we take the first 334 * byte to the low part of the sum the 335 * next time around the loop. 336 */ 337 if (is_odd(mlen)) { 338 #ifdef _LITTLE_ENDIAN 339 sum += *(uchar_t *)w << 8; 340 #else 341 sum += *(uchar_t *)w; 342 #endif 343 } 344 else 345 pmlen = -1; 346 } 347 } 348 /* 349 * Locate the next block with some data. 350 * If there is a word split across a boundary we 351 * will wrap to the top with mlen == -1 and 352 * then add it in shifted appropriately. 353 */ 354 offset = 0; 355 if (! pmp) { 356 for (; ; ) { 357 if (mp == 0) { 358 goto done; 359 } 360 if (mp_len(mp)) 361 break; 362 mp = mp->b_cont; 363 } 364 dp = mp->b_datap; 365 if (dp->db_struioflag & STRUIO_IP) 366 psum = *(ushort_t *)dp->db_struioun.data; 367 } else 368 mp = pmp; 369 } 370 done: 371 /* 372 * Add together high and low parts of sum 373 * and carry to get cksum. 374 * Have to be careful to not drop the last 375 * carry here. 376 */ 377 sum = (sum & 0xFFFF) + (sum >> 16); 378 sum = (sum & 0xFFFF) + (sum >> 16); 379 TRACE_3(TR_FAC_IP, TR_IP_CKSUM_END, 380 "ip_cksum_end:(%S) type %d (%X)", "ip_cksum", 1, sum); 381 return (sum); 382 } 383 384 uint32_t 385 sctp_cksum(mblk_t *mp, int offset) 386 { 387 uint32_t crc32; 388 uchar_t *p = NULL; 389 390 crc32 = 0xFFFFFFFF; 391 p = mp->b_rptr + offset; 392 crc32 = sctp_crc32(crc32, p, mp->b_wptr - p); 393 for (mp = mp->b_cont; mp != NULL; mp = mp->b_cont) { 394 crc32 = sctp_crc32(crc32, mp->b_rptr, MBLKL(mp)); 395 } 396 397 /* Complement the result */ 398 crc32 = ~crc32; 399 400 return (crc32); 401 } 402 403 /* 404 * Routine to compute Internet checksum (16-bit 1's complement) of a given 405 * Multidata packet descriptor. As in the non-Multidata routine, this doesn't 406 * 1's complement the result, such that it may be used to compute partial 407 * checksums. Since it works on buffer spans rather than mblks, this routine 408 * does not handle existing partial checksum value as in the STRUIO_IP special 409 * mblk case (supporting this is rather trivial, but is perhaps of no use at 410 * the moment unless synchronous streams and delayed checksum calculation are 411 * revived.) 412 * 413 * Note also here that the given Multidata packet descriptor must refer to 414 * a header buffer, i.e. it must have a header fragment. In addition, the 415 * offset must lie within the boundary of the header fragment. For the 416 * outbound tcp (MDT) case, this will not be an issue because the stack 417 * ensures that such conditions are met, and that there is no need whatsoever 418 * to compute partial checksums on an arbitrary offset that is not part of 419 * the header fragment. We may need to revisit this routine to handle all 420 * cases of the inbound (MDR) case, especially when we need to perform partial 421 * checksum calculation due to padded bytes (non-zeroes) in the frame. 422 */ 423 uint_t 424 ip_md_cksum(pdesc_t *pd, int offset, uint_t sum) 425 { 426 pdescinfo_t *pdi = &pd->pd_pdi; 427 uchar_t *reg_start, *reg_end; 428 ssize_t mlen, i; 429 ushort_t *w; 430 boolean_t byteleft = B_FALSE; 431 432 ASSERT((pdi->flags & PDESC_HAS_REF) != 0); 433 ASSERT(pdi->hdr_rptr != NULL && pdi->hdr_wptr != NULL); 434 ASSERT(offset <= PDESC_HDRL(pdi)); 435 436 for (i = 0; i < pdi->pld_cnt + 1; i++) { 437 if (i == 0) { 438 reg_start = pdi->hdr_rptr; 439 reg_end = pdi->hdr_wptr; 440 } else { 441 reg_start = pdi->pld_ary[i - 1].pld_rptr; 442 reg_end = pdi->pld_ary[i - 1].pld_wptr; 443 offset = 0; 444 } 445 446 w = (ushort_t *)(reg_start + offset); 447 mlen = reg_end - (uchar_t *)w; 448 449 if (mlen > 0 && byteleft) { 450 /* 451 * There is a byte left from the last 452 * segment; add it into the checksum. 453 * Don't have to worry about a carry- 454 * out here because we make sure that 455 * high part of (32 bit) sum is small 456 * below. 457 */ 458 #ifdef _LITTLE_ENDIAN 459 sum += *(uchar_t *)w << 8; 460 #else 461 sum += *(uchar_t *)w; 462 #endif 463 w = (ushort_t *)((char *)w + 1); 464 mlen--; 465 byteleft = B_FALSE; 466 } 467 468 if (mlen == 0) 469 continue; 470 471 if (is_even(w)) { 472 sum = ip_ocsum(w, mlen >> 1, sum); 473 w += mlen >> 1; 474 /* 475 * If we had an odd number of bytes, 476 * then the last byte goes in the high 477 * part of the sum, and we take the 478 * first byte to the low part of the sum 479 * the next time around the loop. 480 */ 481 if (is_odd(mlen)) { 482 #ifdef _LITTLE_ENDIAN 483 sum += *(uchar_t *)w; 484 #else 485 sum += *(uchar_t *)w << 8; 486 #endif 487 byteleft = B_TRUE; 488 } 489 } else { 490 ushort_t swsum; 491 #ifdef _LITTLE_ENDIAN 492 sum += *(uchar_t *)w; 493 #else 494 sum += *(uchar_t *)w << 8; 495 #endif 496 mlen--; 497 w = (ushort_t *)(1 + (uintptr_t)w); 498 499 /* Do a separate checksum and copy operation */ 500 swsum = ip_ocsum(w, mlen >> 1, 0); 501 sum += ((swsum << 8) & 0xffff) | (swsum >> 8); 502 w += mlen >> 1; 503 /* 504 * If we had an even number of bytes, 505 * then the last byte goes in the low 506 * part of the sum. Otherwise we had an 507 * odd number of bytes and we take the first 508 * byte to the low part of the sum the 509 * next time around the loop. 510 */ 511 if (is_odd(mlen)) { 512 #ifdef _LITTLE_ENDIAN 513 sum += *(uchar_t *)w << 8; 514 #else 515 sum += *(uchar_t *)w; 516 #endif 517 } else { 518 byteleft = B_TRUE; 519 } 520 } 521 } 522 523 /* 524 * Add together high and low parts of sum and carry to get cksum. 525 * Have to be careful to not drop the last carry here. 526 */ 527 sum = (sum & 0xffff) + (sum >> 16); 528 sum = (sum & 0xffff) + (sum >> 16); 529 530 return (sum); 531 } 532 533 /* Return the IP checksum for the IP header at "iph". */ 534 uint16_t 535 ip_csum_hdr(ipha_t *ipha) 536 { 537 uint16_t *uph; 538 uint32_t sum; 539 int opt_len; 540 541 opt_len = (ipha->ipha_version_and_hdr_length & 0xF) - 542 IP_SIMPLE_HDR_LENGTH_IN_WORDS; 543 uph = (uint16_t *)ipha; 544 sum = uph[0] + uph[1] + uph[2] + uph[3] + uph[4] + 545 uph[5] + uph[6] + uph[7] + uph[8] + uph[9]; 546 if (opt_len > 0) { 547 do { 548 sum += uph[10]; 549 sum += uph[11]; 550 uph += 2; 551 } while (--opt_len); 552 } 553 sum = (sum & 0xFFFF) + (sum >> 16); 554 sum = ~(sum + (sum >> 16)) & 0xFFFF; 555 if (sum == 0xffff) 556 sum = 0; 557 return ((uint16_t)sum); 558 } 559