1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 /* 27 * MAC Services Module - misc utilities 28 */ 29 30 #include <sys/types.h> 31 #include <sys/mac.h> 32 #include <sys/mac_impl.h> 33 #include <sys/mac_client_priv.h> 34 #include <sys/mac_client_impl.h> 35 #include <sys/mac_soft_ring.h> 36 #include <sys/strsubr.h> 37 #include <sys/strsun.h> 38 #include <sys/vlan.h> 39 #include <sys/pattr.h> 40 #include <sys/pci_tools.h> 41 #include <inet/ip.h> 42 #include <inet/ip_impl.h> 43 #include <inet/ip6.h> 44 #include <sys/vtrace.h> 45 #include <sys/dlpi.h> 46 #include <sys/sunndi.h> 47 #include <inet/ipsec_impl.h> 48 #include <inet/sadb.h> 49 #include <inet/ipsecesp.h> 50 #include <inet/ipsecah.h> 51 52 /* 53 * Copy an mblk, preserving its hardware checksum flags. 54 */ 55 static mblk_t * 56 mac_copymsg_cksum(mblk_t *mp) 57 { 58 mblk_t *mp1; 59 uint32_t start, stuff, end, value, flags; 60 61 mp1 = copymsg(mp); 62 if (mp1 == NULL) 63 return (NULL); 64 65 hcksum_retrieve(mp, NULL, NULL, &start, &stuff, &end, &value, &flags); 66 (void) hcksum_assoc(mp1, NULL, NULL, start, stuff, end, value, 67 flags, KM_NOSLEEP); 68 69 return (mp1); 70 } 71 72 /* 73 * Copy an mblk chain, presenting the hardware checksum flags of the 74 * individual mblks. 75 */ 76 mblk_t * 77 mac_copymsgchain_cksum(mblk_t *mp) 78 { 79 mblk_t *nmp = NULL; 80 mblk_t **nmpp = &nmp; 81 82 for (; mp != NULL; mp = mp->b_next) { 83 if ((*nmpp = mac_copymsg_cksum(mp)) == NULL) { 84 freemsgchain(nmp); 85 return (NULL); 86 } 87 88 nmpp = &((*nmpp)->b_next); 89 } 90 91 return (nmp); 92 } 93 94 /* 95 * Process the specified mblk chain for proper handling of hardware 96 * checksum offload. This routine is invoked for loopback traffic 97 * between MAC clients. 98 * The function handles a NULL mblk chain passed as argument. 99 */ 100 mblk_t * 101 mac_fix_cksum(mblk_t *mp_chain) 102 { 103 mblk_t *mp, *prev = NULL, *new_chain = mp_chain, *mp1; 104 uint32_t flags, start, stuff, end, value; 105 106 for (mp = mp_chain; mp != NULL; prev = mp, mp = mp->b_next) { 107 uint16_t len; 108 uint32_t offset; 109 struct ether_header *ehp; 110 uint16_t sap; 111 112 hcksum_retrieve(mp, NULL, NULL, &start, &stuff, &end, &value, 113 &flags); 114 if (flags == 0) 115 continue; 116 117 /* 118 * Since the processing of checksum offload for loopback 119 * traffic requires modification of the packet contents, 120 * ensure sure that we are always modifying our own copy. 121 */ 122 if (DB_REF(mp) > 1) { 123 mp1 = copymsg(mp); 124 if (mp1 == NULL) 125 continue; 126 mp1->b_next = mp->b_next; 127 mp->b_next = NULL; 128 freemsg(mp); 129 if (prev != NULL) 130 prev->b_next = mp1; 131 else 132 new_chain = mp1; 133 mp = mp1; 134 } 135 136 /* 137 * Ethernet, and optionally VLAN header. 138 */ 139 /* LINTED: improper alignment cast */ 140 ehp = (struct ether_header *)mp->b_rptr; 141 if (ntohs(ehp->ether_type) == VLAN_TPID) { 142 struct ether_vlan_header *evhp; 143 144 ASSERT(MBLKL(mp) >= sizeof (struct ether_vlan_header)); 145 /* LINTED: improper alignment cast */ 146 evhp = (struct ether_vlan_header *)mp->b_rptr; 147 sap = ntohs(evhp->ether_type); 148 offset = sizeof (struct ether_vlan_header); 149 } else { 150 sap = ntohs(ehp->ether_type); 151 offset = sizeof (struct ether_header); 152 } 153 154 if (MBLKL(mp) <= offset) { 155 offset -= MBLKL(mp); 156 if (mp->b_cont == NULL) { 157 /* corrupted packet, skip it */ 158 if (prev != NULL) 159 prev->b_next = mp->b_next; 160 else 161 new_chain = mp->b_next; 162 mp1 = mp->b_next; 163 mp->b_next = NULL; 164 freemsg(mp); 165 mp = mp1; 166 continue; 167 } 168 mp = mp->b_cont; 169 } 170 171 if (flags & (HCK_FULLCKSUM | HCK_IPV4_HDRCKSUM)) { 172 ipha_t *ipha = NULL; 173 174 /* 175 * In order to compute the full and header 176 * checksums, we need to find and parse 177 * the IP and/or ULP headers. 178 */ 179 180 sap = (sap < ETHERTYPE_802_MIN) ? 0 : sap; 181 182 /* 183 * IP header. 184 */ 185 if (sap != ETHERTYPE_IP) 186 continue; 187 188 ASSERT(MBLKL(mp) >= offset + sizeof (ipha_t)); 189 /* LINTED: improper alignment cast */ 190 ipha = (ipha_t *)(mp->b_rptr + offset); 191 192 if (flags & HCK_FULLCKSUM) { 193 ipaddr_t src, dst; 194 uint32_t cksum; 195 uint16_t *up; 196 uint8_t proto; 197 198 /* 199 * Pointer to checksum field in ULP header. 200 */ 201 proto = ipha->ipha_protocol; 202 ASSERT(ipha->ipha_version_and_hdr_length == 203 IP_SIMPLE_HDR_VERSION); 204 205 switch (proto) { 206 case IPPROTO_TCP: 207 /* LINTED: improper alignment cast */ 208 up = IPH_TCPH_CHECKSUMP(ipha, 209 IP_SIMPLE_HDR_LENGTH); 210 break; 211 212 case IPPROTO_UDP: 213 /* LINTED: improper alignment cast */ 214 up = IPH_UDPH_CHECKSUMP(ipha, 215 IP_SIMPLE_HDR_LENGTH); 216 break; 217 218 default: 219 cmn_err(CE_WARN, "mac_fix_cksum: " 220 "unexpected protocol: %d", proto); 221 continue; 222 } 223 224 /* 225 * Pseudo-header checksum. 226 */ 227 src = ipha->ipha_src; 228 dst = ipha->ipha_dst; 229 len = ntohs(ipha->ipha_length) - 230 IP_SIMPLE_HDR_LENGTH; 231 232 cksum = (dst >> 16) + (dst & 0xFFFF) + 233 (src >> 16) + (src & 0xFFFF); 234 cksum += htons(len); 235 236 /* 237 * The checksum value stored in the packet needs 238 * to be correct. Compute it here. 239 */ 240 *up = 0; 241 cksum += (((proto) == IPPROTO_UDP) ? 242 IP_UDP_CSUM_COMP : IP_TCP_CSUM_COMP); 243 cksum = IP_CSUM(mp, IP_SIMPLE_HDR_LENGTH + 244 offset, cksum); 245 *(up) = (uint16_t)(cksum ? cksum : ~cksum); 246 247 /* 248 * Flag the packet so that it appears 249 * that the checksum has already been 250 * verified by the hardware. 251 */ 252 flags &= ~HCK_FULLCKSUM; 253 flags |= HCK_FULLCKSUM_OK; 254 value = 0; 255 } 256 257 if (flags & HCK_IPV4_HDRCKSUM) { 258 ASSERT(ipha != NULL); 259 ipha->ipha_hdr_checksum = 260 (uint16_t)ip_csum_hdr(ipha); 261 flags &= ~HCK_IPV4_HDRCKSUM; 262 flags |= HCK_IPV4_HDRCKSUM_OK; 263 264 } 265 } 266 267 if (flags & HCK_PARTIALCKSUM) { 268 uint16_t *up, partial, cksum; 269 uchar_t *ipp; /* ptr to beginning of IP header */ 270 271 if (mp->b_cont != NULL) { 272 mblk_t *mp1; 273 274 mp1 = msgpullup(mp, offset + end); 275 if (mp1 == NULL) 276 continue; 277 mp1->b_next = mp->b_next; 278 mp->b_next = NULL; 279 freemsg(mp); 280 if (prev != NULL) 281 prev->b_next = mp1; 282 else 283 new_chain = mp1; 284 mp = mp1; 285 } 286 287 ipp = mp->b_rptr + offset; 288 /* LINTED: cast may result in improper alignment */ 289 up = (uint16_t *)((uchar_t *)ipp + stuff); 290 partial = *up; 291 *up = 0; 292 293 cksum = IP_BCSUM_PARTIAL(mp->b_rptr + offset + start, 294 end - start, partial); 295 cksum = ~cksum; 296 *up = cksum ? cksum : ~cksum; 297 298 /* 299 * Since we already computed the whole checksum, 300 * indicate to the stack that it has already 301 * been verified by the hardware. 302 */ 303 flags &= ~HCK_PARTIALCKSUM; 304 flags |= HCK_FULLCKSUM_OK; 305 value = 0; 306 } 307 308 (void) hcksum_assoc(mp, NULL, NULL, start, stuff, end, 309 value, flags, KM_NOSLEEP); 310 } 311 312 return (new_chain); 313 } 314 315 /* 316 * Add VLAN tag to the specified mblk. 317 */ 318 mblk_t * 319 mac_add_vlan_tag(mblk_t *mp, uint_t pri, uint16_t vid) 320 { 321 mblk_t *hmp; 322 struct ether_vlan_header *evhp; 323 struct ether_header *ehp; 324 uint32_t start, stuff, end, value, flags; 325 326 ASSERT(pri != 0 || vid != 0); 327 328 /* 329 * Allocate an mblk for the new tagged ethernet header, 330 * and copy the MAC addresses and ethertype from the 331 * original header. 332 */ 333 334 hmp = allocb(sizeof (struct ether_vlan_header), BPRI_MED); 335 if (hmp == NULL) { 336 freemsg(mp); 337 return (NULL); 338 } 339 340 evhp = (struct ether_vlan_header *)hmp->b_rptr; 341 ehp = (struct ether_header *)mp->b_rptr; 342 343 bcopy(ehp, evhp, (ETHERADDRL * 2)); 344 evhp->ether_type = ehp->ether_type; 345 evhp->ether_tpid = htons(ETHERTYPE_VLAN); 346 347 hmp->b_wptr += sizeof (struct ether_vlan_header); 348 mp->b_rptr += sizeof (struct ether_header); 349 350 /* 351 * Free the original message if it's now empty. Link the 352 * rest of messages to the header message. 353 */ 354 hcksum_retrieve(mp, NULL, NULL, &start, &stuff, &end, &value, &flags); 355 (void) hcksum_assoc(hmp, NULL, NULL, start, stuff, end, value, flags, 356 KM_NOSLEEP); 357 if (MBLKL(mp) == 0) { 358 hmp->b_cont = mp->b_cont; 359 freeb(mp); 360 } else { 361 hmp->b_cont = mp; 362 } 363 ASSERT(MBLKL(hmp) >= sizeof (struct ether_vlan_header)); 364 365 /* 366 * Initialize the new TCI (Tag Control Information). 367 */ 368 evhp->ether_tci = htons(VLAN_TCI(pri, 0, vid)); 369 370 return (hmp); 371 } 372 373 /* 374 * Adds a VLAN tag with the specified VID and priority to each mblk of 375 * the specified chain. 376 */ 377 mblk_t * 378 mac_add_vlan_tag_chain(mblk_t *mp_chain, uint_t pri, uint16_t vid) 379 { 380 mblk_t *next_mp, **prev, *mp; 381 382 mp = mp_chain; 383 prev = &mp_chain; 384 385 while (mp != NULL) { 386 next_mp = mp->b_next; 387 mp->b_next = NULL; 388 if ((mp = mac_add_vlan_tag(mp, pri, vid)) == NULL) { 389 freemsgchain(next_mp); 390 break; 391 } 392 *prev = mp; 393 prev = &mp->b_next; 394 mp = mp->b_next = next_mp; 395 } 396 397 return (mp_chain); 398 } 399 400 /* 401 * Strip VLAN tag 402 */ 403 mblk_t * 404 mac_strip_vlan_tag(mblk_t *mp) 405 { 406 mblk_t *newmp; 407 struct ether_vlan_header *evhp; 408 409 evhp = (struct ether_vlan_header *)mp->b_rptr; 410 if (ntohs(evhp->ether_tpid) == ETHERTYPE_VLAN) { 411 ASSERT(MBLKL(mp) >= sizeof (struct ether_vlan_header)); 412 413 if (DB_REF(mp) > 1) { 414 newmp = copymsg(mp); 415 if (newmp == NULL) 416 return (NULL); 417 freemsg(mp); 418 mp = newmp; 419 } 420 421 evhp = (struct ether_vlan_header *)mp->b_rptr; 422 423 ovbcopy(mp->b_rptr, mp->b_rptr + VLAN_TAGSZ, 2 * ETHERADDRL); 424 mp->b_rptr += VLAN_TAGSZ; 425 } 426 return (mp); 427 } 428 429 /* 430 * Strip VLAN tag from each mblk of the chain. 431 */ 432 mblk_t * 433 mac_strip_vlan_tag_chain(mblk_t *mp_chain) 434 { 435 mblk_t *mp, *next_mp, **prev; 436 437 mp = mp_chain; 438 prev = &mp_chain; 439 440 while (mp != NULL) { 441 next_mp = mp->b_next; 442 mp->b_next = NULL; 443 if ((mp = mac_strip_vlan_tag(mp)) == NULL) { 444 freemsgchain(next_mp); 445 break; 446 } 447 *prev = mp; 448 prev = &mp->b_next; 449 mp = mp->b_next = next_mp; 450 } 451 452 return (mp_chain); 453 } 454 455 /* 456 * Default callback function. Used when the datapath is not yet initialized. 457 */ 458 /* ARGSUSED */ 459 void 460 mac_pkt_drop(void *arg, mac_resource_handle_t resource, mblk_t *mp, 461 boolean_t loopback) 462 { 463 mblk_t *mp1 = mp; 464 465 while (mp1 != NULL) { 466 mp1->b_prev = NULL; 467 mp1->b_queue = NULL; 468 mp1 = mp1->b_next; 469 } 470 freemsgchain(mp); 471 } 472 473 /* 474 * Determines the IPv6 header length accounting for all the optional IPv6 475 * headers (hop-by-hop, destination, routing and fragment). The header length 476 * and next header value (a transport header) is captured. 477 * 478 * Returns B_FALSE if all the IP headers are not in the same mblk otherwise 479 * returns B_TRUE. 480 */ 481 boolean_t 482 mac_ip_hdr_length_v6(ip6_t *ip6h, uint8_t *endptr, uint16_t *hdr_length, 483 uint8_t *next_hdr, ip6_frag_t **fragp) 484 { 485 uint16_t length; 486 uint_t ehdrlen; 487 uint8_t *whereptr; 488 uint8_t *nexthdrp; 489 ip6_dest_t *desthdr; 490 ip6_rthdr_t *rthdr; 491 ip6_frag_t *fraghdr; 492 493 if (((uchar_t *)ip6h + IPV6_HDR_LEN) > endptr) 494 return (B_FALSE); 495 ASSERT(IPH_HDR_VERSION(ip6h) == IPV6_VERSION); 496 length = IPV6_HDR_LEN; 497 whereptr = ((uint8_t *)&ip6h[1]); /* point to next hdr */ 498 499 if (fragp != NULL) 500 *fragp = NULL; 501 502 nexthdrp = &ip6h->ip6_nxt; 503 while (whereptr < endptr) { 504 /* Is there enough left for len + nexthdr? */ 505 if (whereptr + MIN_EHDR_LEN > endptr) 506 break; 507 508 switch (*nexthdrp) { 509 case IPPROTO_HOPOPTS: 510 case IPPROTO_DSTOPTS: 511 /* Assumes the headers are identical for hbh and dst */ 512 desthdr = (ip6_dest_t *)whereptr; 513 ehdrlen = 8 * (desthdr->ip6d_len + 1); 514 if ((uchar_t *)desthdr + ehdrlen > endptr) 515 return (B_FALSE); 516 nexthdrp = &desthdr->ip6d_nxt; 517 break; 518 case IPPROTO_ROUTING: 519 rthdr = (ip6_rthdr_t *)whereptr; 520 ehdrlen = 8 * (rthdr->ip6r_len + 1); 521 if ((uchar_t *)rthdr + ehdrlen > endptr) 522 return (B_FALSE); 523 nexthdrp = &rthdr->ip6r_nxt; 524 break; 525 case IPPROTO_FRAGMENT: 526 fraghdr = (ip6_frag_t *)whereptr; 527 ehdrlen = sizeof (ip6_frag_t); 528 if ((uchar_t *)&fraghdr[1] > endptr) 529 return (B_FALSE); 530 nexthdrp = &fraghdr->ip6f_nxt; 531 if (fragp != NULL) 532 *fragp = fraghdr; 533 break; 534 case IPPROTO_NONE: 535 /* No next header means we're finished */ 536 default: 537 *hdr_length = length; 538 *next_hdr = *nexthdrp; 539 return (B_TRUE); 540 } 541 length += ehdrlen; 542 whereptr += ehdrlen; 543 *hdr_length = length; 544 *next_hdr = *nexthdrp; 545 } 546 switch (*nexthdrp) { 547 case IPPROTO_HOPOPTS: 548 case IPPROTO_DSTOPTS: 549 case IPPROTO_ROUTING: 550 case IPPROTO_FRAGMENT: 551 /* 552 * If any know extension headers are still to be processed, 553 * the packet's malformed (or at least all the IP header(s) are 554 * not in the same mblk - and that should never happen. 555 */ 556 return (B_FALSE); 557 558 default: 559 /* 560 * If we get here, we know that all of the IP headers were in 561 * the same mblk, even if the ULP header is in the next mblk. 562 */ 563 *hdr_length = length; 564 *next_hdr = *nexthdrp; 565 return (B_TRUE); 566 } 567 } 568 569 /* 570 * The following set of routines are there to take care of interrupt 571 * re-targeting for legacy (fixed) interrupts. Some older versions 572 * of the popular NICs like e1000g do not support MSI-X interrupts 573 * and they reserve fixed interrupts for RX/TX rings. To re-target 574 * these interrupts, PCITOOL ioctls need to be used. 575 */ 576 typedef struct mac_dladm_intr { 577 int ino; 578 int cpu_id; 579 char driver_path[MAXPATHLEN]; 580 char nexus_path[MAXPATHLEN]; 581 } mac_dladm_intr_t; 582 583 /* Bind the interrupt to cpu_num */ 584 static int 585 mac_set_intr(ldi_handle_t lh, processorid_t cpu_num, int ino) 586 { 587 pcitool_intr_set_t iset; 588 int err; 589 590 iset.ino = ino; 591 iset.cpu_id = cpu_num; 592 iset.user_version = PCITOOL_VERSION; 593 err = ldi_ioctl(lh, PCITOOL_DEVICE_SET_INTR, (intptr_t)&iset, FKIOCTL, 594 kcred, NULL); 595 596 return (err); 597 } 598 599 /* 600 * Search interrupt information. iget is filled in with the info to search 601 */ 602 static boolean_t 603 mac_search_intrinfo(pcitool_intr_get_t *iget_p, mac_dladm_intr_t *dln) 604 { 605 int i; 606 char driver_path[2 * MAXPATHLEN]; 607 608 for (i = 0; i < iget_p->num_devs; i++) { 609 (void) strlcpy(driver_path, iget_p->dev[i].path, MAXPATHLEN); 610 (void) snprintf(&driver_path[strlen(driver_path)], MAXPATHLEN, 611 ":%s%d", iget_p->dev[i].driver_name, 612 iget_p->dev[i].dev_inst); 613 /* Match the device path for the device path */ 614 if (strcmp(driver_path, dln->driver_path) == 0) { 615 dln->ino = iget_p->ino; 616 dln->cpu_id = iget_p->cpu_id; 617 return (B_TRUE); 618 } 619 } 620 return (B_FALSE); 621 } 622 623 /* 624 * Get information about ino, i.e. if this is the interrupt for our 625 * device and where it is bound etc. 626 */ 627 static boolean_t 628 mac_get_single_intr(ldi_handle_t lh, int ino, mac_dladm_intr_t *dln) 629 { 630 pcitool_intr_get_t *iget_p; 631 int ipsz; 632 int nipsz; 633 int err; 634 uint8_t inum; 635 636 /* 637 * Check if SLEEP is OK, i.e if could come here in response to 638 * changing the fanout due to some callback from the driver, say 639 * link speed changes. 640 */ 641 ipsz = PCITOOL_IGET_SIZE(0); 642 iget_p = kmem_zalloc(ipsz, KM_SLEEP); 643 644 iget_p->num_devs_ret = 0; 645 iget_p->user_version = PCITOOL_VERSION; 646 iget_p->ino = ino; 647 648 err = ldi_ioctl(lh, PCITOOL_DEVICE_GET_INTR, (intptr_t)iget_p, 649 FKIOCTL, kcred, NULL); 650 if (err != 0) { 651 kmem_free(iget_p, ipsz); 652 return (B_FALSE); 653 } 654 if (iget_p->num_devs == 0) { 655 kmem_free(iget_p, ipsz); 656 return (B_FALSE); 657 } 658 inum = iget_p->num_devs; 659 if (iget_p->num_devs_ret < iget_p->num_devs) { 660 /* Reallocate */ 661 nipsz = PCITOOL_IGET_SIZE(iget_p->num_devs); 662 663 kmem_free(iget_p, ipsz); 664 ipsz = nipsz; 665 iget_p = kmem_zalloc(ipsz, KM_SLEEP); 666 667 iget_p->num_devs_ret = inum; 668 iget_p->ino = ino; 669 iget_p->user_version = PCITOOL_VERSION; 670 err = ldi_ioctl(lh, PCITOOL_DEVICE_GET_INTR, (intptr_t)iget_p, 671 FKIOCTL, kcred, NULL); 672 if (err != 0) { 673 kmem_free(iget_p, ipsz); 674 return (B_FALSE); 675 } 676 /* defensive */ 677 if (iget_p->num_devs != iget_p->num_devs_ret) { 678 kmem_free(iget_p, ipsz); 679 return (B_FALSE); 680 } 681 } 682 683 if (mac_search_intrinfo(iget_p, dln)) { 684 kmem_free(iget_p, ipsz); 685 return (B_TRUE); 686 } 687 kmem_free(iget_p, ipsz); 688 return (B_FALSE); 689 } 690 691 /* 692 * Get the interrupts and check each one to see if it is for our device. 693 */ 694 static int 695 mac_validate_intr(ldi_handle_t lh, mac_dladm_intr_t *dln, processorid_t cpuid) 696 { 697 pcitool_intr_info_t intr_info; 698 int err; 699 int ino; 700 701 err = ldi_ioctl(lh, PCITOOL_SYSTEM_INTR_INFO, (intptr_t)&intr_info, 702 FKIOCTL, kcred, NULL); 703 if (err != 0) 704 return (-1); 705 706 for (ino = 0; ino < intr_info.num_intr; ino++) { 707 if (mac_get_single_intr(lh, ino, dln)) { 708 if (dln->cpu_id == cpuid) 709 return (0); 710 return (1); 711 } 712 } 713 return (-1); 714 } 715 716 /* 717 * Obtain the nexus parent node info. for mdip. 718 */ 719 static dev_info_t * 720 mac_get_nexus_node(dev_info_t *mdip, mac_dladm_intr_t *dln) 721 { 722 struct dev_info *tdip = (struct dev_info *)mdip; 723 struct ddi_minor_data *minordata; 724 int circ; 725 dev_info_t *pdip; 726 char pathname[MAXPATHLEN]; 727 728 while (tdip != NULL) { 729 /* 730 * The netboot code could call this function while walking the 731 * device tree so we need to use ndi_devi_tryenter() here to 732 * avoid deadlock. 733 */ 734 if (ndi_devi_tryenter((dev_info_t *)tdip, &circ) == 0) 735 break; 736 737 for (minordata = tdip->devi_minor; minordata != NULL; 738 minordata = minordata->next) { 739 if (strncmp(minordata->ddm_node_type, DDI_NT_INTRCTL, 740 strlen(DDI_NT_INTRCTL)) == 0) { 741 pdip = minordata->dip; 742 (void) ddi_pathname(pdip, pathname); 743 (void) snprintf(dln->nexus_path, MAXPATHLEN, 744 "/devices%s:intr", pathname); 745 (void) ddi_pathname_minor(minordata, pathname); 746 ndi_devi_exit((dev_info_t *)tdip, circ); 747 return (pdip); 748 } 749 } 750 ndi_devi_exit((dev_info_t *)tdip, circ); 751 tdip = tdip->devi_parent; 752 } 753 return (NULL); 754 } 755 756 /* 757 * For a primary MAC client, if the user has set a list or CPUs or 758 * we have obtained it implicitly, we try to retarget the interrupt 759 * for that device on one of the CPUs in the list. 760 * We assign the interrupt to the same CPU as the poll thread. 761 */ 762 static boolean_t 763 mac_check_interrupt_binding(dev_info_t *mdip, int32_t cpuid) 764 { 765 ldi_handle_t lh = NULL; 766 ldi_ident_t li = NULL; 767 int err; 768 int ret; 769 mac_dladm_intr_t dln; 770 dev_info_t *dip; 771 struct ddi_minor_data *minordata; 772 773 dln.nexus_path[0] = '\0'; 774 dln.driver_path[0] = '\0'; 775 776 minordata = ((struct dev_info *)mdip)->devi_minor; 777 while (minordata != NULL) { 778 if (minordata->type == DDM_MINOR) 779 break; 780 minordata = minordata->next; 781 } 782 if (minordata == NULL) 783 return (B_FALSE); 784 785 (void) ddi_pathname_minor(minordata, dln.driver_path); 786 787 dip = mac_get_nexus_node(mdip, &dln); 788 /* defensive */ 789 if (dip == NULL) 790 return (B_FALSE); 791 792 err = ldi_ident_from_major(ddi_driver_major(dip), &li); 793 if (err != 0) 794 return (B_FALSE); 795 796 err = ldi_open_by_name(dln.nexus_path, FREAD|FWRITE, kcred, &lh, li); 797 if (err != 0) 798 return (B_FALSE); 799 800 ret = mac_validate_intr(lh, &dln, cpuid); 801 if (ret < 0) { 802 (void) ldi_close(lh, FREAD|FWRITE, kcred); 803 return (B_FALSE); 804 } 805 /* cmn_note? */ 806 if (ret != 0) 807 if ((err = (mac_set_intr(lh, cpuid, dln.ino))) != 0) { 808 (void) ldi_close(lh, FREAD|FWRITE, kcred); 809 return (B_FALSE); 810 } 811 (void) ldi_close(lh, FREAD|FWRITE, kcred); 812 return (B_TRUE); 813 } 814 815 void 816 mac_client_set_intr_cpu(void *arg, mac_client_handle_t mch, int32_t cpuid) 817 { 818 dev_info_t *mdip = (dev_info_t *)arg; 819 mac_client_impl_t *mcip = (mac_client_impl_t *)mch; 820 mac_resource_props_t *mrp; 821 mac_perim_handle_t mph; 822 flow_entry_t *flent = mcip->mci_flent; 823 mac_soft_ring_set_t *rx_srs; 824 mac_cpus_t *srs_cpu; 825 826 if (!mac_check_interrupt_binding(mdip, cpuid)) 827 cpuid = -1; 828 mac_perim_enter_by_mh((mac_handle_t)mcip->mci_mip, &mph); 829 mrp = MCIP_RESOURCE_PROPS(mcip); 830 mrp->mrp_rx_intr_cpu = cpuid; 831 if (flent != NULL && flent->fe_rx_srs_cnt == 2) { 832 rx_srs = flent->fe_rx_srs[1]; 833 srs_cpu = &rx_srs->srs_cpu; 834 srs_cpu->mc_rx_intr_cpu = cpuid; 835 } 836 mac_perim_exit(mph); 837 } 838 839 int32_t 840 mac_client_intr_cpu(mac_client_handle_t mch) 841 { 842 mac_client_impl_t *mcip = (mac_client_impl_t *)mch; 843 mac_cpus_t *srs_cpu; 844 mac_soft_ring_set_t *rx_srs; 845 flow_entry_t *flent = mcip->mci_flent; 846 mac_resource_props_t *mrp = MCIP_RESOURCE_PROPS(mcip); 847 mac_ring_t *ring; 848 mac_intr_t *mintr; 849 850 /* 851 * Check if we need to retarget the interrupt. We do this only 852 * for the primary MAC client. We do this if we have the only 853 * exclusive ring in the group. 854 */ 855 if (mac_is_primary_client(mcip) && flent->fe_rx_srs_cnt == 2) { 856 rx_srs = flent->fe_rx_srs[1]; 857 srs_cpu = &rx_srs->srs_cpu; 858 ring = rx_srs->srs_ring; 859 mintr = &ring->mr_info.mri_intr; 860 /* 861 * If ddi_handle is present or the poll CPU is 862 * already bound to the interrupt CPU, return -1. 863 */ 864 if (mintr->mi_ddi_handle != NULL || 865 ((mrp->mrp_ncpus != 0) && 866 (mrp->mrp_rx_intr_cpu == srs_cpu->mc_rx_pollid))) { 867 return (-1); 868 } 869 return (srs_cpu->mc_rx_pollid); 870 } 871 return (-1); 872 } 873 874 void * 875 mac_get_devinfo(mac_handle_t mh) 876 { 877 mac_impl_t *mip = (mac_impl_t *)mh; 878 879 return ((void *)mip->mi_dip); 880 } 881 882 #define PKT_HASH_2BYTES(x) ((x)[0] ^ (x)[1]) 883 #define PKT_HASH_4BYTES(x) ((x)[0] ^ (x)[1] ^ (x)[2] ^ (x)[3]) 884 #define PKT_HASH_MAC(x) ((x)[0] ^ (x)[1] ^ (x)[2] ^ (x)[3] ^ (x)[4] ^ (x)[5]) 885 886 uint64_t 887 mac_pkt_hash(uint_t media, mblk_t *mp, uint8_t policy, boolean_t is_outbound) 888 { 889 struct ether_header *ehp; 890 uint64_t hash = 0; 891 uint16_t sap; 892 uint_t skip_len; 893 uint8_t proto; 894 boolean_t ip_fragmented; 895 896 /* 897 * We may want to have one of these per MAC type plugin in the 898 * future. For now supports only ethernet. 899 */ 900 if (media != DL_ETHER) 901 return (0L); 902 903 /* for now we support only outbound packets */ 904 ASSERT(is_outbound); 905 ASSERT(IS_P2ALIGNED(mp->b_rptr, sizeof (uint16_t))); 906 ASSERT(MBLKL(mp) >= sizeof (struct ether_header)); 907 908 /* compute L2 hash */ 909 910 ehp = (struct ether_header *)mp->b_rptr; 911 912 if ((policy & MAC_PKT_HASH_L2) != 0) { 913 uchar_t *mac_src = ehp->ether_shost.ether_addr_octet; 914 uchar_t *mac_dst = ehp->ether_dhost.ether_addr_octet; 915 hash = PKT_HASH_MAC(mac_src) ^ PKT_HASH_MAC(mac_dst); 916 policy &= ~MAC_PKT_HASH_L2; 917 } 918 919 if (policy == 0) 920 goto done; 921 922 /* skip ethernet header */ 923 924 sap = ntohs(ehp->ether_type); 925 if (sap == ETHERTYPE_VLAN) { 926 struct ether_vlan_header *evhp; 927 mblk_t *newmp = NULL; 928 929 skip_len = sizeof (struct ether_vlan_header); 930 if (MBLKL(mp) < skip_len) { 931 /* the vlan tag is the payload, pull up first */ 932 newmp = msgpullup(mp, -1); 933 if ((newmp == NULL) || (MBLKL(newmp) < skip_len)) { 934 goto done; 935 } 936 evhp = (struct ether_vlan_header *)newmp->b_rptr; 937 } else { 938 evhp = (struct ether_vlan_header *)mp->b_rptr; 939 } 940 941 sap = ntohs(evhp->ether_type); 942 freemsg(newmp); 943 } else { 944 skip_len = sizeof (struct ether_header); 945 } 946 947 /* if ethernet header is in its own mblk, skip it */ 948 if (MBLKL(mp) <= skip_len) { 949 skip_len -= MBLKL(mp); 950 mp = mp->b_cont; 951 if (mp == NULL) 952 goto done; 953 } 954 955 sap = (sap < ETHERTYPE_802_MIN) ? 0 : sap; 956 957 /* compute IP src/dst addresses hash and skip IPv{4,6} header */ 958 959 switch (sap) { 960 case ETHERTYPE_IP: { 961 ipha_t *iphp; 962 963 /* 964 * If the header is not aligned or the header doesn't fit 965 * in the mblk, bail now. Note that this may cause packets 966 * reordering. 967 */ 968 iphp = (ipha_t *)(mp->b_rptr + skip_len); 969 if (((unsigned char *)iphp + sizeof (ipha_t) > mp->b_wptr) || 970 !OK_32PTR((char *)iphp)) 971 goto done; 972 973 proto = iphp->ipha_protocol; 974 skip_len += IPH_HDR_LENGTH(iphp); 975 976 /* Check if the packet is fragmented. */ 977 ip_fragmented = ntohs(iphp->ipha_fragment_offset_and_flags) & 978 IPH_OFFSET; 979 980 /* 981 * For fragmented packets, use addresses in addition to 982 * the frag_id to generate the hash inorder to get 983 * better distribution. 984 */ 985 if (ip_fragmented || (policy & MAC_PKT_HASH_L3) != 0) { 986 uint8_t *ip_src = (uint8_t *)&(iphp->ipha_src); 987 uint8_t *ip_dst = (uint8_t *)&(iphp->ipha_dst); 988 989 hash ^= (PKT_HASH_4BYTES(ip_src) ^ 990 PKT_HASH_4BYTES(ip_dst)); 991 policy &= ~MAC_PKT_HASH_L3; 992 } 993 994 if (ip_fragmented) { 995 uint8_t *identp = (uint8_t *)&iphp->ipha_ident; 996 hash ^= PKT_HASH_2BYTES(identp); 997 goto done; 998 } 999 break; 1000 } 1001 case ETHERTYPE_IPV6: { 1002 ip6_t *ip6hp; 1003 ip6_frag_t *frag = NULL; 1004 uint16_t hdr_length; 1005 1006 /* 1007 * If the header is not aligned or the header doesn't fit 1008 * in the mblk, bail now. Note that this may cause packets 1009 * reordering. 1010 */ 1011 1012 ip6hp = (ip6_t *)(mp->b_rptr + skip_len); 1013 if (((unsigned char *)ip6hp + IPV6_HDR_LEN > mp->b_wptr) || 1014 !OK_32PTR((char *)ip6hp)) 1015 goto done; 1016 1017 if (!mac_ip_hdr_length_v6(ip6hp, mp->b_wptr, &hdr_length, 1018 &proto, &frag)) 1019 goto done; 1020 skip_len += hdr_length; 1021 1022 /* 1023 * For fragmented packets, use addresses in addition to 1024 * the frag_id to generate the hash inorder to get 1025 * better distribution. 1026 */ 1027 if (frag != NULL || (policy & MAC_PKT_HASH_L3) != 0) { 1028 uint8_t *ip_src = &(ip6hp->ip6_src.s6_addr8[12]); 1029 uint8_t *ip_dst = &(ip6hp->ip6_dst.s6_addr8[12]); 1030 1031 hash ^= (PKT_HASH_4BYTES(ip_src) ^ 1032 PKT_HASH_4BYTES(ip_dst)); 1033 policy &= ~MAC_PKT_HASH_L3; 1034 } 1035 1036 if (frag != NULL) { 1037 uint8_t *identp = (uint8_t *)&frag->ip6f_ident; 1038 hash ^= PKT_HASH_4BYTES(identp); 1039 goto done; 1040 } 1041 break; 1042 } 1043 default: 1044 goto done; 1045 } 1046 1047 if (policy == 0) 1048 goto done; 1049 1050 /* if ip header is in its own mblk, skip it */ 1051 if (MBLKL(mp) <= skip_len) { 1052 skip_len -= MBLKL(mp); 1053 mp = mp->b_cont; 1054 if (mp == NULL) 1055 goto done; 1056 } 1057 1058 /* parse ULP header */ 1059 again: 1060 switch (proto) { 1061 case IPPROTO_TCP: 1062 case IPPROTO_UDP: 1063 case IPPROTO_ESP: 1064 case IPPROTO_SCTP: 1065 /* 1066 * These Internet Protocols are intentionally designed 1067 * for hashing from the git-go. Port numbers are in the first 1068 * word for transports, SPI is first for ESP. 1069 */ 1070 if (mp->b_rptr + skip_len + 4 > mp->b_wptr) 1071 goto done; 1072 hash ^= PKT_HASH_4BYTES((mp->b_rptr + skip_len)); 1073 break; 1074 1075 case IPPROTO_AH: { 1076 ah_t *ah = (ah_t *)(mp->b_rptr + skip_len); 1077 uint_t ah_length = AH_TOTAL_LEN(ah); 1078 1079 if ((unsigned char *)ah + sizeof (ah_t) > mp->b_wptr) 1080 goto done; 1081 1082 proto = ah->ah_nexthdr; 1083 skip_len += ah_length; 1084 1085 /* if AH header is in its own mblk, skip it */ 1086 if (MBLKL(mp) <= skip_len) { 1087 skip_len -= MBLKL(mp); 1088 mp = mp->b_cont; 1089 if (mp == NULL) 1090 goto done; 1091 } 1092 1093 goto again; 1094 } 1095 } 1096 1097 done: 1098 return (hash); 1099 } 1100