1 /* 2 * This file is provided under a CDDLv1 license. When using or 3 * redistributing this file, you may do so under this license. 4 * In redistributing this file this license must be included 5 * and no other modification of this header file is permitted. 6 * 7 * CDDL LICENSE SUMMARY 8 * 9 * Copyright(c) 1999 - 2009 Intel Corporation. All rights reserved. 10 * 11 * The contents of this file are subject to the terms of Version 12 * 1.0 of the Common Development and Distribution License (the "License"). 13 * 14 * You should have received a copy of the License with this software. 15 * You can obtain a copy of the License at 16 * http://www.opensolaris.org/os/licensing. 17 * See the License for the specific language governing permissions 18 * and limitations under the License. 19 */ 20 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 /* 27 * ********************************************************************** 28 * * 29 * Module Name: * 30 * e1000g_rx.c * 31 * * 32 * Abstract: * 33 * This file contains some routines that take care of Receive * 34 * interrupt and also for the received packets it sends up to * 35 * upper layer. * 36 * It tries to do a zero copy if free buffers are available in * 37 * the pool. * 38 * * 39 * ********************************************************************** 40 */ 41 42 #include "e1000g_sw.h" 43 #include "e1000g_debug.h" 44 45 static p_rx_sw_packet_t e1000g_get_buf(e1000g_rx_data_t *rx_data); 46 #pragma inline(e1000g_get_buf) 47 48 /* 49 * e1000g_rxfree_func - the call-back function to reclaim rx buffer 50 * 51 * This function is called when an mp is freed by the user thru 52 * freeb call (Only for mp constructed through desballoc call) 53 * It returns back the freed buffer to the freelist 54 */ 55 void 56 e1000g_rxfree_func(p_rx_sw_packet_t packet) 57 { 58 e1000g_rx_data_t *rx_data; 59 private_devi_list_t *devi_node; 60 struct e1000g *Adapter; 61 uint32_t ring_cnt; 62 uint32_t ref_cnt; 63 unsigned char *address; 64 65 if (packet->ref_cnt == 0) { 66 /* 67 * This case only happens when rx buffers are being freed 68 * in e1000g_stop() and freemsg() is called. 69 */ 70 return; 71 } 72 73 rx_data = (e1000g_rx_data_t *)(uintptr_t)packet->rx_data; 74 75 if (packet->mp == NULL) { 76 /* 77 * Allocate a mblk that binds to the data buffer 78 */ 79 address = (unsigned char *)packet->rx_buf->address; 80 if (address != NULL) { 81 packet->mp = desballoc((unsigned char *) 82 address, packet->rx_buf->size, 83 BPRI_MED, &packet->free_rtn); 84 } 85 } 86 87 /* 88 * Enqueue the recycled packets in a recycle queue. When freelist 89 * dries up, move the entire chain of packets from recycle queue 90 * to freelist. This helps in avoiding per packet mutex contention 91 * around freelist. 92 */ 93 mutex_enter(&rx_data->recycle_lock); 94 QUEUE_PUSH_TAIL(&rx_data->recycle_list, &packet->Link); 95 rx_data->recycle_freepkt++; 96 mutex_exit(&rx_data->recycle_lock); 97 98 ref_cnt = atomic_dec_32_nv(&packet->ref_cnt); 99 if (ref_cnt == 0) { 100 mutex_enter(&e1000g_rx_detach_lock); 101 e1000g_free_rx_sw_packet(packet, B_FALSE); 102 103 atomic_dec_32(&rx_data->pending_count); 104 atomic_dec_32(&e1000g_mblks_pending); 105 106 if ((rx_data->pending_count == 0) && 107 (rx_data->flag & E1000G_RX_STOPPED)) { 108 devi_node = rx_data->priv_devi_node; 109 110 if (devi_node != NULL) { 111 ring_cnt = atomic_dec_32_nv( 112 &devi_node->pending_rx_count); 113 if ((ring_cnt == 0) && 114 (devi_node->flag & 115 E1000G_PRIV_DEVI_DETACH)) { 116 e1000g_free_priv_devi_node( 117 devi_node); 118 } 119 } else { 120 Adapter = rx_data->rx_ring->adapter; 121 atomic_dec_32( 122 &Adapter->pending_rx_count); 123 } 124 125 e1000g_free_rx_pending_buffers(rx_data); 126 e1000g_free_rx_data(rx_data); 127 } 128 mutex_exit(&e1000g_rx_detach_lock); 129 } 130 } 131 132 /* 133 * e1000g_rx_setup - setup rx data structures 134 * 135 * This routine initializes all of the receive related 136 * structures. This includes the receive descriptors, the 137 * actual receive buffers, and the rx_sw_packet software 138 * structures. 139 */ 140 void 141 e1000g_rx_setup(struct e1000g *Adapter) 142 { 143 struct e1000_hw *hw; 144 p_rx_sw_packet_t packet; 145 struct e1000_rx_desc *descriptor; 146 uint32_t buf_low; 147 uint32_t buf_high; 148 uint32_t reg_val; 149 uint32_t rctl; 150 uint32_t rxdctl; 151 uint32_t ert; 152 int i; 153 int size; 154 e1000g_rx_data_t *rx_data; 155 156 hw = &Adapter->shared; 157 rx_data = Adapter->rx_ring->rx_data; 158 159 /* 160 * zero out all of the receive buffer descriptor memory 161 * assures any previous data or status is erased 162 */ 163 bzero(rx_data->rbd_area, 164 sizeof (struct e1000_rx_desc) * Adapter->rx_desc_num); 165 166 if (!Adapter->rx_buffer_setup) { 167 /* Init the list of "Receive Buffer" */ 168 QUEUE_INIT_LIST(&rx_data->recv_list); 169 170 /* Init the list of "Free Receive Buffer" */ 171 QUEUE_INIT_LIST(&rx_data->free_list); 172 173 /* Init the list of "Free Receive Buffer" */ 174 QUEUE_INIT_LIST(&rx_data->recycle_list); 175 /* 176 * Setup Receive list and the Free list. Note that 177 * the both were allocated in one packet area. 178 */ 179 packet = rx_data->packet_area; 180 descriptor = rx_data->rbd_first; 181 182 for (i = 0; i < Adapter->rx_desc_num; 183 i++, packet = packet->next, descriptor++) { 184 ASSERT(packet != NULL); 185 ASSERT(descriptor != NULL); 186 descriptor->buffer_addr = 187 packet->rx_buf->dma_address; 188 189 /* Add this rx_sw_packet to the receive list */ 190 QUEUE_PUSH_TAIL(&rx_data->recv_list, 191 &packet->Link); 192 } 193 194 for (i = 0; i < Adapter->rx_freelist_num; 195 i++, packet = packet->next) { 196 ASSERT(packet != NULL); 197 /* Add this rx_sw_packet to the free list */ 198 QUEUE_PUSH_TAIL(&rx_data->free_list, 199 &packet->Link); 200 } 201 rx_data->avail_freepkt = Adapter->rx_freelist_num; 202 rx_data->recycle_freepkt = 0; 203 204 Adapter->rx_buffer_setup = B_TRUE; 205 } else { 206 /* Setup the initial pointer to the first rx descriptor */ 207 packet = (p_rx_sw_packet_t) 208 QUEUE_GET_HEAD(&rx_data->recv_list); 209 descriptor = rx_data->rbd_first; 210 211 for (i = 0; i < Adapter->rx_desc_num; i++) { 212 ASSERT(packet != NULL); 213 ASSERT(descriptor != NULL); 214 descriptor->buffer_addr = 215 packet->rx_buf->dma_address; 216 217 /* Get next rx_sw_packet */ 218 packet = (p_rx_sw_packet_t) 219 QUEUE_GET_NEXT(&rx_data->recv_list, &packet->Link); 220 descriptor++; 221 } 222 } 223 224 E1000_WRITE_REG(&Adapter->shared, E1000_RDTR, Adapter->rx_intr_delay); 225 E1000G_DEBUGLOG_1(Adapter, E1000G_INFO_LEVEL, 226 "E1000_RDTR: 0x%x\n", Adapter->rx_intr_delay); 227 if (hw->mac.type >= e1000_82540) { 228 E1000_WRITE_REG(&Adapter->shared, E1000_RADV, 229 Adapter->rx_intr_abs_delay); 230 E1000G_DEBUGLOG_1(Adapter, E1000G_INFO_LEVEL, 231 "E1000_RADV: 0x%x\n", Adapter->rx_intr_abs_delay); 232 } 233 234 /* 235 * Setup our descriptor pointers 236 */ 237 rx_data->rbd_next = rx_data->rbd_first; 238 239 size = Adapter->rx_desc_num * sizeof (struct e1000_rx_desc); 240 E1000_WRITE_REG(hw, E1000_RDLEN(0), size); 241 size = E1000_READ_REG(hw, E1000_RDLEN(0)); 242 243 /* To get lower order bits */ 244 buf_low = (uint32_t)rx_data->rbd_dma_addr; 245 /* To get the higher order bits */ 246 buf_high = (uint32_t)(rx_data->rbd_dma_addr >> 32); 247 248 E1000_WRITE_REG(hw, E1000_RDBAH(0), buf_high); 249 E1000_WRITE_REG(hw, E1000_RDBAL(0), buf_low); 250 251 /* 252 * Setup our HW Rx Head & Tail descriptor pointers 253 */ 254 E1000_WRITE_REG(hw, E1000_RDT(0), 255 (uint32_t)(rx_data->rbd_last - rx_data->rbd_first)); 256 E1000_WRITE_REG(hw, E1000_RDH(0), 0); 257 258 /* 259 * Setup the Receive Control Register (RCTL), and ENABLE the 260 * receiver. The initial configuration is to: Enable the receiver, 261 * accept broadcasts, discard bad packets (and long packets), 262 * disable VLAN filter checking, set the receive descriptor 263 * minimum threshold size to 1/2, and the receive buffer size to 264 * 2k. 265 */ 266 rctl = E1000_RCTL_EN | /* Enable Receive Unit */ 267 E1000_RCTL_BAM | /* Accept Broadcast Packets */ 268 E1000_RCTL_LPE | /* Large Packet Enable bit */ 269 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT) | 270 E1000_RCTL_RDMTS_HALF | 271 E1000_RCTL_LBM_NO; /* Loopback Mode = none */ 272 273 if (Adapter->strip_crc) 274 rctl |= E1000_RCTL_SECRC; /* Strip Ethernet CRC */ 275 276 if (Adapter->mem_workaround_82546 && 277 ((hw->mac.type == e1000_82545) || 278 (hw->mac.type == e1000_82546) || 279 (hw->mac.type == e1000_82546_rev_3))) { 280 rctl |= E1000_RCTL_SZ_2048; 281 } else { 282 if ((Adapter->max_frame_size > FRAME_SIZE_UPTO_2K) && 283 (Adapter->max_frame_size <= FRAME_SIZE_UPTO_4K)) 284 rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX; 285 else if ((Adapter->max_frame_size > FRAME_SIZE_UPTO_4K) && 286 (Adapter->max_frame_size <= FRAME_SIZE_UPTO_8K)) 287 rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX; 288 else if ((Adapter->max_frame_size > FRAME_SIZE_UPTO_8K) && 289 (Adapter->max_frame_size <= FRAME_SIZE_UPTO_16K)) 290 rctl |= E1000_RCTL_SZ_16384 | E1000_RCTL_BSEX; 291 else 292 rctl |= E1000_RCTL_SZ_2048; 293 } 294 295 if (e1000_tbi_sbp_enabled_82543(hw)) 296 rctl |= E1000_RCTL_SBP; 297 298 /* 299 * Enable Early Receive Threshold (ERT) on supported devices. 300 * Only takes effect when packet size is equal or larger than the 301 * specified value (in 8 byte units), e.g. using jumbo frames. 302 */ 303 if ((hw->mac.type == e1000_82573) || 304 (hw->mac.type == e1000_82574) || 305 (hw->mac.type == e1000_ich9lan) || 306 (hw->mac.type == e1000_ich10lan)) { 307 308 ert = E1000_ERT_2048; 309 310 /* 311 * Special modification when ERT and 312 * jumbo frames are enabled 313 */ 314 if (Adapter->default_mtu > ETHERMTU) { 315 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(0)); 316 E1000_WRITE_REG(hw, E1000_RXDCTL(0), rxdctl | 0x3); 317 ert |= (1 << 13); 318 } 319 320 E1000_WRITE_REG(hw, E1000_ERT, ert); 321 } 322 323 reg_val = 324 E1000_RXCSUM_TUOFL | /* TCP/UDP checksum offload Enable */ 325 E1000_RXCSUM_IPOFL; /* IP checksum offload Enable */ 326 327 E1000_WRITE_REG(hw, E1000_RXCSUM, reg_val); 328 329 /* 330 * Workaround: Set bit 16 (IPv6_ExDIS) to disable the 331 * processing of received IPV6 extension headers 332 */ 333 if ((hw->mac.type == e1000_82571) || (hw->mac.type == e1000_82572)) { 334 reg_val = E1000_READ_REG(hw, E1000_RFCTL); 335 reg_val |= (E1000_RFCTL_IPV6_EX_DIS | 336 E1000_RFCTL_NEW_IPV6_EXT_DIS); 337 E1000_WRITE_REG(hw, E1000_RFCTL, reg_val); 338 } 339 340 /* Write to enable the receive unit */ 341 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 342 } 343 344 /* 345 * e1000g_get_buf - get an rx sw packet from the free_list 346 */ 347 static p_rx_sw_packet_t 348 e1000g_get_buf(e1000g_rx_data_t *rx_data) 349 { 350 p_rx_sw_packet_t packet; 351 352 mutex_enter(&rx_data->freelist_lock); 353 packet = (p_rx_sw_packet_t) 354 QUEUE_POP_HEAD(&rx_data->free_list); 355 if (packet != NULL) { 356 rx_data->avail_freepkt--; 357 } else { 358 /* 359 * If the freelist has no packets, check the recycle list 360 * to see if there are any available descriptor there. 361 */ 362 mutex_enter(&rx_data->recycle_lock); 363 QUEUE_SWITCH(&rx_data->free_list, &rx_data->recycle_list); 364 rx_data->avail_freepkt = rx_data->recycle_freepkt; 365 rx_data->recycle_freepkt = 0; 366 mutex_exit(&rx_data->recycle_lock); 367 packet = (p_rx_sw_packet_t) 368 QUEUE_POP_HEAD(&rx_data->free_list); 369 if (packet != NULL) 370 rx_data->avail_freepkt--; 371 } 372 mutex_exit(&rx_data->freelist_lock); 373 374 return (packet); 375 } 376 377 /* 378 * e1000g_receive - main receive routine 379 * 380 * This routine will process packets received in an interrupt 381 */ 382 mblk_t * 383 e1000g_receive(e1000g_rx_ring_t *rx_ring, mblk_t **tail, uint_t sz) 384 { 385 struct e1000_hw *hw; 386 mblk_t *nmp; 387 mblk_t *ret_mp; 388 mblk_t *ret_nmp; 389 struct e1000_rx_desc *current_desc; 390 struct e1000_rx_desc *last_desc; 391 p_rx_sw_packet_t packet; 392 p_rx_sw_packet_t newpkt; 393 uint16_t length; 394 uint32_t pkt_count; 395 uint32_t desc_count; 396 boolean_t accept_frame; 397 boolean_t end_of_packet; 398 boolean_t need_copy; 399 struct e1000g *Adapter; 400 dma_buffer_t *rx_buf; 401 uint16_t cksumflags; 402 uint_t chain_sz = 0; 403 e1000g_rx_data_t *rx_data; 404 uint32_t max_size; 405 uint32_t min_size; 406 407 ret_mp = NULL; 408 ret_nmp = NULL; 409 pkt_count = 0; 410 desc_count = 0; 411 cksumflags = 0; 412 413 Adapter = rx_ring->adapter; 414 rx_data = rx_ring->rx_data; 415 hw = &Adapter->shared; 416 417 /* Sync the Rx descriptor DMA buffers */ 418 (void) ddi_dma_sync(rx_data->rbd_dma_handle, 419 0, 0, DDI_DMA_SYNC_FORKERNEL); 420 421 if (e1000g_check_dma_handle(rx_data->rbd_dma_handle) != DDI_FM_OK) { 422 ddi_fm_service_impact(Adapter->dip, DDI_SERVICE_DEGRADED); 423 Adapter->e1000g_state |= E1000G_ERROR; 424 } 425 426 current_desc = rx_data->rbd_next; 427 if (!(current_desc->status & E1000_RXD_STAT_DD)) { 428 /* 429 * don't send anything up. just clear the RFD 430 */ 431 E1000G_DEBUG_STAT(rx_ring->stat_none); 432 return (ret_mp); 433 } 434 435 max_size = Adapter->max_frame_size - ETHERFCSL - VLAN_TAGSZ; 436 min_size = ETHERMIN; 437 438 /* 439 * Loop through the receive descriptors starting at the last known 440 * descriptor owned by the hardware that begins a packet. 441 */ 442 while ((current_desc->status & E1000_RXD_STAT_DD) && 443 (pkt_count < Adapter->rx_limit_onintr) && 444 ((sz == E1000G_CHAIN_NO_LIMIT) || (chain_sz <= sz))) { 445 446 desc_count++; 447 /* 448 * Now this can happen in Jumbo frame situation. 449 */ 450 if (current_desc->status & E1000_RXD_STAT_EOP) { 451 /* packet has EOP set */ 452 end_of_packet = B_TRUE; 453 } else { 454 /* 455 * If this received buffer does not have the 456 * End-Of-Packet bit set, the received packet 457 * will consume multiple buffers. We won't send this 458 * packet upstack till we get all the related buffers. 459 */ 460 end_of_packet = B_FALSE; 461 } 462 463 /* 464 * Get a pointer to the actual receive buffer 465 * The mp->b_rptr is mapped to The CurrentDescriptor 466 * Buffer Address. 467 */ 468 packet = 469 (p_rx_sw_packet_t)QUEUE_GET_HEAD(&rx_data->recv_list); 470 ASSERT(packet != NULL); 471 472 rx_buf = packet->rx_buf; 473 474 length = current_desc->length; 475 476 #ifdef __sparc 477 if (packet->dma_type == USE_DVMA) 478 dvma_sync(rx_buf->dma_handle, 0, 479 DDI_DMA_SYNC_FORKERNEL); 480 else 481 (void) ddi_dma_sync(rx_buf->dma_handle, 482 E1000G_IPALIGNROOM, length, 483 DDI_DMA_SYNC_FORKERNEL); 484 #else 485 (void) ddi_dma_sync(rx_buf->dma_handle, 486 E1000G_IPALIGNROOM, length, 487 DDI_DMA_SYNC_FORKERNEL); 488 #endif 489 490 if (e1000g_check_dma_handle( 491 rx_buf->dma_handle) != DDI_FM_OK) { 492 ddi_fm_service_impact(Adapter->dip, 493 DDI_SERVICE_DEGRADED); 494 Adapter->e1000g_state |= E1000G_ERROR; 495 } 496 497 accept_frame = (current_desc->errors == 0) || 498 ((current_desc->errors & 499 (E1000_RXD_ERR_TCPE | E1000_RXD_ERR_IPE)) != 0); 500 501 if (hw->mac.type == e1000_82543) { 502 unsigned char last_byte; 503 504 last_byte = 505 *((unsigned char *)rx_buf->address + length - 1); 506 507 if (TBI_ACCEPT(hw, 508 current_desc->status, current_desc->errors, 509 current_desc->length, last_byte, 510 Adapter->min_frame_size, Adapter->max_frame_size)) { 511 512 e1000_tbi_adjust_stats(Adapter, 513 length, hw->mac.addr); 514 515 length--; 516 accept_frame = B_TRUE; 517 } else if (e1000_tbi_sbp_enabled_82543(hw) && 518 (current_desc->errors == E1000_RXD_ERR_CE)) { 519 accept_frame = B_TRUE; 520 } 521 } 522 523 /* 524 * Indicate the packet to the NOS if it was good. 525 * Normally, hardware will discard bad packets for us. 526 * Check for the packet to be a valid Ethernet packet 527 */ 528 if (!accept_frame) { 529 /* 530 * error in incoming packet, either the packet is not a 531 * ethernet size packet, or the packet has an error. In 532 * either case, the packet will simply be discarded. 533 */ 534 E1000G_DEBUGLOG_0(Adapter, E1000G_INFO_LEVEL, 535 "Process Receive Interrupts: Error in Packet\n"); 536 537 E1000G_STAT(rx_ring->stat_error); 538 /* 539 * Returning here as we are done here. There is 540 * no point in waiting for while loop to elapse 541 * and the things which were done. More efficient 542 * and less error prone... 543 */ 544 goto rx_drop; 545 } 546 547 /* 548 * If the Ethernet CRC is not stripped by the hardware, 549 * we need to strip it before sending it up to the stack. 550 */ 551 if (end_of_packet && !Adapter->strip_crc) { 552 if (length > ETHERFCSL) { 553 length -= ETHERFCSL; 554 } else { 555 /* 556 * If the fragment is smaller than the CRC, 557 * drop this fragment, do the processing of 558 * the end of the packet. 559 */ 560 ASSERT(rx_data->rx_mblk_tail != NULL); 561 rx_data->rx_mblk_tail->b_wptr -= 562 ETHERFCSL - length; 563 rx_data->rx_mblk_len -= 564 ETHERFCSL - length; 565 566 QUEUE_POP_HEAD(&rx_data->recv_list); 567 568 goto rx_end_of_packet; 569 } 570 } 571 572 need_copy = B_TRUE; 573 574 if (length <= Adapter->rx_bcopy_thresh) 575 goto rx_copy; 576 577 /* 578 * Get the pre-constructed mblk that was associated 579 * to the receive data buffer. 580 */ 581 if (packet->mp == NULL) { 582 packet->mp = desballoc((unsigned char *) 583 rx_buf->address, length, 584 BPRI_MED, &packet->free_rtn); 585 } 586 587 if (packet->mp != NULL) { 588 /* 589 * We have two sets of buffer pool. One associated with 590 * the Rxdescriptors and other a freelist buffer pool. 591 * Each time we get a good packet, Try to get a buffer 592 * from the freelist pool using e1000g_get_buf. If we 593 * get free buffer, then replace the descriptor buffer 594 * address with the free buffer we just got, and pass 595 * the pre-constructed mblk upstack. (note no copying) 596 * 597 * If we failed to get a free buffer, then try to 598 * allocate a new buffer(mp) and copy the recv buffer 599 * content to our newly allocated buffer(mp). Don't 600 * disturb the desriptor buffer address. (note copying) 601 */ 602 newpkt = e1000g_get_buf(rx_data); 603 604 if (newpkt != NULL) { 605 /* 606 * Get the mblk associated to the data, 607 * and strip it off the sw packet. 608 */ 609 nmp = packet->mp; 610 packet->mp = NULL; 611 atomic_inc_32(&packet->ref_cnt); 612 613 /* 614 * Now replace old buffer with the new 615 * one we got from free list 616 * Both the RxSwPacket as well as the 617 * Receive Buffer Descriptor will now 618 * point to this new packet. 619 */ 620 packet = newpkt; 621 622 current_desc->buffer_addr = 623 newpkt->rx_buf->dma_address; 624 625 need_copy = B_FALSE; 626 } else { 627 E1000G_DEBUG_STAT(rx_ring->stat_no_freepkt); 628 } 629 } 630 631 rx_copy: 632 if (need_copy) { 633 /* 634 * No buffers available on free list, 635 * bcopy the data from the buffer and 636 * keep the original buffer. Dont want to 637 * do this.. Yack but no other way 638 */ 639 if ((nmp = allocb(length + E1000G_IPALIGNROOM, 640 BPRI_MED)) == NULL) { 641 /* 642 * The system has no buffers available 643 * to send up the incoming packet, hence 644 * the packet will have to be processed 645 * when there're more buffers available. 646 */ 647 E1000G_STAT(rx_ring->stat_allocb_fail); 648 goto rx_drop; 649 } 650 nmp->b_rptr += E1000G_IPALIGNROOM; 651 nmp->b_wptr += E1000G_IPALIGNROOM; 652 /* 653 * The free list did not have any buffers 654 * available, so, the received packet will 655 * have to be copied into a mp and the original 656 * buffer will have to be retained for future 657 * packet reception. 658 */ 659 bcopy(rx_buf->address, nmp->b_wptr, length); 660 } 661 662 /* 663 * The rx_sw_packet MUST be popped off the 664 * RxSwPacketList before either a putnext or freemsg 665 * is done on the mp that has now been created by the 666 * desballoc. If not, it is possible that the free 667 * routine will get called from the interrupt context 668 * and try to put this packet on the free list 669 */ 670 (p_rx_sw_packet_t)QUEUE_POP_HEAD(&rx_data->recv_list); 671 672 ASSERT(nmp != NULL); 673 nmp->b_wptr += length; 674 675 if (rx_data->rx_mblk == NULL) { 676 /* 677 * TCP/UDP checksum offload and 678 * IP checksum offload 679 */ 680 if (!(current_desc->status & E1000_RXD_STAT_IXSM)) { 681 /* 682 * Check TCP/UDP checksum 683 */ 684 if ((current_desc->status & 685 E1000_RXD_STAT_TCPCS) && 686 !(current_desc->errors & 687 E1000_RXD_ERR_TCPE)) 688 cksumflags |= HCK_FULLCKSUM | 689 HCK_FULLCKSUM_OK; 690 /* 691 * Check IP Checksum 692 */ 693 if ((current_desc->status & 694 E1000_RXD_STAT_IPCS) && 695 !(current_desc->errors & 696 E1000_RXD_ERR_IPE)) 697 cksumflags |= HCK_IPV4_HDRCKSUM; 698 } 699 } 700 701 /* 702 * We need to maintain our packet chain in the global 703 * Adapter structure, for the Rx processing can end 704 * with a fragment that has no EOP set. 705 */ 706 if (rx_data->rx_mblk == NULL) { 707 /* Get the head of the message chain */ 708 rx_data->rx_mblk = nmp; 709 rx_data->rx_mblk_tail = nmp; 710 rx_data->rx_mblk_len = length; 711 } else { /* Not the first packet */ 712 /* Continue adding buffers */ 713 rx_data->rx_mblk_tail->b_cont = nmp; 714 rx_data->rx_mblk_tail = nmp; 715 rx_data->rx_mblk_len += length; 716 } 717 ASSERT(rx_data->rx_mblk != NULL); 718 ASSERT(rx_data->rx_mblk_tail != NULL); 719 ASSERT(rx_data->rx_mblk_tail->b_cont == NULL); 720 721 /* 722 * Now this MP is ready to travel upwards but some more 723 * fragments are coming. 724 * We will send packet upwards as soon as we get EOP 725 * set on the packet. 726 */ 727 if (!end_of_packet) { 728 /* 729 * continue to get the next descriptor, 730 * Tail would be advanced at the end 731 */ 732 goto rx_next_desc; 733 } 734 735 rx_end_of_packet: 736 if (E1000G_IS_VLAN_PACKET(rx_data->rx_mblk->b_rptr)) 737 max_size = Adapter->max_frame_size - ETHERFCSL; 738 739 if ((rx_data->rx_mblk_len > max_size) || 740 (rx_data->rx_mblk_len < min_size)) { 741 E1000G_STAT(rx_ring->stat_size_error); 742 goto rx_drop; 743 } 744 745 /* 746 * Found packet with EOP 747 * Process the last fragment. 748 */ 749 if (cksumflags != 0) { 750 (void) hcksum_assoc(rx_data->rx_mblk, 751 NULL, NULL, 0, 0, 0, 0, cksumflags, 0); 752 cksumflags = 0; 753 } 754 755 /* 756 * Count packets that span multi-descriptors 757 */ 758 E1000G_DEBUG_STAT_COND(rx_ring->stat_multi_desc, 759 (rx_data->rx_mblk->b_cont != NULL)); 760 761 /* 762 * Append to list to send upstream 763 */ 764 if (ret_mp == NULL) { 765 ret_mp = ret_nmp = rx_data->rx_mblk; 766 } else { 767 ret_nmp->b_next = rx_data->rx_mblk; 768 ret_nmp = rx_data->rx_mblk; 769 } 770 ret_nmp->b_next = NULL; 771 *tail = ret_nmp; 772 chain_sz += length; 773 774 rx_data->rx_mblk = NULL; 775 rx_data->rx_mblk_tail = NULL; 776 rx_data->rx_mblk_len = 0; 777 778 pkt_count++; 779 780 rx_next_desc: 781 /* 782 * Zero out the receive descriptors status 783 */ 784 current_desc->status = 0; 785 786 if (current_desc == rx_data->rbd_last) 787 rx_data->rbd_next = rx_data->rbd_first; 788 else 789 rx_data->rbd_next++; 790 791 last_desc = current_desc; 792 current_desc = rx_data->rbd_next; 793 794 /* 795 * Put the buffer that we just indicated back 796 * at the end of our list 797 */ 798 QUEUE_PUSH_TAIL(&rx_data->recv_list, 799 &packet->Link); 800 } /* while loop */ 801 802 /* Sync the Rx descriptor DMA buffers */ 803 (void) ddi_dma_sync(rx_data->rbd_dma_handle, 804 0, 0, DDI_DMA_SYNC_FORDEV); 805 806 /* 807 * Advance the E1000's Receive Queue #0 "Tail Pointer". 808 */ 809 E1000_WRITE_REG(hw, E1000_RDT(0), 810 (uint32_t)(last_desc - rx_data->rbd_first)); 811 812 if (e1000g_check_acc_handle(Adapter->osdep.reg_handle) != DDI_FM_OK) { 813 ddi_fm_service_impact(Adapter->dip, DDI_SERVICE_DEGRADED); 814 Adapter->e1000g_state |= E1000G_ERROR; 815 } 816 817 Adapter->rx_pkt_cnt = pkt_count; 818 819 return (ret_mp); 820 821 rx_drop: 822 /* 823 * Zero out the receive descriptors status 824 */ 825 current_desc->status = 0; 826 827 /* Sync the Rx descriptor DMA buffers */ 828 (void) ddi_dma_sync(rx_data->rbd_dma_handle, 829 0, 0, DDI_DMA_SYNC_FORDEV); 830 831 if (current_desc == rx_data->rbd_last) 832 rx_data->rbd_next = rx_data->rbd_first; 833 else 834 rx_data->rbd_next++; 835 836 last_desc = current_desc; 837 838 (p_rx_sw_packet_t)QUEUE_POP_HEAD(&rx_data->recv_list); 839 840 QUEUE_PUSH_TAIL(&rx_data->recv_list, &packet->Link); 841 /* 842 * Reclaim all old buffers already allocated during 843 * Jumbo receives.....for incomplete reception 844 */ 845 if (rx_data->rx_mblk != NULL) { 846 freemsg(rx_data->rx_mblk); 847 rx_data->rx_mblk = NULL; 848 rx_data->rx_mblk_tail = NULL; 849 rx_data->rx_mblk_len = 0; 850 } 851 /* 852 * Advance the E1000's Receive Queue #0 "Tail Pointer". 853 */ 854 E1000_WRITE_REG(hw, E1000_RDT(0), 855 (uint32_t)(last_desc - rx_data->rbd_first)); 856 857 if (e1000g_check_acc_handle(Adapter->osdep.reg_handle) != DDI_FM_OK) { 858 ddi_fm_service_impact(Adapter->dip, DDI_SERVICE_DEGRADED); 859 Adapter->e1000g_state |= E1000G_ERROR; 860 } 861 862 return (ret_mp); 863 } 864