1 /* 2 * CDDL HEADER START 3 * 4 * Copyright(c) 2007-2008 Intel Corporation. All rights reserved. 5 * The contents of this file are subject to the terms of the 6 * Common Development and Distribution License (the "License"). 7 * You may not use this file except in compliance with the License. 8 * 9 * You can obtain a copy of the license at: 10 * http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When using or redistributing this file, you may do so under the 15 * License only. No other modification of this header is permitted. 16 * 17 * If applicable, add the following below this CDDL HEADER, with the 18 * fields enclosed by brackets "[]" replaced with your own identifying 19 * information: Portions Copyright [yyyy] [name of copyright owner] 20 * 21 * CDDL HEADER END 22 */ 23 24 /* 25 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 26 * Use is subject to license terms of the CDDL. 27 */ 28 29 #pragma ident "%Z%%M% %I% %E% SMI" 30 31 #include "igb_sw.h" 32 33 /* function prototypes */ 34 static mblk_t *igb_rx_bind(igb_rx_ring_t *, uint32_t, uint32_t); 35 static mblk_t *igb_rx_copy(igb_rx_ring_t *, uint32_t, uint32_t); 36 static void igb_rx_assoc_hcksum(mblk_t *, uint32_t); 37 38 #ifndef IGB_DEBUG 39 #pragma inline(igb_rx_assoc_hcksum) 40 #endif 41 42 43 /* 44 * igb_rx_recycle - the call-back function to reclaim rx buffer 45 * 46 * This function is called when an mp is freed by the user thru 47 * freeb call (Only for mp constructed through desballoc call). 48 * It returns back the freed buffer to the free list. 49 */ 50 void 51 igb_rx_recycle(caddr_t arg) 52 { 53 igb_rx_ring_t *rx_ring; 54 rx_control_block_t *recycle_rcb; 55 uint32_t free_index; 56 57 recycle_rcb = (rx_control_block_t *)(uintptr_t)arg; 58 rx_ring = recycle_rcb->rx_ring; 59 60 if (recycle_rcb->state == RCB_FREE) 61 return; 62 63 recycle_rcb->state = RCB_FREE; 64 65 ASSERT(recycle_rcb->mp == NULL); 66 67 /* 68 * Using the recycled data buffer to generate a new mblk 69 */ 70 recycle_rcb->mp = desballoc((unsigned char *) 71 (recycle_rcb->rx_buf.address - IPHDR_ALIGN_ROOM), 72 (recycle_rcb->rx_buf.size + IPHDR_ALIGN_ROOM), 73 0, &recycle_rcb->free_rtn); 74 if (recycle_rcb->mp != NULL) { 75 recycle_rcb->mp->b_rptr += IPHDR_ALIGN_ROOM; 76 recycle_rcb->mp->b_wptr += IPHDR_ALIGN_ROOM; 77 } 78 79 /* 80 * Put the recycled rx control block into free list 81 */ 82 mutex_enter(&rx_ring->recycle_lock); 83 84 free_index = rx_ring->rcb_tail; 85 ASSERT(rx_ring->free_list[free_index] == NULL); 86 87 rx_ring->free_list[free_index] = recycle_rcb; 88 rx_ring->rcb_tail = NEXT_INDEX(free_index, 1, rx_ring->free_list_size); 89 90 mutex_exit(&rx_ring->recycle_lock); 91 92 /* 93 * The atomic operation on the number of the available rx control 94 * blocks in the free list is used to make the recycling mutual 95 * exclusive with the receiving. 96 */ 97 atomic_inc_32(&rx_ring->rcb_free); 98 ASSERT(rx_ring->rcb_free <= rx_ring->free_list_size); 99 } 100 101 /* 102 * igb_rx_copy - Use copy to process the received packet 103 * 104 * This function will use bcopy to process the packet 105 * and send the copied packet upstream 106 */ 107 static mblk_t * 108 igb_rx_copy(igb_rx_ring_t *rx_ring, uint32_t index, uint32_t pkt_len) 109 { 110 rx_control_block_t *current_rcb; 111 mblk_t *mp; 112 113 current_rcb = rx_ring->work_list[index]; 114 115 DMA_SYNC(¤t_rcb->rx_buf, DDI_DMA_SYNC_FORKERNEL); 116 117 /* 118 * Allocate buffer to receive this packet 119 */ 120 mp = allocb(pkt_len + IPHDR_ALIGN_ROOM, 0); 121 if (mp == NULL) { 122 igb_log(rx_ring->igb, "igb_rx_copy: allocate buffer failed"); 123 return (NULL); 124 } 125 126 /* 127 * Copy the data received into the new cluster 128 */ 129 mp->b_rptr += IPHDR_ALIGN_ROOM; 130 bcopy(current_rcb->rx_buf.address, mp->b_rptr, pkt_len); 131 mp->b_wptr = mp->b_rptr + pkt_len; 132 133 return (mp); 134 } 135 136 /* 137 * igb_rx_bind - Use existing DMA buffer to build mblk for receiving 138 * 139 * This function will use pre-bound DMA buffer to receive the packet 140 * and build mblk that will be sent upstream. 141 */ 142 static mblk_t * 143 igb_rx_bind(igb_rx_ring_t *rx_ring, uint32_t index, uint32_t pkt_len) 144 { 145 rx_control_block_t *current_rcb; 146 rx_control_block_t *free_rcb; 147 uint32_t free_index; 148 mblk_t *mp; 149 150 /* 151 * If the free list is empty, we cannot proceed to send 152 * the current DMA buffer upstream. We'll have to return 153 * and use bcopy to process the packet. 154 */ 155 if (igb_atomic_reserve(&rx_ring->rcb_free, 1) < 0) 156 return (NULL); 157 158 current_rcb = rx_ring->work_list[index]; 159 /* 160 * If the mp of the rx control block is NULL, try to do 161 * desballoc again. 162 */ 163 if (current_rcb->mp == NULL) { 164 current_rcb->mp = desballoc((unsigned char *) 165 (current_rcb->rx_buf.address - IPHDR_ALIGN_ROOM), 166 (current_rcb->rx_buf.size + IPHDR_ALIGN_ROOM), 167 0, ¤t_rcb->free_rtn); 168 /* 169 * If it is failed to built a mblk using the current 170 * DMA buffer, we have to return and use bcopy to 171 * process the packet. 172 */ 173 if (current_rcb->mp == NULL) { 174 atomic_inc_32(&rx_ring->rcb_free); 175 return (NULL); 176 } 177 } 178 /* 179 * Sync up the data received 180 */ 181 DMA_SYNC(¤t_rcb->rx_buf, DDI_DMA_SYNC_FORKERNEL); 182 183 mp = current_rcb->mp; 184 current_rcb->mp = NULL; 185 current_rcb->state = RCB_SENDUP; 186 187 mp->b_wptr = mp->b_rptr + pkt_len; 188 mp->b_next = mp->b_cont = NULL; 189 190 /* 191 * Strip off one free rx control block from the free list 192 */ 193 free_index = rx_ring->rcb_head; 194 free_rcb = rx_ring->free_list[free_index]; 195 ASSERT(free_rcb != NULL); 196 rx_ring->free_list[free_index] = NULL; 197 rx_ring->rcb_head = NEXT_INDEX(free_index, 1, rx_ring->free_list_size); 198 199 /* 200 * Put the rx control block to the work list 201 */ 202 rx_ring->work_list[index] = free_rcb; 203 204 return (mp); 205 } 206 207 /* 208 * igb_rx_assoc_hcksum 209 * 210 * Check the rx hardware checksum status and associate the hcksum flags 211 */ 212 static void 213 igb_rx_assoc_hcksum(mblk_t *mp, uint32_t status_error) 214 { 215 uint32_t hcksum_flags = 0; 216 217 /* Ignore Checksum Indication */ 218 if (status_error & E1000_RXD_STAT_IXSM) 219 return; 220 221 /* 222 * Check TCP/UDP checksum 223 */ 224 if (((status_error & E1000_RXD_STAT_TCPCS) || 225 (status_error & E1000_RXD_STAT_UDPCS)) && 226 !(status_error & E1000_RXDEXT_STATERR_TCPE)) 227 hcksum_flags |= HCK_FULLCKSUM | HCK_FULLCKSUM_OK; 228 229 /* 230 * Check IP Checksum 231 */ 232 if ((status_error & E1000_RXD_STAT_IPCS) && 233 !(status_error & E1000_RXDEXT_STATERR_IPE)) 234 hcksum_flags |= HCK_IPV4_HDRCKSUM; 235 236 if (hcksum_flags != 0) { 237 (void) hcksum_assoc(mp, 238 NULL, NULL, 0, 0, 0, 0, hcksum_flags, 0); 239 } 240 } 241 242 /* 243 * igb_rx - Receive the data of one ring 244 * 245 * This function goes throught h/w descriptor in one specified rx ring, 246 * receives the data if the descriptor status shows the data is ready. 247 * It returns a chain of mblks containing the received data, to be 248 * passed up to mac_rx(). 249 */ 250 mblk_t * 251 igb_rx(igb_rx_ring_t *rx_ring) 252 { 253 union e1000_adv_rx_desc *current_rbd; 254 rx_control_block_t *current_rcb; 255 mblk_t *mp; 256 mblk_t *mblk_head; 257 mblk_t **mblk_tail; 258 uint32_t rx_next; 259 uint32_t rx_tail; 260 uint32_t pkt_len; 261 uint32_t status_error; 262 uint32_t pkt_num; 263 igb_t *igb = rx_ring->igb; 264 265 mblk_head = NULL; 266 mblk_tail = &mblk_head; 267 268 /* 269 * Sync the receive descriptors before 270 * accepting the packets 271 */ 272 DMA_SYNC(&rx_ring->rbd_area, DDI_DMA_SYNC_FORKERNEL); 273 274 /* 275 * Get the start point of rx bd ring which should be examined 276 * during this cycle. 277 */ 278 rx_next = rx_ring->rbd_next; 279 280 current_rbd = &rx_ring->rbd_ring[rx_next]; 281 pkt_num = 0; 282 status_error = current_rbd->wb.upper.status_error; 283 while (status_error & E1000_RXD_STAT_DD) { 284 /* 285 * If hardware has found the errors, but the error 286 * is hardware checksum error, here does not discard the 287 * packet, and let upper layer compute the checksum; 288 * Otherwise discard the packet. 289 */ 290 if ((status_error & E1000_RXDEXT_ERR_FRAME_ERR_MASK) || 291 !(status_error & E1000_RXD_STAT_EOP)) { 292 IGB_DEBUG_STAT(rx_ring->stat_frame_error); 293 goto rx_discard; 294 } 295 296 IGB_DEBUG_STAT_COND(rx_ring->stat_cksum_error, 297 (status_error & E1000_RXDEXT_STATERR_TCPE) || 298 (status_error & E1000_RXDEXT_STATERR_IPE)); 299 300 pkt_len = current_rbd->wb.upper.length; 301 mp = NULL; 302 /* 303 * For packets with length more than the copy threshold, 304 * we'll firstly try to use the existed DMA buffer to built 305 * a mblk and send the mblk upstream. 306 * 307 * If the first method fails, or the packet length is less 308 * than the copy threshold, we'll allocate a new mblk and 309 * copy the packet data to the mblk. 310 */ 311 if (pkt_len > rx_ring->copy_thresh) 312 mp = igb_rx_bind(rx_ring, rx_next, pkt_len); 313 314 if (mp == NULL) 315 mp = igb_rx_copy(rx_ring, rx_next, pkt_len); 316 317 if (mp != NULL) { 318 /* 319 * Check h/w checksum offload status 320 */ 321 if (igb->rx_hcksum_enable) 322 igb_rx_assoc_hcksum(mp, status_error); 323 324 *mblk_tail = mp; 325 mblk_tail = &mp->b_next; 326 } 327 328 rx_discard: 329 /* 330 * Reset rx descriptor read bits 331 */ 332 current_rcb = rx_ring->work_list[rx_next]; 333 current_rbd->read.pkt_addr = current_rcb->rx_buf.dma_address; 334 current_rbd->read.hdr_addr = 0; 335 336 rx_next = NEXT_INDEX(rx_next, 1, rx_ring->ring_size); 337 338 /* 339 * The receive function is in interrupt context, so here 340 * limit_per_intr is used to avoid doing receiving too long 341 * per interrupt. 342 */ 343 if (++pkt_num > rx_ring->limit_per_intr) { 344 IGB_DEBUG_STAT(rx_ring->stat_exceed_pkt); 345 break; 346 } 347 348 current_rbd = &rx_ring->rbd_ring[rx_next]; 349 status_error = current_rbd->wb.upper.status_error; 350 } 351 352 DMA_SYNC(&rx_ring->rbd_area, DDI_DMA_SYNC_FORDEV); 353 354 rx_ring->rbd_next = rx_next; 355 356 /* 357 * Update the h/w tail accordingly 358 */ 359 rx_tail = PREV_INDEX(rx_next, 1, rx_ring->ring_size); 360 361 E1000_WRITE_REG(&igb->hw, E1000_RDT(rx_ring->index), rx_tail); 362 363 return (mblk_head); 364 } 365