xref: /titanic_41/usr/src/uts/common/io/ixgbe/ixgbe_rx.c (revision 6a634c9dca3093f3922e4b7ab826d7bdf17bf78e)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright(c) 2007-2010 Intel Corporation. All rights reserved.
24  */
25 
26 /*
27  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
28  */
29 
30 #include "ixgbe_sw.h"
31 
32 /* function prototypes */
33 static mblk_t *ixgbe_rx_bind(ixgbe_rx_data_t *, uint32_t, uint32_t);
34 static mblk_t *ixgbe_rx_copy(ixgbe_rx_data_t *, uint32_t, uint32_t);
35 static void ixgbe_rx_assoc_hcksum(mblk_t *, uint32_t);
36 static mblk_t *ixgbe_lro_bind(ixgbe_rx_data_t *, uint32_t, uint32_t, uint32_t);
37 static mblk_t *ixgbe_lro_copy(ixgbe_rx_data_t *, uint32_t, uint32_t, uint32_t);
38 static int ixgbe_lro_get_start(ixgbe_rx_data_t *, uint32_t);
39 static uint32_t ixgbe_lro_get_first(ixgbe_rx_data_t *, uint32_t);
40 
41 #ifndef IXGBE_DEBUG
42 #pragma inline(ixgbe_rx_assoc_hcksum)
43 #pragma inline(ixgbe_lro_get_start)
44 #pragma inline(ixgbe_lro_get_first)
45 #endif
46 
47 /*
48  * ixgbe_rx_recycle - The call-back function to reclaim rx buffer.
49  *
50  * This function is called when an mp is freed by the user thru
51  * freeb call (Only for mp constructed through desballoc call).
52  * It returns back the freed buffer to the free list.
53  */
54 void
ixgbe_rx_recycle(caddr_t arg)55 ixgbe_rx_recycle(caddr_t arg)
56 {
57 	ixgbe_t *ixgbe;
58 	ixgbe_rx_ring_t *rx_ring;
59 	ixgbe_rx_data_t	*rx_data;
60 	rx_control_block_t *recycle_rcb;
61 	uint32_t free_index;
62 	uint32_t ref_cnt;
63 
64 	recycle_rcb = (rx_control_block_t *)(uintptr_t)arg;
65 	rx_data = recycle_rcb->rx_data;
66 	rx_ring = rx_data->rx_ring;
67 	ixgbe = rx_ring->ixgbe;
68 
69 	if (recycle_rcb->ref_cnt == 0) {
70 		/*
71 		 * This case only happens when rx buffers are being freed
72 		 * in ixgbe_stop() and freemsg() is called.
73 		 */
74 		return;
75 	}
76 
77 	ASSERT(recycle_rcb->mp == NULL);
78 
79 	/*
80 	 * Using the recycled data buffer to generate a new mblk
81 	 */
82 	recycle_rcb->mp = desballoc((unsigned char *)
83 	    recycle_rcb->rx_buf.address,
84 	    recycle_rcb->rx_buf.size,
85 	    0, &recycle_rcb->free_rtn);
86 
87 	/*
88 	 * Put the recycled rx control block into free list
89 	 */
90 	mutex_enter(&rx_data->recycle_lock);
91 
92 	free_index = rx_data->rcb_tail;
93 	ASSERT(rx_data->free_list[free_index] == NULL);
94 
95 	rx_data->free_list[free_index] = recycle_rcb;
96 	rx_data->rcb_tail = NEXT_INDEX(free_index, 1, rx_data->free_list_size);
97 
98 	mutex_exit(&rx_data->recycle_lock);
99 
100 	/*
101 	 * The atomic operation on the number of the available rx control
102 	 * blocks in the free list is used to make the recycling mutual
103 	 * exclusive with the receiving.
104 	 */
105 	atomic_inc_32(&rx_data->rcb_free);
106 	ASSERT(rx_data->rcb_free <= rx_data->free_list_size);
107 
108 	/*
109 	 * Considering the case that the interface is unplumbed
110 	 * and there are still some buffers held by the upper layer.
111 	 * When the buffer is returned back, we need to free it.
112 	 */
113 	ref_cnt = atomic_dec_32_nv(&recycle_rcb->ref_cnt);
114 	if (ref_cnt == 0) {
115 		if (recycle_rcb->mp != NULL) {
116 			freemsg(recycle_rcb->mp);
117 			recycle_rcb->mp = NULL;
118 		}
119 
120 		ixgbe_free_dma_buffer(&recycle_rcb->rx_buf);
121 
122 		mutex_enter(&ixgbe->rx_pending_lock);
123 		atomic_dec_32(&rx_data->rcb_pending);
124 		atomic_dec_32(&ixgbe->rcb_pending);
125 
126 		/*
127 		 * When there is not any buffer belonging to this rx_data
128 		 * held by the upper layer, the rx_data can be freed.
129 		 */
130 		if ((rx_data->flag & IXGBE_RX_STOPPED) &&
131 		    (rx_data->rcb_pending == 0))
132 			ixgbe_free_rx_ring_data(rx_data);
133 
134 		mutex_exit(&ixgbe->rx_pending_lock);
135 	}
136 }
137 
138 /*
139  * ixgbe_rx_copy - Use copy to process the received packet.
140  *
141  * This function will use bcopy to process the packet
142  * and send the copied packet upstream.
143  */
144 static mblk_t *
ixgbe_rx_copy(ixgbe_rx_data_t * rx_data,uint32_t index,uint32_t pkt_len)145 ixgbe_rx_copy(ixgbe_rx_data_t *rx_data, uint32_t index, uint32_t pkt_len)
146 {
147 	ixgbe_t *ixgbe;
148 	rx_control_block_t *current_rcb;
149 	mblk_t *mp;
150 
151 	ixgbe = rx_data->rx_ring->ixgbe;
152 	current_rcb = rx_data->work_list[index];
153 
154 	DMA_SYNC(&current_rcb->rx_buf, DDI_DMA_SYNC_FORKERNEL);
155 
156 	if (ixgbe_check_dma_handle(current_rcb->rx_buf.dma_handle) !=
157 	    DDI_FM_OK) {
158 		ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
159 		atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
160 		return (NULL);
161 	}
162 
163 	/*
164 	 * Allocate buffer to receive this packet
165 	 */
166 	mp = allocb(pkt_len + IPHDR_ALIGN_ROOM, 0);
167 	if (mp == NULL) {
168 		ixgbe_log(ixgbe, "ixgbe_rx_copy: allocate buffer failed");
169 		return (NULL);
170 	}
171 
172 	/*
173 	 * Copy the data received into the new cluster
174 	 */
175 	mp->b_rptr += IPHDR_ALIGN_ROOM;
176 	bcopy(current_rcb->rx_buf.address, mp->b_rptr, pkt_len);
177 	mp->b_wptr = mp->b_rptr + pkt_len;
178 
179 	return (mp);
180 }
181 
182 /*
183  * ixgbe_rx_bind - Use existing DMA buffer to build mblk for receiving.
184  *
185  * This function will use pre-bound DMA buffer to receive the packet
186  * and build mblk that will be sent upstream.
187  */
188 static mblk_t *
ixgbe_rx_bind(ixgbe_rx_data_t * rx_data,uint32_t index,uint32_t pkt_len)189 ixgbe_rx_bind(ixgbe_rx_data_t *rx_data, uint32_t index, uint32_t pkt_len)
190 {
191 	rx_control_block_t *current_rcb;
192 	rx_control_block_t *free_rcb;
193 	uint32_t free_index;
194 	mblk_t *mp;
195 	ixgbe_t	*ixgbe = rx_data->rx_ring->ixgbe;
196 
197 	/*
198 	 * If the free list is empty, we cannot proceed to send
199 	 * the current DMA buffer upstream. We'll have to return
200 	 * and use bcopy to process the packet.
201 	 */
202 	if (ixgbe_atomic_reserve(&rx_data->rcb_free, 1) < 0)
203 		return (NULL);
204 
205 	current_rcb = rx_data->work_list[index];
206 	/*
207 	 * If the mp of the rx control block is NULL, try to do
208 	 * desballoc again.
209 	 */
210 	if (current_rcb->mp == NULL) {
211 		current_rcb->mp = desballoc((unsigned char *)
212 		    current_rcb->rx_buf.address,
213 		    current_rcb->rx_buf.size,
214 		    0, &current_rcb->free_rtn);
215 		/*
216 		 * If it is failed to built a mblk using the current
217 		 * DMA buffer, we have to return and use bcopy to
218 		 * process the packet.
219 		 */
220 		if (current_rcb->mp == NULL) {
221 			atomic_inc_32(&rx_data->rcb_free);
222 			return (NULL);
223 		}
224 	}
225 	/*
226 	 * Sync up the data received
227 	 */
228 	DMA_SYNC(&current_rcb->rx_buf, DDI_DMA_SYNC_FORKERNEL);
229 
230 	if (ixgbe_check_dma_handle(current_rcb->rx_buf.dma_handle) !=
231 	    DDI_FM_OK) {
232 		ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
233 		atomic_inc_32(&rx_data->rcb_free);
234 		atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
235 		return (NULL);
236 	}
237 
238 	mp = current_rcb->mp;
239 	current_rcb->mp = NULL;
240 	atomic_inc_32(&current_rcb->ref_cnt);
241 
242 	mp->b_wptr = mp->b_rptr + pkt_len;
243 	mp->b_next = mp->b_cont = NULL;
244 
245 	/*
246 	 * Strip off one free rx control block from the free list
247 	 */
248 	free_index = rx_data->rcb_head;
249 	free_rcb = rx_data->free_list[free_index];
250 	ASSERT(free_rcb != NULL);
251 	rx_data->free_list[free_index] = NULL;
252 	rx_data->rcb_head = NEXT_INDEX(free_index, 1, rx_data->free_list_size);
253 
254 	/*
255 	 * Put the rx control block to the work list
256 	 */
257 	rx_data->work_list[index] = free_rcb;
258 
259 	return (mp);
260 }
261 
262 /*
263  * ixgbe_lro_bind - Use existing DMA buffer to build LRO mblk for receiving.
264  *
265  * This function will use pre-bound DMA buffers to receive the packet
266  * and build LRO mblk that will be sent upstream.
267  */
268 static mblk_t *
ixgbe_lro_bind(ixgbe_rx_data_t * rx_data,uint32_t lro_start,uint32_t lro_num,uint32_t pkt_len)269 ixgbe_lro_bind(ixgbe_rx_data_t *rx_data, uint32_t lro_start,
270     uint32_t lro_num, uint32_t pkt_len)
271 {
272 	rx_control_block_t *current_rcb;
273 	union ixgbe_adv_rx_desc *current_rbd;
274 	rx_control_block_t *free_rcb;
275 	uint32_t free_index;
276 	int lro_next;
277 	uint32_t last_pkt_len;
278 	uint32_t i;
279 	mblk_t *mp;
280 	mblk_t *mblk_head;
281 	mblk_t **mblk_tail;
282 	ixgbe_t	*ixgbe = rx_data->rx_ring->ixgbe;
283 
284 	/*
285 	 * If the free list is empty, we cannot proceed to send
286 	 * the current DMA buffer upstream. We'll have to return
287 	 * and use bcopy to process the packet.
288 	 */
289 	if (ixgbe_atomic_reserve(&rx_data->rcb_free, lro_num) < 0)
290 		return (NULL);
291 	current_rcb = rx_data->work_list[lro_start];
292 
293 	/*
294 	 * If any one of the rx data blocks can not support
295 	 * lro bind  operation,  We'll have to return and use
296 	 * bcopy to process the lro  packet.
297 	 */
298 	for (i = lro_num; i > 0; i--) {
299 		/*
300 		 * Sync up the data received
301 		 */
302 		DMA_SYNC(&current_rcb->rx_buf, DDI_DMA_SYNC_FORKERNEL);
303 
304 		if (ixgbe_check_dma_handle(current_rcb->rx_buf.dma_handle) !=
305 		    DDI_FM_OK) {
306 			ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
307 			atomic_add_32(&rx_data->rcb_free, lro_num);
308 			atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
309 			return (NULL);
310 		}
311 
312 		/*
313 		 * If the mp of the rx control block is NULL, try to do
314 		 * desballoc again.
315 		 */
316 		if (current_rcb->mp == NULL) {
317 			current_rcb->mp = desballoc((unsigned char *)
318 			    current_rcb->rx_buf.address,
319 			    current_rcb->rx_buf.size,
320 			    0, &current_rcb->free_rtn);
321 			/*
322 			 * If it is failed to built a mblk using the current
323 			 * DMA buffer, we have to return and use bcopy to
324 			 * process the packet.
325 			 */
326 			if (current_rcb->mp == NULL) {
327 				atomic_add_32(&rx_data->rcb_free, lro_num);
328 				return (NULL);
329 			}
330 		}
331 		if (current_rcb->lro_next != -1)
332 			lro_next = current_rcb->lro_next;
333 		current_rcb = rx_data->work_list[lro_next];
334 	}
335 
336 	mblk_head = NULL;
337 	mblk_tail = &mblk_head;
338 	lro_next = lro_start;
339 	last_pkt_len = pkt_len - ixgbe->rx_buf_size * (lro_num - 1);
340 	current_rcb = rx_data->work_list[lro_next];
341 	current_rbd = &rx_data->rbd_ring[lro_next];
342 	while (lro_num --) {
343 		mp = current_rcb->mp;
344 		current_rcb->mp = NULL;
345 		atomic_inc_32(&current_rcb->ref_cnt);
346 		if (lro_num != 0)
347 			mp->b_wptr = mp->b_rptr + ixgbe->rx_buf_size;
348 		else
349 			mp->b_wptr = mp->b_rptr + last_pkt_len;
350 		mp->b_next = mp->b_cont = NULL;
351 		*mblk_tail = mp;
352 		mblk_tail = &mp->b_cont;
353 
354 		/*
355 		 * Strip off one free rx control block from the free list
356 		 */
357 		free_index = rx_data->rcb_head;
358 		free_rcb = rx_data->free_list[free_index];
359 		ASSERT(free_rcb != NULL);
360 		rx_data->free_list[free_index] = NULL;
361 		rx_data->rcb_head = NEXT_INDEX(free_index, 1,
362 		    rx_data->free_list_size);
363 
364 		/*
365 		 * Put the rx control block to the work list
366 		 */
367 		rx_data->work_list[lro_next] = free_rcb;
368 		lro_next = current_rcb->lro_next;
369 		current_rcb->lro_next = -1;
370 		current_rcb->lro_prev = -1;
371 		current_rcb->lro_pkt = B_FALSE;
372 		current_rbd->read.pkt_addr = free_rcb->rx_buf.dma_address;
373 		current_rbd->read.hdr_addr = 0;
374 		if (lro_next == -1)
375 			break;
376 		current_rcb = rx_data->work_list[lro_next];
377 		current_rbd = &rx_data->rbd_ring[lro_next];
378 	}
379 	return (mblk_head);
380 }
381 
382 /*
383  * ixgbe_lro_copy - Use copy to process the received LRO packet.
384  *
385  * This function will use bcopy to process the LRO  packet
386  * and send the copied packet upstream.
387  */
388 static mblk_t *
ixgbe_lro_copy(ixgbe_rx_data_t * rx_data,uint32_t lro_start,uint32_t lro_num,uint32_t pkt_len)389 ixgbe_lro_copy(ixgbe_rx_data_t *rx_data, uint32_t lro_start,
390     uint32_t lro_num, uint32_t pkt_len)
391 {
392 	ixgbe_t *ixgbe;
393 	rx_control_block_t *current_rcb;
394 	union ixgbe_adv_rx_desc *current_rbd;
395 	mblk_t *mp;
396 	uint32_t last_pkt_len;
397 	int lro_next;
398 	uint32_t i;
399 
400 	ixgbe = rx_data->rx_ring->ixgbe;
401 
402 	/*
403 	 * Allocate buffer to receive this LRO packet
404 	 */
405 	mp = allocb(pkt_len + IPHDR_ALIGN_ROOM, 0);
406 	if (mp == NULL) {
407 		ixgbe_log(ixgbe, "LRO copy MP alloc failed");
408 		return (NULL);
409 	}
410 
411 	current_rcb = rx_data->work_list[lro_start];
412 
413 	/*
414 	 * Sync up the LRO packet data received
415 	 */
416 	for (i = lro_num; i > 0; i--) {
417 		DMA_SYNC(&current_rcb->rx_buf, DDI_DMA_SYNC_FORKERNEL);
418 
419 		if (ixgbe_check_dma_handle(current_rcb->rx_buf.dma_handle) !=
420 		    DDI_FM_OK) {
421 			ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
422 			atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
423 			return (NULL);
424 		}
425 		if (current_rcb->lro_next != -1)
426 			lro_next = current_rcb->lro_next;
427 		current_rcb = rx_data->work_list[lro_next];
428 	}
429 	lro_next = lro_start;
430 	current_rcb = rx_data->work_list[lro_next];
431 	current_rbd = &rx_data->rbd_ring[lro_next];
432 	last_pkt_len = pkt_len - ixgbe->rx_buf_size * (lro_num - 1);
433 
434 	/*
435 	 * Copy the data received into the new cluster
436 	 */
437 	mp->b_rptr += IPHDR_ALIGN_ROOM;
438 	mp->b_wptr += IPHDR_ALIGN_ROOM;
439 	while (lro_num --) {
440 		if (lro_num != 0) {
441 			bcopy(current_rcb->rx_buf.address, mp->b_wptr,
442 			    ixgbe->rx_buf_size);
443 			mp->b_wptr += ixgbe->rx_buf_size;
444 		} else {
445 			bcopy(current_rcb->rx_buf.address, mp->b_wptr,
446 			    last_pkt_len);
447 			mp->b_wptr += last_pkt_len;
448 		}
449 		lro_next = current_rcb->lro_next;
450 		current_rcb->lro_next = -1;
451 		current_rcb->lro_prev = -1;
452 		current_rcb->lro_pkt = B_FALSE;
453 		current_rbd->read.pkt_addr = current_rcb->rx_buf.dma_address;
454 		current_rbd->read.hdr_addr = 0;
455 		if (lro_next == -1)
456 			break;
457 		current_rcb = rx_data->work_list[lro_next];
458 		current_rbd = &rx_data->rbd_ring[lro_next];
459 	}
460 
461 	return (mp);
462 }
463 
464 /*
465  * ixgbe_lro_get_start - get the start rcb index in one LRO packet
466  */
467 static int
ixgbe_lro_get_start(ixgbe_rx_data_t * rx_data,uint32_t rx_next)468 ixgbe_lro_get_start(ixgbe_rx_data_t *rx_data, uint32_t rx_next)
469 {
470 	int lro_prev;
471 	int lro_start;
472 	uint32_t lro_num = 1;
473 	rx_control_block_t *prev_rcb;
474 	rx_control_block_t *current_rcb = rx_data->work_list[rx_next];
475 	lro_prev = current_rcb->lro_prev;
476 
477 	while (lro_prev != -1) {
478 		lro_num ++;
479 		prev_rcb = rx_data->work_list[lro_prev];
480 		lro_start = lro_prev;
481 		lro_prev = prev_rcb->lro_prev;
482 	}
483 	rx_data->lro_num = lro_num;
484 	return (lro_start);
485 }
486 
487 /*
488  * ixgbe_lro_get_first - get the first LRO rcb index
489  */
490 static uint32_t
ixgbe_lro_get_first(ixgbe_rx_data_t * rx_data,uint32_t rx_next)491 ixgbe_lro_get_first(ixgbe_rx_data_t *rx_data, uint32_t rx_next)
492 {
493 	rx_control_block_t *current_rcb;
494 	uint32_t lro_first;
495 	lro_first = rx_data->lro_first;
496 	current_rcb = rx_data->work_list[lro_first];
497 	while ((!current_rcb->lro_pkt) && (lro_first != rx_next)) {
498 		lro_first =  NEXT_INDEX(lro_first, 1, rx_data->ring_size);
499 		current_rcb = rx_data->work_list[lro_first];
500 	}
501 	rx_data->lro_first = lro_first;
502 	return (lro_first);
503 }
504 
505 /*
506  * ixgbe_rx_assoc_hcksum - Check the rx hardware checksum status and associate
507  * the hcksum flags.
508  */
509 static void
ixgbe_rx_assoc_hcksum(mblk_t * mp,uint32_t status_error)510 ixgbe_rx_assoc_hcksum(mblk_t *mp, uint32_t status_error)
511 {
512 	uint32_t hcksum_flags = 0;
513 
514 	/*
515 	 * Check TCP/UDP checksum
516 	 */
517 	if ((status_error & IXGBE_RXD_STAT_L4CS) &&
518 	    !(status_error & IXGBE_RXDADV_ERR_TCPE))
519 		hcksum_flags |= HCK_FULLCKSUM_OK;
520 
521 	/*
522 	 * Check IP Checksum
523 	 */
524 	if ((status_error & IXGBE_RXD_STAT_IPCS) &&
525 	    !(status_error & IXGBE_RXDADV_ERR_IPE))
526 		hcksum_flags |= HCK_IPV4_HDRCKSUM_OK;
527 
528 	if (hcksum_flags != 0) {
529 		mac_hcksum_set(mp, 0, 0, 0, 0, hcksum_flags);
530 	}
531 }
532 
533 /*
534  * ixgbe_ring_rx - Receive the data of one ring.
535  *
536  * This function goes throught h/w descriptor in one specified rx ring,
537  * receives the data if the descriptor status shows the data is ready.
538  * It returns a chain of mblks containing the received data, to be
539  * passed up to mac_rx().
540  */
541 mblk_t *
ixgbe_ring_rx(ixgbe_rx_ring_t * rx_ring,int poll_bytes)542 ixgbe_ring_rx(ixgbe_rx_ring_t *rx_ring, int poll_bytes)
543 {
544 	union ixgbe_adv_rx_desc *current_rbd;
545 	rx_control_block_t *current_rcb;
546 	mblk_t *mp;
547 	mblk_t *mblk_head;
548 	mblk_t **mblk_tail;
549 	uint32_t rx_next;
550 	uint32_t rx_tail;
551 	uint32_t pkt_len;
552 	uint32_t status_error;
553 	uint32_t pkt_num;
554 	uint32_t rsc_cnt;
555 	uint32_t lro_first;
556 	uint32_t lro_start;
557 	uint32_t lro_next;
558 	boolean_t lro_eop;
559 	uint32_t received_bytes;
560 	ixgbe_t *ixgbe = rx_ring->ixgbe;
561 	ixgbe_rx_data_t *rx_data;
562 
563 	if ((ixgbe->ixgbe_state & IXGBE_SUSPENDED) ||
564 	    (ixgbe->ixgbe_state & IXGBE_ERROR) ||
565 	    (ixgbe->ixgbe_state & IXGBE_OVERTEMP) ||
566 	    !(ixgbe->ixgbe_state & IXGBE_STARTED))
567 		return (NULL);
568 
569 	rx_data = rx_ring->rx_data;
570 	lro_eop = B_FALSE;
571 	mblk_head = NULL;
572 	mblk_tail = &mblk_head;
573 
574 	/*
575 	 * Sync the receive descriptors before accepting the packets
576 	 */
577 	DMA_SYNC(&rx_data->rbd_area, DDI_DMA_SYNC_FORKERNEL);
578 
579 	if (ixgbe_check_dma_handle(rx_data->rbd_area.dma_handle) != DDI_FM_OK) {
580 		ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
581 		atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
582 		return (NULL);
583 	}
584 
585 	/*
586 	 * Get the start point of rx bd ring which should be examined
587 	 * during this cycle.
588 	 */
589 	rx_next = rx_data->rbd_next;
590 	current_rbd = &rx_data->rbd_ring[rx_next];
591 	received_bytes = 0;
592 	pkt_num = 0;
593 	status_error = current_rbd->wb.upper.status_error;
594 	while (status_error & IXGBE_RXD_STAT_DD) {
595 		/*
596 		 * If adapter has found errors, but the error
597 		 * is hardware checksum error, this does not discard the
598 		 * packet: let upper layer compute the checksum;
599 		 * Otherwise discard the packet.
600 		 */
601 		if ((status_error & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) ||
602 		    ((!ixgbe->lro_enable) &&
603 		    (!(status_error & IXGBE_RXD_STAT_EOP)))) {
604 			IXGBE_DEBUG_STAT(rx_ring->stat_frame_error);
605 			goto rx_discard;
606 		}
607 
608 		IXGBE_DEBUG_STAT_COND(rx_ring->stat_cksum_error,
609 		    (status_error & IXGBE_RXDADV_ERR_TCPE) ||
610 		    (status_error & IXGBE_RXDADV_ERR_IPE));
611 
612 		if (ixgbe->lro_enable) {
613 			rsc_cnt =  (current_rbd->wb.lower.lo_dword.data &
614 			    IXGBE_RXDADV_RSCCNT_MASK) >>
615 			    IXGBE_RXDADV_RSCCNT_SHIFT;
616 			if (rsc_cnt != 0) {
617 				if (status_error & IXGBE_RXD_STAT_EOP) {
618 					pkt_len = current_rbd->wb.upper.length;
619 					if (rx_data->work_list[rx_next]->
620 					    lro_prev != -1) {
621 						lro_start =
622 						    ixgbe_lro_get_start(rx_data,
623 						    rx_next);
624 						ixgbe->lro_pkt_count++;
625 						pkt_len +=
626 						    (rx_data->lro_num  - 1) *
627 						    ixgbe->rx_buf_size;
628 						lro_eop = B_TRUE;
629 					}
630 				} else {
631 					lro_next = (status_error &
632 					    IXGBE_RXDADV_NEXTP_MASK) >>
633 					    IXGBE_RXDADV_NEXTP_SHIFT;
634 					rx_data->work_list[lro_next]->lro_prev
635 					    = rx_next;
636 					rx_data->work_list[rx_next]->lro_next =
637 					    lro_next;
638 					rx_data->work_list[rx_next]->lro_pkt =
639 					    B_TRUE;
640 					goto rx_discard;
641 				}
642 
643 			} else {
644 				pkt_len = current_rbd->wb.upper.length;
645 			}
646 		} else {
647 			pkt_len = current_rbd->wb.upper.length;
648 		}
649 
650 
651 		if ((poll_bytes != IXGBE_POLL_NULL) &&
652 		    ((received_bytes + pkt_len) > poll_bytes))
653 			break;
654 
655 		received_bytes += pkt_len;
656 		mp = NULL;
657 
658 		/*
659 		 * For packets with length more than the copy threshold,
660 		 * we'll first try to use the existing DMA buffer to build
661 		 * an mblk and send the mblk upstream.
662 		 *
663 		 * If the first method fails, or the packet length is less
664 		 * than the copy threshold, we'll allocate a new mblk and
665 		 * copy the packet data to the new mblk.
666 		 */
667 		if (lro_eop) {
668 			mp = ixgbe_lro_bind(rx_data, lro_start,
669 			    rx_data->lro_num, pkt_len);
670 			if (mp == NULL)
671 				mp = ixgbe_lro_copy(rx_data, lro_start,
672 				    rx_data->lro_num, pkt_len);
673 			lro_eop = B_FALSE;
674 			rx_data->lro_num = 0;
675 
676 		} else {
677 			if (pkt_len > ixgbe->rx_copy_thresh)
678 				mp = ixgbe_rx_bind(rx_data, rx_next, pkt_len);
679 
680 			if (mp == NULL)
681 				mp = ixgbe_rx_copy(rx_data, rx_next, pkt_len);
682 		}
683 		if (mp != NULL) {
684 			/*
685 			 * Check h/w checksum offload status
686 			 */
687 			if (ixgbe->rx_hcksum_enable)
688 				ixgbe_rx_assoc_hcksum(mp, status_error);
689 
690 			*mblk_tail = mp;
691 			mblk_tail = &mp->b_next;
692 		}
693 
694 rx_discard:
695 		/*
696 		 * Reset rx descriptor read bits
697 		 */
698 		current_rcb = rx_data->work_list[rx_next];
699 		if (ixgbe->lro_enable) {
700 			if (!current_rcb->lro_pkt) {
701 				current_rbd->read.pkt_addr =
702 				    current_rcb->rx_buf.dma_address;
703 				current_rbd->read.hdr_addr = 0;
704 			}
705 		} else {
706 			current_rbd->read.pkt_addr =
707 			    current_rcb->rx_buf.dma_address;
708 			current_rbd->read.hdr_addr = 0;
709 		}
710 
711 		rx_next = NEXT_INDEX(rx_next, 1, rx_data->ring_size);
712 
713 		/*
714 		 * The receive function is in interrupt context, so here
715 		 * rx_limit_per_intr is used to avoid doing receiving too long
716 		 * per interrupt.
717 		 */
718 		if (++pkt_num > ixgbe->rx_limit_per_intr) {
719 			IXGBE_DEBUG_STAT(rx_ring->stat_exceed_pkt);
720 			break;
721 		}
722 
723 		current_rbd = &rx_data->rbd_ring[rx_next];
724 		status_error = current_rbd->wb.upper.status_error;
725 	}
726 
727 	rx_ring->stat_rbytes += received_bytes;
728 	rx_ring->stat_ipackets += pkt_num;
729 
730 	DMA_SYNC(&rx_data->rbd_area, DDI_DMA_SYNC_FORDEV);
731 
732 	rx_data->rbd_next = rx_next;
733 
734 	/*
735 	 * Update the h/w tail accordingly
736 	 */
737 	if (ixgbe->lro_enable) {
738 		lro_first = ixgbe_lro_get_first(rx_data, rx_next);
739 		rx_tail = PREV_INDEX(lro_first, 1, rx_data->ring_size);
740 	} else
741 		rx_tail = PREV_INDEX(rx_next, 1, rx_data->ring_size);
742 
743 	IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_RDT(rx_ring->hw_index), rx_tail);
744 
745 	if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
746 		ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
747 		atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
748 	}
749 
750 	return (mblk_head);
751 }
752 
753 mblk_t *
ixgbe_ring_rx_poll(void * arg,int n_bytes)754 ixgbe_ring_rx_poll(void *arg, int n_bytes)
755 {
756 	ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)arg;
757 	mblk_t *mp = NULL;
758 
759 	ASSERT(n_bytes >= 0);
760 
761 	if (n_bytes == 0)
762 		return (NULL);
763 
764 	mutex_enter(&rx_ring->rx_lock);
765 	mp = ixgbe_ring_rx(rx_ring, n_bytes);
766 	mutex_exit(&rx_ring->rx_lock);
767 
768 	return (mp);
769 }
770