xref: /titanic_41/usr/src/uts/common/io/igb/igb_tx.c (revision b135e6b9f5f1d0abe81c35d8477965bdd89203c0)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright(c) 2007-2010 Intel Corporation. All rights reserved.
24  */
25 
26 /*
27  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
28  */
29 
30 #include "igb_sw.h"
31 
32 static boolean_t igb_tx(igb_tx_ring_t *, mblk_t *);
33 static int igb_tx_copy(igb_tx_ring_t *, tx_control_block_t *, mblk_t *,
34     uint32_t, boolean_t);
35 static int igb_tx_bind(igb_tx_ring_t *, tx_control_block_t *, mblk_t *,
36     uint32_t);
37 static int igb_tx_fill_ring(igb_tx_ring_t *, link_list_t *, tx_context_t *,
38     size_t);
39 static void igb_save_desc(tx_control_block_t *, uint64_t, size_t);
40 static tx_control_block_t *igb_get_free_list(igb_tx_ring_t *);
41 static int igb_get_tx_context(mblk_t *, tx_context_t *);
42 static boolean_t igb_check_tx_context(igb_tx_ring_t *, tx_context_t *);
43 static void igb_fill_tx_context(struct e1000_adv_tx_context_desc *,
44     tx_context_t *, uint32_t);
45 
46 #ifndef IGB_DEBUG
47 #pragma inline(igb_save_desc)
48 #pragma inline(igb_get_tx_context)
49 #pragma inline(igb_check_tx_context)
50 #pragma inline(igb_fill_tx_context)
51 #endif
52 
53 mblk_t *
igb_tx_ring_send(void * arg,mblk_t * mp)54 igb_tx_ring_send(void *arg, mblk_t *mp)
55 {
56 	igb_tx_ring_t *tx_ring = (igb_tx_ring_t *)arg;
57 	igb_t *igb;
58 
59 	ASSERT(tx_ring != NULL);
60 
61 	igb = tx_ring->igb;
62 
63 	if ((igb->igb_state & IGB_SUSPENDED) ||
64 	    (igb->igb_state & IGB_ERROR) ||
65 	    !(igb->igb_state & IGB_STARTED) ||
66 	    igb->link_state != LINK_STATE_UP) {
67 		freemsg(mp);
68 		return (NULL);
69 	}
70 
71 	return ((igb_tx(tx_ring, mp)) ? NULL : mp);
72 }
73 
74 /*
75  * igb_tx - Main transmit processing
76  *
77  * Called from igb_m_tx with an mblk ready to transmit. this
78  * routine sets up the transmit descriptors and sends data to
79  * the wire.
80  *
81  * One mblk can consist of several fragments, each fragment
82  * will be processed with different methods based on the size.
83  * For the fragments with size less than the bcopy threshold,
84  * they will be processed by using bcopy; otherwise, they will
85  * be processed by using DMA binding.
86  *
87  * To process the mblk, a tx control block is got from the
88  * free list. One tx control block contains one tx buffer, which
89  * is used to copy mblk fragments' data; and one tx DMA handle,
90  * which is used to bind a mblk fragment with DMA resource.
91  *
92  * Several small mblk fragments can be copied into one tx control
93  * block's buffer, and then the buffer will be transmitted with
94  * one tx descriptor.
95  *
96  * A large fragment only binds with one tx control block's DMA
97  * handle, and it can span several tx descriptors for transmitting.
98  *
99  * So to transmit a packet (mblk), several tx control blocks can
100  * be used. After the processing, those tx control blocks will
101  * be put to the work list.
102  */
103 static boolean_t
igb_tx(igb_tx_ring_t * tx_ring,mblk_t * mp)104 igb_tx(igb_tx_ring_t *tx_ring, mblk_t *mp)
105 {
106 	igb_t *igb = tx_ring->igb;
107 	tx_type_t current_flag, next_flag;
108 	uint32_t current_len, next_len;
109 	uint32_t desc_total;
110 	size_t mbsize;
111 	int desc_num;
112 	boolean_t copy_done, eop;
113 	mblk_t *current_mp, *next_mp, *nmp;
114 	tx_control_block_t *tcb;
115 	tx_context_t tx_context, *ctx;
116 	link_list_t pending_list;
117 	mblk_t *hdr_new_mp = NULL;
118 	mblk_t *hdr_previous_mp = NULL;
119 	mblk_t *hdr_current_mp = NULL;
120 	uint32_t hdr_frag_len;
121 	uint32_t hdr_len, len;
122 	uint32_t copy_thresh;
123 
124 	copy_thresh = igb->tx_copy_thresh;
125 
126 	/* Get the mblk size */
127 	mbsize = 0;
128 	for (nmp = mp; nmp != NULL; nmp = nmp->b_cont) {
129 		mbsize += MBLKL(nmp);
130 	}
131 
132 	if (igb->tx_hcksum_enable) {
133 		ctx = &tx_context;
134 		/*
135 		 * Retrieve offloading context information from the mblk
136 		 * that will be used to decide whether/how to fill the
137 		 * context descriptor.
138 		 */
139 		if (igb_get_tx_context(mp, ctx) != TX_CXT_SUCCESS) {
140 			freemsg(mp);
141 			return (B_TRUE);
142 		}
143 
144 		if ((ctx->lso_flag &&
145 		    (mbsize > (ctx->mac_hdr_len + IGB_LSO_MAXLEN))) ||
146 		    (!ctx->lso_flag &&
147 		    (mbsize > (igb->max_frame_size - ETHERFCSL)))) {
148 			freemsg(mp);
149 			IGB_DEBUGLOG_0(igb, "igb_tx: packet oversize");
150 			return (B_TRUE);
151 		}
152 	} else {
153 		ctx = NULL;
154 		if (mbsize > (igb->max_frame_size - ETHERFCSL)) {
155 			freemsg(mp);
156 			IGB_DEBUGLOG_0(igb, "igb_tx: packet oversize");
157 			return (B_TRUE);
158 		}
159 	}
160 
161 	/*
162 	 * Check and recycle tx descriptors.
163 	 * The recycle threshold here should be selected carefully
164 	 */
165 	if (tx_ring->tbd_free < igb->tx_recycle_thresh)
166 		tx_ring->tx_recycle(tx_ring);
167 
168 	/*
169 	 * After the recycling, if the tbd_free is less than the
170 	 * tx_overload_threshold, assert overload, return B_FALSE;
171 	 * and we need to re-schedule the tx again.
172 	 */
173 	if (tx_ring->tbd_free < igb->tx_overload_thresh) {
174 		tx_ring->reschedule = B_TRUE;
175 		IGB_DEBUG_STAT(tx_ring->stat_overload);
176 		return (B_FALSE);
177 	}
178 
179 	/*
180 	 * The software should guarantee LSO packet header(MAC+IP+TCP)
181 	 * to be within one descriptor - this is required by h/w.
182 	 * Here will reallocate and refill the header if
183 	 * the headers(MAC+IP+TCP) is physical memory non-contiguous.
184 	 */
185 	if (ctx && ctx->lso_flag) {
186 		hdr_len = ctx->mac_hdr_len + ctx->ip_hdr_len + ctx->l4_hdr_len;
187 		len = MBLKL(mp);
188 		hdr_current_mp = mp;
189 		while (len < hdr_len) {
190 			hdr_previous_mp = hdr_current_mp;
191 			hdr_current_mp = hdr_current_mp->b_cont;
192 			len += MBLKL(hdr_current_mp);
193 		}
194 		/*
195 		 * If the header and the payload are in different mblks,
196 		 * we simply force the header to be copied into pre-allocated
197 		 * page-aligned buffer.
198 		 */
199 		if (len == hdr_len)
200 			goto adjust_threshold;
201 
202 		hdr_frag_len = hdr_len - (len - MBLKL(hdr_current_mp));
203 		/*
204 		 * There are two cases we will reallocate
205 		 * a mblk for the last header fragment.
206 		 * 1. the header is in multiple mblks and
207 		 *    the last fragment shares the same mblk
208 		 *    with the payload
209 		 * 2. the header is in a single mblk shared
210 		 *    with the payload but the header crosses
211 		 *    a page.
212 		 */
213 		if ((hdr_current_mp != mp) ||
214 		    (P2NPHASE((uintptr_t)hdr_current_mp->b_rptr, igb->page_size)
215 		    < hdr_len)) {
216 			/*
217 			 * reallocate the mblk for the last header fragment,
218 			 * expect it to be copied into pre-allocated
219 			 * page-aligned buffer
220 			 */
221 			hdr_new_mp = allocb(hdr_frag_len, NULL);
222 			if (!hdr_new_mp) {
223 				return (B_FALSE);
224 			}
225 
226 			/* link the new header fragment with the other parts */
227 			bcopy(hdr_current_mp->b_rptr,
228 			    hdr_new_mp->b_rptr, hdr_frag_len);
229 			hdr_new_mp->b_wptr = hdr_new_mp->b_rptr + hdr_frag_len;
230 			hdr_new_mp->b_cont = hdr_current_mp;
231 			if (hdr_previous_mp)
232 				hdr_previous_mp->b_cont = hdr_new_mp;
233 			else
234 				mp = hdr_new_mp;
235 			hdr_current_mp->b_rptr += hdr_frag_len;
236 		}
237 adjust_threshold:
238 		/*
239 		 * adjust the bcopy threshhold to guarantee
240 		 * the header to use bcopy way
241 		 */
242 		if (copy_thresh < hdr_len)
243 			copy_thresh = hdr_len;
244 	}
245 
246 	/*
247 	 * The pending_list is a linked list that is used to save
248 	 * the tx control blocks that have packet data processed
249 	 * but have not put the data to the tx descriptor ring.
250 	 * It is used to reduce the lock contention of the tx_lock.
251 	 */
252 	LINK_LIST_INIT(&pending_list);
253 	desc_num = 0;
254 	desc_total = 0;
255 
256 	current_mp = mp;
257 	current_len = MBLKL(current_mp);
258 	/*
259 	 * Decide which method to use for the first fragment
260 	 */
261 	current_flag = (current_len <= copy_thresh) ?
262 	    USE_COPY : USE_DMA;
263 	/*
264 	 * If the mblk includes several contiguous small fragments,
265 	 * they may be copied into one buffer. This flag is used to
266 	 * indicate whether there are pending fragments that need to
267 	 * be copied to the current tx buffer.
268 	 *
269 	 * If this flag is B_TRUE, it indicates that a new tx control
270 	 * block is needed to process the next fragment using either
271 	 * copy or DMA binding.
272 	 *
273 	 * Otherwise, it indicates that the next fragment will be
274 	 * copied to the current tx buffer that is maintained by the
275 	 * current tx control block. No new tx control block is needed.
276 	 */
277 	copy_done = B_TRUE;
278 	while (current_mp) {
279 		next_mp = current_mp->b_cont;
280 		eop = (next_mp == NULL); /* Last fragment of the packet? */
281 		next_len = eop ? 0: MBLKL(next_mp);
282 
283 		/*
284 		 * When the current fragment is an empty fragment, if
285 		 * the next fragment will still be copied to the current
286 		 * tx buffer, we cannot skip this fragment here. Because
287 		 * the copy processing is pending for completion. We have
288 		 * to process this empty fragment in the tx_copy routine.
289 		 *
290 		 * If the copy processing is completed or a DMA binding
291 		 * processing is just completed, we can just skip this
292 		 * empty fragment.
293 		 */
294 		if ((current_len == 0) && (copy_done)) {
295 			current_mp = next_mp;
296 			current_len = next_len;
297 			current_flag = (current_len <= copy_thresh) ?
298 			    USE_COPY : USE_DMA;
299 			continue;
300 		}
301 
302 		if (copy_done) {
303 			/*
304 			 * Get a new tx control block from the free list
305 			 */
306 			tcb = igb_get_free_list(tx_ring);
307 
308 			if (tcb == NULL) {
309 				IGB_DEBUG_STAT(tx_ring->stat_fail_no_tcb);
310 				goto tx_failure;
311 			}
312 
313 			/*
314 			 * Push the tx control block to the pending list
315 			 * to avoid using lock too early
316 			 */
317 			LIST_PUSH_TAIL(&pending_list, &tcb->link);
318 		}
319 
320 		if (current_flag == USE_COPY) {
321 			/*
322 			 * Check whether to use bcopy or DMA binding to process
323 			 * the next fragment, and if using bcopy, whether we
324 			 * need to continue copying the next fragment into the
325 			 * current tx buffer.
326 			 */
327 			ASSERT((tcb->tx_buf.len + current_len) <=
328 			    tcb->tx_buf.size);
329 
330 			if (eop) {
331 				/*
332 				 * This is the last fragment of the packet, so
333 				 * the copy processing will be completed with
334 				 * this fragment.
335 				 */
336 				next_flag = USE_NONE;
337 				copy_done = B_TRUE;
338 			} else if ((tcb->tx_buf.len + current_len + next_len) >
339 			    tcb->tx_buf.size) {
340 				/*
341 				 * If the next fragment is too large to be
342 				 * copied to the current tx buffer, we need
343 				 * to complete the current copy processing.
344 				 */
345 				next_flag = (next_len > copy_thresh) ?
346 				    USE_DMA: USE_COPY;
347 				copy_done = B_TRUE;
348 			} else if (next_len > copy_thresh) {
349 				/*
350 				 * The next fragment needs to be processed with
351 				 * DMA binding. So the copy prcessing will be
352 				 * completed with the current fragment.
353 				 */
354 				next_flag = USE_DMA;
355 				copy_done = B_TRUE;
356 			} else {
357 				/*
358 				 * Continue to copy the next fragment to the
359 				 * current tx buffer.
360 				 */
361 				next_flag = USE_COPY;
362 				copy_done = B_FALSE;
363 			}
364 
365 			desc_num = igb_tx_copy(tx_ring, tcb, current_mp,
366 			    current_len, copy_done);
367 		} else {
368 			/*
369 			 * Check whether to use bcopy or DMA binding to process
370 			 * the next fragment.
371 			 */
372 			next_flag = (next_len > copy_thresh) ?
373 			    USE_DMA: USE_COPY;
374 			ASSERT(copy_done == B_TRUE);
375 
376 			desc_num = igb_tx_bind(tx_ring, tcb, current_mp,
377 			    current_len);
378 		}
379 
380 		if (desc_num > 0)
381 			desc_total += desc_num;
382 		else if (desc_num < 0)
383 			goto tx_failure;
384 
385 		current_mp = next_mp;
386 		current_len = next_len;
387 		current_flag = next_flag;
388 	}
389 
390 	/*
391 	 * Attach the mblk to the last tx control block
392 	 */
393 	ASSERT(tcb);
394 	ASSERT(tcb->mp == NULL);
395 	tcb->mp = mp;
396 
397 	/*
398 	 * Before fill the tx descriptor ring with the data, we need to
399 	 * ensure there are adequate free descriptors for transmit
400 	 * (including one context descriptor).
401 	 * Do not use up all the tx descriptors.
402 	 * Otherwise tx recycle will fail and cause false hang.
403 	 */
404 	if (tx_ring->tbd_free <= (desc_total + 1)) {
405 		tx_ring->tx_recycle(tx_ring);
406 	}
407 
408 	mutex_enter(&tx_ring->tx_lock);
409 
410 	/*
411 	 * If the number of free tx descriptors is not enough for transmit
412 	 * then return failure.
413 	 *
414 	 * Note: we must put this check under the mutex protection to
415 	 * ensure the correctness when multiple threads access it in
416 	 * parallel.
417 	 */
418 	if (tx_ring->tbd_free <= (desc_total + 1)) {
419 		IGB_DEBUG_STAT(tx_ring->stat_fail_no_tbd);
420 		mutex_exit(&tx_ring->tx_lock);
421 		goto tx_failure;
422 	}
423 
424 	desc_num = igb_tx_fill_ring(tx_ring, &pending_list, ctx, mbsize);
425 
426 	ASSERT((desc_num == desc_total) || (desc_num == (desc_total + 1)));
427 
428 	/* Update per-ring tx statistics */
429 	tx_ring->tx_pkts++;
430 	tx_ring->tx_bytes += mbsize;
431 
432 	mutex_exit(&tx_ring->tx_lock);
433 
434 	return (B_TRUE);
435 
436 tx_failure:
437 	/*
438 	 * If new mblk has been allocted for the last header
439 	 * fragment of a LSO packet, we should restore the
440 	 * modified mp.
441 	 */
442 	if (hdr_new_mp) {
443 		hdr_new_mp->b_cont = NULL;
444 		freeb(hdr_new_mp);
445 		hdr_current_mp->b_rptr -= hdr_frag_len;
446 		if (hdr_previous_mp)
447 			hdr_previous_mp->b_cont = hdr_current_mp;
448 		else
449 			mp = hdr_current_mp;
450 	}
451 
452 	/*
453 	 * Discard the mblk and free the used resources
454 	 */
455 	tcb = (tx_control_block_t *)LIST_GET_HEAD(&pending_list);
456 	while (tcb) {
457 		tcb->mp = NULL;
458 
459 		igb_free_tcb(tcb);
460 
461 		tcb = (tx_control_block_t *)
462 		    LIST_GET_NEXT(&pending_list, &tcb->link);
463 	}
464 
465 	/*
466 	 * Return the tx control blocks in the pending list to the free list.
467 	 */
468 	igb_put_free_list(tx_ring, &pending_list);
469 
470 	/* Transmit failed, do not drop the mblk, rechedule the transmit */
471 	tx_ring->reschedule = B_TRUE;
472 
473 	return (B_FALSE);
474 }
475 
476 /*
477  * igb_tx_copy
478  *
479  * Copy the mblk fragment to the pre-allocated tx buffer
480  */
481 static int
igb_tx_copy(igb_tx_ring_t * tx_ring,tx_control_block_t * tcb,mblk_t * mp,uint32_t len,boolean_t copy_done)482 igb_tx_copy(igb_tx_ring_t *tx_ring, tx_control_block_t *tcb, mblk_t *mp,
483     uint32_t len, boolean_t copy_done)
484 {
485 	dma_buffer_t *tx_buf;
486 	uint32_t desc_num;
487 	_NOTE(ARGUNUSED(tx_ring));
488 
489 	tx_buf = &tcb->tx_buf;
490 
491 	/*
492 	 * Copy the packet data of the mblk fragment into the
493 	 * pre-allocated tx buffer, which is maintained by the
494 	 * tx control block.
495 	 *
496 	 * Several mblk fragments can be copied into one tx buffer.
497 	 * The destination address of the current copied fragment in
498 	 * the tx buffer is next to the end of the previous copied
499 	 * fragment.
500 	 */
501 	if (len > 0) {
502 		bcopy(mp->b_rptr, tx_buf->address + tx_buf->len, len);
503 
504 		tx_buf->len += len;
505 		tcb->frag_num++;
506 	}
507 
508 	desc_num = 0;
509 
510 	/*
511 	 * If it is the last fragment copied to the current tx buffer,
512 	 * in other words, if there's no remaining fragment or the remaining
513 	 * fragment requires a new tx control block to process, we need to
514 	 * complete the current copy processing by syncing up the current
515 	 * DMA buffer and saving the descriptor data.
516 	 */
517 	if (copy_done) {
518 		/*
519 		 * Sync the DMA buffer of the packet data
520 		 */
521 		DMA_SYNC(tx_buf, DDI_DMA_SYNC_FORDEV);
522 
523 		tcb->tx_type = USE_COPY;
524 
525 		/*
526 		 * Save the address and length to the private data structure
527 		 * of the tx control block, which will be used to fill the
528 		 * tx descriptor ring after all the fragments are processed.
529 		 */
530 		igb_save_desc(tcb, tx_buf->dma_address, tx_buf->len);
531 		desc_num++;
532 	}
533 
534 	return (desc_num);
535 }
536 
537 /*
538  * igb_tx_bind
539  *
540  * Bind the mblk fragment with DMA
541  */
542 static int
igb_tx_bind(igb_tx_ring_t * tx_ring,tx_control_block_t * tcb,mblk_t * mp,uint32_t len)543 igb_tx_bind(igb_tx_ring_t *tx_ring, tx_control_block_t *tcb, mblk_t *mp,
544     uint32_t len)
545 {
546 	int status, i;
547 	ddi_dma_cookie_t dma_cookie;
548 	uint_t ncookies;
549 	int desc_num;
550 
551 	/*
552 	 * Use DMA binding to process the mblk fragment
553 	 */
554 	status = ddi_dma_addr_bind_handle(tcb->tx_dma_handle, NULL,
555 	    (caddr_t)mp->b_rptr, len,
556 	    DDI_DMA_WRITE | DDI_DMA_STREAMING, DDI_DMA_DONTWAIT,
557 	    0, &dma_cookie, &ncookies);
558 
559 	if (status != DDI_DMA_MAPPED) {
560 		IGB_DEBUG_STAT(tx_ring->stat_fail_dma_bind);
561 		return (-1);
562 	}
563 
564 	tcb->frag_num++;
565 	tcb->tx_type = USE_DMA;
566 	/*
567 	 * Each fragment can span several cookies. One cookie will have
568 	 * one tx descriptor to transmit.
569 	 */
570 	desc_num = 0;
571 	for (i = ncookies; i > 0; i--) {
572 		/*
573 		 * Save the address and length to the private data structure
574 		 * of the tx control block, which will be used to fill the
575 		 * tx descriptor ring after all the fragments are processed.
576 		 */
577 		igb_save_desc(tcb,
578 		    dma_cookie.dmac_laddress,
579 		    dma_cookie.dmac_size);
580 
581 		desc_num++;
582 
583 		if (i > 1)
584 			ddi_dma_nextcookie(tcb->tx_dma_handle, &dma_cookie);
585 	}
586 
587 	return (desc_num);
588 }
589 
590 /*
591  * igb_get_tx_context
592  *
593  * Get the tx context information from the mblk
594  */
595 static int
igb_get_tx_context(mblk_t * mp,tx_context_t * ctx)596 igb_get_tx_context(mblk_t *mp, tx_context_t *ctx)
597 {
598 	uint32_t start;
599 	uint32_t flags;
600 	uint32_t lso_flag;
601 	uint32_t mss;
602 	uint32_t len;
603 	uint32_t size;
604 	uint32_t offset;
605 	unsigned char *pos;
606 	ushort_t etype;
607 	uint32_t mac_hdr_len;
608 	uint32_t l4_proto;
609 	uint32_t l4_hdr_len;
610 
611 	ASSERT(mp != NULL);
612 
613 	mac_hcksum_get(mp, &start, NULL, NULL, NULL, &flags);
614 	bzero(ctx, sizeof (tx_context_t));
615 
616 	ctx->hcksum_flags = flags;
617 
618 	if (flags == 0)
619 		return (TX_CXT_SUCCESS);
620 
621 	mac_lso_get(mp, &mss, &lso_flag);
622 	ctx->mss = mss;
623 	ctx->lso_flag = (lso_flag == HW_LSO);
624 
625 	/*
626 	 * LSO relies on tx h/w checksum, so here the packet will be
627 	 * dropped if the h/w checksum flags are not set.
628 	 */
629 	if (ctx->lso_flag) {
630 		if (!((ctx->hcksum_flags & HCK_PARTIALCKSUM) &&
631 		    (ctx->hcksum_flags & HCK_IPV4_HDRCKSUM))) {
632 			IGB_DEBUGLOG_0(NULL, "igb_tx: h/w "
633 			    "checksum flags are not set for LSO");
634 			return (TX_CXT_E_LSO_CSUM);
635 		}
636 	}
637 
638 	etype = 0;
639 	mac_hdr_len = 0;
640 	l4_proto = 0;
641 
642 	/*
643 	 * Firstly get the position of the ether_type/ether_tpid.
644 	 * Here we don't assume the ether (VLAN) header is fully included
645 	 * in one mblk fragment, so we go thourgh the fragments to parse
646 	 * the ether type.
647 	 */
648 	size = len = MBLKL(mp);
649 	offset = offsetof(struct ether_header, ether_type);
650 	while (size <= offset) {
651 		mp = mp->b_cont;
652 		ASSERT(mp != NULL);
653 		len = MBLKL(mp);
654 		size += len;
655 	}
656 	pos = mp->b_rptr + offset + len - size;
657 
658 	etype = ntohs(*(ushort_t *)(uintptr_t)pos);
659 	if (etype == ETHERTYPE_VLAN) {
660 		/*
661 		 * Get the position of the ether_type in VLAN header
662 		 */
663 		offset = offsetof(struct ether_vlan_header, ether_type);
664 		while (size <= offset) {
665 			mp = mp->b_cont;
666 			ASSERT(mp != NULL);
667 			len = MBLKL(mp);
668 			size += len;
669 		}
670 		pos = mp->b_rptr + offset + len - size;
671 
672 		etype = ntohs(*(ushort_t *)(uintptr_t)pos);
673 		mac_hdr_len = sizeof (struct ether_vlan_header);
674 	} else {
675 		mac_hdr_len = sizeof (struct ether_header);
676 	}
677 
678 	/*
679 	 * Here we assume the IP(V6) header is fully included in one
680 	 * mblk fragment.
681 	 */
682 	switch (etype) {
683 	case ETHERTYPE_IP:
684 		offset = mac_hdr_len;
685 		while (size <= offset) {
686 			mp = mp->b_cont;
687 			ASSERT(mp != NULL);
688 			len = MBLKL(mp);
689 			size += len;
690 		}
691 		pos = mp->b_rptr + offset + len - size;
692 
693 		if (ctx->lso_flag) {
694 			*((uint16_t *)(uintptr_t)(pos + offsetof(ipha_t,
695 			    ipha_length))) = 0;
696 
697 			/*
698 			 * To utilize igb LSO, here need to fill
699 			 * the tcp checksum field of the packet with the
700 			 * following pseudo-header checksum:
701 			 * (ip_source_addr, ip_destination_addr, l4_proto)
702 			 * and also need to fill the ip header checksum
703 			 * with zero. Currently the tcp/ip stack has done
704 			 * these.
705 			 */
706 		}
707 
708 		l4_proto = *(uint8_t *)(pos + offsetof(ipha_t, ipha_protocol));
709 		break;
710 	case ETHERTYPE_IPV6:
711 		offset = offsetof(ip6_t, ip6_nxt) + mac_hdr_len;
712 		while (size <= offset) {
713 			mp = mp->b_cont;
714 			ASSERT(mp != NULL);
715 			len = MBLKL(mp);
716 			size += len;
717 		}
718 		pos = mp->b_rptr + offset + len - size;
719 
720 		l4_proto = *(uint8_t *)pos;
721 		break;
722 	default:
723 		/* Unrecoverable error */
724 		IGB_DEBUGLOG_0(NULL, "Ethernet type field error with "
725 		    "tx hcksum flag set");
726 		return (TX_CXT_E_ETHER_TYPE);
727 	}
728 
729 	if (ctx->lso_flag) {
730 		offset = mac_hdr_len + start;
731 		while (size <= offset) {
732 			mp = mp->b_cont;
733 			ASSERT(mp != NULL);
734 			len = MBLKL(mp);
735 			size += len;
736 		}
737 		pos = mp->b_rptr + offset + len - size;
738 
739 		l4_hdr_len = TCP_HDR_LENGTH((tcph_t *)pos);
740 	} else {
741 		/*
742 		 * l4 header length is only required for LSO
743 		 */
744 		l4_hdr_len = 0;
745 	}
746 
747 	ctx->mac_hdr_len = mac_hdr_len;
748 	ctx->ip_hdr_len = start;
749 	ctx->l4_proto = l4_proto;
750 	ctx->l4_hdr_len = l4_hdr_len;
751 
752 	return (TX_CXT_SUCCESS);
753 }
754 
755 /*
756  * igb_check_tx_context
757  *
758  * Check if a new context descriptor is needed
759  */
760 static boolean_t
igb_check_tx_context(igb_tx_ring_t * tx_ring,tx_context_t * ctx)761 igb_check_tx_context(igb_tx_ring_t *tx_ring, tx_context_t *ctx)
762 {
763 	tx_context_t *last;
764 
765 	if (ctx == NULL)
766 		return (B_FALSE);
767 
768 	/*
769 	 * Compare the context data retrieved from the mblk and the
770 	 * stored context data of the last context descriptor. The data
771 	 * need to be checked are:
772 	 *	hcksum_flags
773 	 *	l4_proto
774 	 *	mss (only check for LSO)
775 	 *	l4_hdr_len (only check for LSO)
776 	 *	ip_hdr_len
777 	 *	mac_hdr_len
778 	 * Either one of the above data is changed, a new context descriptor
779 	 * will be needed.
780 	 */
781 	last = &tx_ring->tx_context;
782 
783 	if (ctx->hcksum_flags != 0) {
784 		if ((ctx->hcksum_flags != last->hcksum_flags) ||
785 		    (ctx->l4_proto != last->l4_proto) ||
786 		    (ctx->lso_flag && ((ctx->mss != last->mss) ||
787 		    (ctx->l4_hdr_len != last->l4_hdr_len))) ||
788 		    (ctx->ip_hdr_len != last->ip_hdr_len) ||
789 		    (ctx->mac_hdr_len != last->mac_hdr_len)) {
790 			return (B_TRUE);
791 		}
792 	}
793 
794 	return (B_FALSE);
795 }
796 
797 /*
798  * igb_fill_tx_context
799  *
800  * Fill the context descriptor with hardware checksum informations
801  */
802 static void
igb_fill_tx_context(struct e1000_adv_tx_context_desc * ctx_tbd,tx_context_t * ctx,uint32_t ring_index)803 igb_fill_tx_context(struct e1000_adv_tx_context_desc *ctx_tbd,
804     tx_context_t *ctx, uint32_t ring_index)
805 {
806 	/*
807 	 * Fill the context descriptor with the checksum
808 	 * context information we've got
809 	 */
810 	ctx_tbd->vlan_macip_lens = ctx->ip_hdr_len;
811 	ctx_tbd->vlan_macip_lens |= ctx->mac_hdr_len <<
812 	    E1000_ADVTXD_MACLEN_SHIFT;
813 
814 	ctx_tbd->type_tucmd_mlhl =
815 	    E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
816 
817 	if (ctx->hcksum_flags & HCK_IPV4_HDRCKSUM)
818 		ctx_tbd->type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
819 
820 	if (ctx->hcksum_flags & HCK_PARTIALCKSUM) {
821 		switch (ctx->l4_proto) {
822 		case IPPROTO_TCP:
823 			ctx_tbd->type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
824 			break;
825 		case IPPROTO_UDP:
826 			/*
827 			 * We don't have to explicitly set:
828 			 *	ctx_tbd->type_tucmd_mlhl |=
829 			 *	    E1000_ADVTXD_TUCMD_L4T_UDP;
830 			 * Because E1000_ADVTXD_TUCMD_L4T_UDP == 0b
831 			 */
832 			break;
833 		default:
834 			/* Unrecoverable error */
835 			IGB_DEBUGLOG_0(NULL, "L4 type error with tx hcksum");
836 			break;
837 		}
838 	}
839 
840 	ctx_tbd->seqnum_seed = 0;
841 	ctx_tbd->mss_l4len_idx = ring_index << 4;
842 	if (ctx->lso_flag) {
843 		ctx_tbd->mss_l4len_idx |=
844 		    (ctx->l4_hdr_len << E1000_ADVTXD_L4LEN_SHIFT) |
845 		    (ctx->mss << E1000_ADVTXD_MSS_SHIFT);
846 	}
847 }
848 
849 /*
850  * igb_tx_fill_ring
851  *
852  * Fill the tx descriptor ring with the data
853  */
854 static int
igb_tx_fill_ring(igb_tx_ring_t * tx_ring,link_list_t * pending_list,tx_context_t * ctx,size_t mbsize)855 igb_tx_fill_ring(igb_tx_ring_t *tx_ring, link_list_t *pending_list,
856     tx_context_t *ctx, size_t mbsize)
857 {
858 	struct e1000_hw *hw = &tx_ring->igb->hw;
859 	boolean_t load_context;
860 	uint32_t index, tcb_index, desc_num;
861 	union e1000_adv_tx_desc *tbd, *first_tbd;
862 	tx_control_block_t *tcb, *first_tcb;
863 	uint32_t hcksum_flags;
864 	int i;
865 	igb_t *igb = tx_ring->igb;
866 
867 	ASSERT(mutex_owned(&tx_ring->tx_lock));
868 
869 	tbd = NULL;
870 	first_tbd = NULL;
871 	first_tcb = NULL;
872 	desc_num = 0;
873 	hcksum_flags = 0;
874 	load_context = B_FALSE;
875 
876 	/*
877 	 * Get the index of the first tx descriptor that will be filled,
878 	 * and the index of the first work list item that will be attached
879 	 * with the first used tx control block in the pending list.
880 	 * Note: the two indexes are the same.
881 	 */
882 	index = tx_ring->tbd_tail;
883 	tcb_index = tx_ring->tbd_tail;
884 
885 	if (ctx != NULL) {
886 		hcksum_flags = ctx->hcksum_flags;
887 
888 		/*
889 		 * Check if a new context descriptor is needed for this packet
890 		 */
891 		load_context = igb_check_tx_context(tx_ring, ctx);
892 		if (load_context) {
893 			tbd = &tx_ring->tbd_ring[index];
894 
895 			/*
896 			 * Fill the context descriptor with the
897 			 * hardware checksum offload informations.
898 			 */
899 			igb_fill_tx_context(
900 			    (struct e1000_adv_tx_context_desc *)tbd,
901 			    ctx, tx_ring->index);
902 
903 			index = NEXT_INDEX(index, 1, tx_ring->ring_size);
904 			desc_num++;
905 
906 			/*
907 			 * Store the checksum context data if
908 			 * a new context descriptor is added
909 			 */
910 			tx_ring->tx_context = *ctx;
911 		}
912 	}
913 
914 	first_tbd = &tx_ring->tbd_ring[index];
915 
916 	/*
917 	 * Fill tx data descriptors with the data saved in the pending list.
918 	 * The tx control blocks in the pending list are added to the work list
919 	 * at the same time.
920 	 *
921 	 * The work list is strictly 1:1 corresponding to the descriptor ring.
922 	 * One item of the work list corresponds to one tx descriptor. Because
923 	 * one tx control block can span multiple tx descriptors, the tx
924 	 * control block will be added to the first work list item that
925 	 * corresponds to the first tx descriptor generated from that tx
926 	 * control block.
927 	 */
928 	tcb = (tx_control_block_t *)LIST_POP_HEAD(pending_list);
929 	first_tcb = tcb;
930 	while (tcb != NULL) {
931 
932 		for (i = 0; i < tcb->desc_num; i++) {
933 			tbd = &tx_ring->tbd_ring[index];
934 
935 			tbd->read.buffer_addr = tcb->desc[i].address;
936 			tbd->read.cmd_type_len = tcb->desc[i].length;
937 
938 			tbd->read.cmd_type_len |= E1000_ADVTXD_DCMD_RS |
939 			    E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_DATA |
940 			    E1000_ADVTXD_DCMD_IFCS;
941 
942 			tbd->read.olinfo_status = 0;
943 
944 			index = NEXT_INDEX(index, 1, tx_ring->ring_size);
945 			desc_num++;
946 		}
947 
948 		/*
949 		 * Add the tx control block to the work list
950 		 */
951 		ASSERT(tx_ring->work_list[tcb_index] == NULL);
952 		tx_ring->work_list[tcb_index] = tcb;
953 
954 		tcb_index = index;
955 		tcb = (tx_control_block_t *)LIST_POP_HEAD(pending_list);
956 	}
957 
958 	if (load_context) {
959 		/*
960 		 * Count the checksum context descriptor for
961 		 * the first tx control block.
962 		 */
963 		first_tcb->desc_num++;
964 	}
965 	first_tcb->last_index = PREV_INDEX(index, 1, tx_ring->ring_size);
966 
967 	/*
968 	 * The Insert Ethernet CRC (IFCS) bit and the checksum fields are only
969 	 * valid in the first descriptor of the packet.
970 	 * 82576 also requires the payload length setting even without LSO
971 	 */
972 	ASSERT(first_tbd != NULL);
973 	first_tbd->read.cmd_type_len |= E1000_ADVTXD_DCMD_IFCS;
974 	if (ctx != NULL && ctx->lso_flag) {
975 		first_tbd->read.cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
976 		first_tbd->read.olinfo_status |=
977 		    (mbsize - ctx->mac_hdr_len - ctx->ip_hdr_len
978 		    - ctx->l4_hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT;
979 	} else {
980 		if (hw->mac.type >= e1000_82576) {
981 			first_tbd->read.olinfo_status |=
982 			    (mbsize << E1000_ADVTXD_PAYLEN_SHIFT);
983 		}
984 	}
985 
986 	/* Set hardware checksum bits */
987 	if (hcksum_flags != 0) {
988 		if (hcksum_flags & HCK_IPV4_HDRCKSUM)
989 			first_tbd->read.olinfo_status |=
990 			    E1000_TXD_POPTS_IXSM << 8;
991 		if (hcksum_flags & HCK_PARTIALCKSUM)
992 			first_tbd->read.olinfo_status |=
993 			    E1000_TXD_POPTS_TXSM << 8;
994 		first_tbd->read.olinfo_status |= tx_ring->index << 4;
995 	}
996 
997 	/*
998 	 * The last descriptor of packet needs End Of Packet (EOP),
999 	 * and Report Status (RS) bits set
1000 	 */
1001 	ASSERT(tbd != NULL);
1002 	tbd->read.cmd_type_len |=
1003 	    E1000_ADVTXD_DCMD_EOP | E1000_ADVTXD_DCMD_RS;
1004 
1005 	IGB_DEBUG_STAT(tx_ring->stat_pkt_cnt);
1006 
1007 	/*
1008 	 * Sync the DMA buffer of the tx descriptor ring
1009 	 */
1010 	DMA_SYNC(&tx_ring->tbd_area, DDI_DMA_SYNC_FORDEV);
1011 
1012 	/*
1013 	 * Update the number of the free tx descriptors.
1014 	 * The mutual exclusion between the transmission and the recycling
1015 	 * (for the tx descriptor ring and the work list) is implemented
1016 	 * with the atomic operation on the number of the free tx descriptors.
1017 	 *
1018 	 * Note: we should always decrement the counter tbd_free before
1019 	 * advancing the hardware TDT pointer to avoid the race condition -
1020 	 * before the counter tbd_free is decremented, the transmit of the
1021 	 * tx descriptors has done and the counter tbd_free is increased by
1022 	 * the tx recycling.
1023 	 */
1024 	i = igb_atomic_reserve(&tx_ring->tbd_free, desc_num);
1025 	ASSERT(i >= 0);
1026 
1027 	tx_ring->tbd_tail = index;
1028 
1029 	/*
1030 	 * Advance the hardware TDT pointer of the tx descriptor ring
1031 	 */
1032 	E1000_WRITE_REG(hw, E1000_TDT(tx_ring->index), index);
1033 
1034 	if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
1035 		ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
1036 		atomic_or_32(&igb->igb_state, IGB_ERROR);
1037 	}
1038 
1039 	return (desc_num);
1040 }
1041 
1042 /*
1043  * igb_save_desc
1044  *
1045  * Save the address/length pair to the private array
1046  * of the tx control block. The address/length pairs
1047  * will be filled into the tx descriptor ring later.
1048  */
1049 static void
igb_save_desc(tx_control_block_t * tcb,uint64_t address,size_t length)1050 igb_save_desc(tx_control_block_t *tcb, uint64_t address, size_t length)
1051 {
1052 	sw_desc_t *desc;
1053 
1054 	desc = &tcb->desc[tcb->desc_num];
1055 	desc->address = address;
1056 	desc->length = length;
1057 
1058 	tcb->desc_num++;
1059 }
1060 
1061 /*
1062  * igb_tx_recycle_legacy
1063  *
1064  * Recycle the tx descriptors and tx control blocks.
1065  *
1066  * The work list is traversed to check if the corresponding
1067  * tx descriptors have been transmitted. If so, the resources
1068  * bound to the tx control blocks will be freed, and those
1069  * tx control blocks will be returned to the free list.
1070  */
1071 uint32_t
igb_tx_recycle_legacy(igb_tx_ring_t * tx_ring)1072 igb_tx_recycle_legacy(igb_tx_ring_t *tx_ring)
1073 {
1074 	uint32_t index, last_index, next_index;
1075 	int desc_num;
1076 	boolean_t desc_done;
1077 	tx_control_block_t *tcb;
1078 	link_list_t pending_list;
1079 	igb_t *igb = tx_ring->igb;
1080 
1081 	/*
1082 	 * The mutex_tryenter() is used to avoid unnecessary
1083 	 * lock contention.
1084 	 */
1085 	if (mutex_tryenter(&tx_ring->recycle_lock) == 0)
1086 		return (0);
1087 
1088 	ASSERT(tx_ring->tbd_free <= tx_ring->ring_size);
1089 
1090 	if (tx_ring->tbd_free == tx_ring->ring_size) {
1091 		tx_ring->recycle_fail = 0;
1092 		tx_ring->stall_watchdog = 0;
1093 		mutex_exit(&tx_ring->recycle_lock);
1094 		return (0);
1095 	}
1096 
1097 	/*
1098 	 * Sync the DMA buffer of the tx descriptor ring
1099 	 */
1100 	DMA_SYNC(&tx_ring->tbd_area, DDI_DMA_SYNC_FORKERNEL);
1101 
1102 	if (igb_check_dma_handle(
1103 	    tx_ring->tbd_area.dma_handle) != DDI_FM_OK) {
1104 		mutex_exit(&tx_ring->recycle_lock);
1105 		ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
1106 		atomic_or_32(&igb->igb_state, IGB_ERROR);
1107 		return (0);
1108 	}
1109 
1110 	LINK_LIST_INIT(&pending_list);
1111 	desc_num = 0;
1112 	index = tx_ring->tbd_head;	/* Index of next tbd/tcb to recycle */
1113 
1114 	tcb = tx_ring->work_list[index];
1115 	ASSERT(tcb != NULL);
1116 
1117 	while (tcb != NULL) {
1118 
1119 		/*
1120 		 * Get the last tx descriptor of this packet.
1121 		 * If the last tx descriptor is done, then
1122 		 * we can recycle all descriptors of a packet
1123 		 * which usually includes several tx control blocks.
1124 		 * For some chips, LSO descriptors can not be recycled
1125 		 * unless the whole packet's transmission is done.
1126 		 * That's why packet level recycling is used here.
1127 		 */
1128 		last_index = tcb->last_index;
1129 		/*
1130 		 * MAX_TX_RING_SIZE is used to judge whether
1131 		 * the index is a valid value or not.
1132 		 */
1133 		if (last_index == MAX_TX_RING_SIZE)
1134 			break;
1135 
1136 		next_index = NEXT_INDEX(last_index, 1, tx_ring->ring_size);
1137 
1138 		/*
1139 		 * Check if the Descriptor Done bit is set
1140 		 */
1141 		desc_done = tx_ring->tbd_ring[last_index].wb.status &
1142 		    E1000_TXD_STAT_DD;
1143 		if (desc_done) {
1144 			while (tcb != NULL) {
1145 				/*
1146 				 * Strip off the tx control block from the work
1147 				 * list, and add it to the pending list.
1148 				 */
1149 				tx_ring->work_list[index] = NULL;
1150 				LIST_PUSH_TAIL(&pending_list, &tcb->link);
1151 
1152 				/*
1153 				 * Count the total number of the tx descriptors
1154 				 * recycled.
1155 				 */
1156 				desc_num += tcb->desc_num;
1157 
1158 				/*
1159 				 * Advance the index of the tx descriptor ring
1160 				 */
1161 				index = NEXT_INDEX(index, tcb->desc_num,
1162 				    tx_ring->ring_size);
1163 
1164 				tcb = tx_ring->work_list[index];
1165 				if (index == next_index)
1166 					break;
1167 			}
1168 		} else {
1169 			break;
1170 		}
1171 	}
1172 
1173 	/*
1174 	 * If no tx descriptors are recycled, no need to do more processing
1175 	 */
1176 	if (desc_num == 0) {
1177 		tx_ring->recycle_fail++;
1178 		mutex_exit(&tx_ring->recycle_lock);
1179 		return (0);
1180 	}
1181 
1182 	tx_ring->recycle_fail = 0;
1183 	tx_ring->stall_watchdog = 0;
1184 
1185 	/*
1186 	 * Update the head index of the tx descriptor ring
1187 	 */
1188 	tx_ring->tbd_head = index;
1189 
1190 	/*
1191 	 * Update the number of the free tx descriptors with atomic operations
1192 	 */
1193 	atomic_add_32(&tx_ring->tbd_free, desc_num);
1194 
1195 	mutex_exit(&tx_ring->recycle_lock);
1196 
1197 	/*
1198 	 * Free the resources used by the tx control blocks
1199 	 * in the pending list
1200 	 */
1201 	tcb = (tx_control_block_t *)LIST_GET_HEAD(&pending_list);
1202 	while (tcb != NULL) {
1203 		/*
1204 		 * Release the resources occupied by the tx control block
1205 		 */
1206 		igb_free_tcb(tcb);
1207 
1208 		tcb = (tx_control_block_t *)
1209 		    LIST_GET_NEXT(&pending_list, &tcb->link);
1210 	}
1211 
1212 	/*
1213 	 * Add the tx control blocks in the pending list to the free list.
1214 	 */
1215 	igb_put_free_list(tx_ring, &pending_list);
1216 
1217 	return (desc_num);
1218 }
1219 
1220 /*
1221  * igb_tx_recycle_head_wb
1222  *
1223  * Check the head write-back, and recycle all the transmitted
1224  * tx descriptors and tx control blocks.
1225  */
1226 uint32_t
igb_tx_recycle_head_wb(igb_tx_ring_t * tx_ring)1227 igb_tx_recycle_head_wb(igb_tx_ring_t *tx_ring)
1228 {
1229 	uint32_t index;
1230 	uint32_t head_wb;
1231 	int desc_num;
1232 	tx_control_block_t *tcb;
1233 	link_list_t pending_list;
1234 	igb_t *igb = tx_ring->igb;
1235 
1236 	/*
1237 	 * The mutex_tryenter() is used to avoid unnecessary
1238 	 * lock contention.
1239 	 */
1240 	if (mutex_tryenter(&tx_ring->recycle_lock) == 0)
1241 		return (0);
1242 
1243 	ASSERT(tx_ring->tbd_free <= tx_ring->ring_size);
1244 
1245 	if (tx_ring->tbd_free == tx_ring->ring_size) {
1246 		tx_ring->recycle_fail = 0;
1247 		tx_ring->stall_watchdog = 0;
1248 		mutex_exit(&tx_ring->recycle_lock);
1249 		return (0);
1250 	}
1251 
1252 	/*
1253 	 * Sync the DMA buffer of the tx descriptor ring
1254 	 *
1255 	 * Note: For head write-back mode, the tx descriptors will not
1256 	 * be written back, but the head write-back value is stored at
1257 	 * the last extra tbd at the end of the DMA area, we still need
1258 	 * to sync the head write-back value for kernel.
1259 	 *
1260 	 * DMA_SYNC(&tx_ring->tbd_area, DDI_DMA_SYNC_FORKERNEL);
1261 	 */
1262 	(void) ddi_dma_sync(tx_ring->tbd_area.dma_handle,
1263 	    sizeof (union e1000_adv_tx_desc) * tx_ring->ring_size,
1264 	    sizeof (uint32_t),
1265 	    DDI_DMA_SYNC_FORKERNEL);
1266 
1267 	if (igb_check_dma_handle(
1268 	    tx_ring->tbd_area.dma_handle) != DDI_FM_OK) {
1269 		mutex_exit(&tx_ring->recycle_lock);
1270 		ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
1271 		atomic_or_32(&igb->igb_state, IGB_ERROR);
1272 		return (0);
1273 	}
1274 
1275 	LINK_LIST_INIT(&pending_list);
1276 	desc_num = 0;
1277 	index = tx_ring->tbd_head;	/* Next index to clean */
1278 
1279 	/*
1280 	 * Get the value of head write-back
1281 	 */
1282 	head_wb = *tx_ring->tbd_head_wb;
1283 	while (index != head_wb) {
1284 		tcb = tx_ring->work_list[index];
1285 		ASSERT(tcb != NULL);
1286 
1287 		if (OFFSET(index, head_wb, tx_ring->ring_size) <
1288 		    tcb->desc_num) {
1289 			/*
1290 			 * The current tx control block is not
1291 			 * completely transmitted, stop recycling
1292 			 */
1293 			break;
1294 		}
1295 
1296 		/*
1297 		 * Strip off the tx control block from the work list,
1298 		 * and add it to the pending list.
1299 		 */
1300 		tx_ring->work_list[index] = NULL;
1301 		LIST_PUSH_TAIL(&pending_list, &tcb->link);
1302 
1303 		/*
1304 		 * Advance the index of the tx descriptor ring
1305 		 */
1306 		index = NEXT_INDEX(index, tcb->desc_num, tx_ring->ring_size);
1307 
1308 		/*
1309 		 * Count the total number of the tx descriptors recycled
1310 		 */
1311 		desc_num += tcb->desc_num;
1312 	}
1313 
1314 	/*
1315 	 * If no tx descriptors are recycled, no need to do more processing
1316 	 */
1317 	if (desc_num == 0) {
1318 		tx_ring->recycle_fail++;
1319 		mutex_exit(&tx_ring->recycle_lock);
1320 		return (0);
1321 	}
1322 
1323 	tx_ring->recycle_fail = 0;
1324 	tx_ring->stall_watchdog = 0;
1325 
1326 	/*
1327 	 * Update the head index of the tx descriptor ring
1328 	 */
1329 	tx_ring->tbd_head = index;
1330 
1331 	/*
1332 	 * Update the number of the free tx descriptors with atomic operations
1333 	 */
1334 	atomic_add_32(&tx_ring->tbd_free, desc_num);
1335 
1336 	mutex_exit(&tx_ring->recycle_lock);
1337 
1338 	/*
1339 	 * Free the resources used by the tx control blocks
1340 	 * in the pending list
1341 	 */
1342 	tcb = (tx_control_block_t *)LIST_GET_HEAD(&pending_list);
1343 	while (tcb) {
1344 		/*
1345 		 * Release the resources occupied by the tx control block
1346 		 */
1347 		igb_free_tcb(tcb);
1348 
1349 		tcb = (tx_control_block_t *)
1350 		    LIST_GET_NEXT(&pending_list, &tcb->link);
1351 	}
1352 
1353 	/*
1354 	 * Add the tx control blocks in the pending list to the free list.
1355 	 */
1356 	igb_put_free_list(tx_ring, &pending_list);
1357 
1358 	return (desc_num);
1359 }
1360 
1361 /*
1362  * igb_free_tcb - free up the tx control block
1363  *
1364  * Free the resources of the tx control block, including
1365  * unbind the previously bound DMA handle, and reset other
1366  * control fields.
1367  */
1368 void
igb_free_tcb(tx_control_block_t * tcb)1369 igb_free_tcb(tx_control_block_t *tcb)
1370 {
1371 	switch (tcb->tx_type) {
1372 	case USE_COPY:
1373 		/*
1374 		 * Reset the buffer length that is used for copy
1375 		 */
1376 		tcb->tx_buf.len = 0;
1377 		break;
1378 	case USE_DMA:
1379 		/*
1380 		 * Release the DMA resource that is used for
1381 		 * DMA binding.
1382 		 */
1383 		(void) ddi_dma_unbind_handle(tcb->tx_dma_handle);
1384 		break;
1385 	default:
1386 		break;
1387 	}
1388 
1389 	/*
1390 	 * Free the mblk
1391 	 */
1392 	if (tcb->mp != NULL) {
1393 		freemsg(tcb->mp);
1394 		tcb->mp = NULL;
1395 	}
1396 
1397 	tcb->tx_type = USE_NONE;
1398 	tcb->last_index = MAX_TX_RING_SIZE;
1399 	tcb->frag_num = 0;
1400 	tcb->desc_num = 0;
1401 }
1402 
1403 /*
1404  * igb_get_free_list - Get a free tx control block from the free list
1405  *
1406  * The atomic operation on the number of the available tx control block
1407  * in the free list is used to keep this routine mutual exclusive with
1408  * the routine igb_put_check_list.
1409  */
1410 static tx_control_block_t *
igb_get_free_list(igb_tx_ring_t * tx_ring)1411 igb_get_free_list(igb_tx_ring_t *tx_ring)
1412 {
1413 	tx_control_block_t *tcb;
1414 
1415 	/*
1416 	 * Check and update the number of the free tx control block
1417 	 * in the free list.
1418 	 */
1419 	if (igb_atomic_reserve(&tx_ring->tcb_free, 1) < 0)
1420 		return (NULL);
1421 
1422 	mutex_enter(&tx_ring->tcb_head_lock);
1423 
1424 	tcb = tx_ring->free_list[tx_ring->tcb_head];
1425 	ASSERT(tcb != NULL);
1426 	tx_ring->free_list[tx_ring->tcb_head] = NULL;
1427 	tx_ring->tcb_head = NEXT_INDEX(tx_ring->tcb_head, 1,
1428 	    tx_ring->free_list_size);
1429 
1430 	mutex_exit(&tx_ring->tcb_head_lock);
1431 
1432 	return (tcb);
1433 }
1434 
1435 /*
1436  * igb_put_free_list
1437  *
1438  * Put a list of used tx control blocks back to the free list
1439  *
1440  * A mutex is used here to ensure the serialization. The mutual exclusion
1441  * between igb_get_free_list and igb_put_free_list is implemented with
1442  * the atomic operation on the counter tcb_free.
1443  */
1444 void
igb_put_free_list(igb_tx_ring_t * tx_ring,link_list_t * pending_list)1445 igb_put_free_list(igb_tx_ring_t *tx_ring, link_list_t *pending_list)
1446 {
1447 	uint32_t index;
1448 	int tcb_num;
1449 	tx_control_block_t *tcb;
1450 
1451 	mutex_enter(&tx_ring->tcb_tail_lock);
1452 
1453 	index = tx_ring->tcb_tail;
1454 
1455 	tcb_num = 0;
1456 	tcb = (tx_control_block_t *)LIST_POP_HEAD(pending_list);
1457 	while (tcb != NULL) {
1458 		ASSERT(tx_ring->free_list[index] == NULL);
1459 		tx_ring->free_list[index] = tcb;
1460 
1461 		tcb_num++;
1462 
1463 		index = NEXT_INDEX(index, 1, tx_ring->free_list_size);
1464 
1465 		tcb = (tx_control_block_t *)LIST_POP_HEAD(pending_list);
1466 	}
1467 
1468 	tx_ring->tcb_tail = index;
1469 
1470 	/*
1471 	 * Update the number of the free tx control block
1472 	 * in the free list. This operation must be placed
1473 	 * under the protection of the lock.
1474 	 */
1475 	atomic_add_32(&tx_ring->tcb_free, tcb_num);
1476 
1477 	mutex_exit(&tx_ring->tcb_tail_lock);
1478 }
1479