xref: /illumos-gate/usr/src/uts/common/io/igb/igb_tx.c (revision 85f496fabdffd32673f6be280a3caa103f7d58a5)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright(c) 2007-2010 Intel Corporation. All rights reserved.
24  */
25 
26 /*
27  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
28  */
29 
30 #include "igb_sw.h"
31 
32 static boolean_t igb_tx(igb_tx_ring_t *, mblk_t *);
33 static int igb_tx_copy(igb_tx_ring_t *, tx_control_block_t *, mblk_t *,
34     uint32_t, boolean_t);
35 static int igb_tx_bind(igb_tx_ring_t *, tx_control_block_t *, mblk_t *,
36     uint32_t);
37 static int igb_tx_fill_ring(igb_tx_ring_t *, link_list_t *, tx_context_t *,
38     size_t);
39 static void igb_save_desc(tx_control_block_t *, uint64_t, size_t);
40 static tx_control_block_t *igb_get_free_list(igb_tx_ring_t *);
41 static int igb_get_tx_context(mblk_t *, tx_context_t *);
42 static boolean_t igb_check_tx_context(igb_tx_ring_t *, tx_context_t *);
43 static void igb_fill_tx_context(struct e1000_adv_tx_context_desc *,
44     tx_context_t *, uint32_t);
45 
46 #ifndef IGB_DEBUG
47 #pragma inline(igb_save_desc)
48 #pragma inline(igb_get_tx_context)
49 #pragma inline(igb_check_tx_context)
50 #pragma inline(igb_fill_tx_context)
51 #endif
52 
53 mblk_t *
igb_tx_ring_send(void * arg,mblk_t * mp)54 igb_tx_ring_send(void *arg, mblk_t *mp)
55 {
56 	igb_tx_ring_t *tx_ring = (igb_tx_ring_t *)arg;
57 	igb_t *igb;
58 
59 	ASSERT(tx_ring != NULL);
60 
61 	igb = tx_ring->igb;
62 
63 	if ((igb->igb_state & IGB_SUSPENDED) ||
64 	    (igb->igb_state & IGB_ERROR) ||
65 	    !(igb->igb_state & IGB_STARTED) ||
66 	    igb->link_state != LINK_STATE_UP) {
67 		freemsg(mp);
68 		return (NULL);
69 	}
70 
71 	return ((igb_tx(tx_ring, mp)) ? NULL : mp);
72 }
73 
74 /*
75  * igb_tx - Main transmit processing
76  *
77  * Called from igb_m_tx with an mblk ready to transmit. this
78  * routine sets up the transmit descriptors and sends data to
79  * the wire.
80  *
81  * One mblk can consist of several fragments, each fragment
82  * will be processed with different methods based on the size.
83  * For the fragments with size less than the bcopy threshold,
84  * they will be processed by using bcopy; otherwise, they will
85  * be processed by using DMA binding.
86  *
87  * To process the mblk, a tx control block is got from the
88  * free list. One tx control block contains one tx buffer, which
89  * is used to copy mblk fragments' data; and one tx DMA handle,
90  * which is used to bind a mblk fragment with DMA resource.
91  *
92  * Several small mblk fragments can be copied into one tx control
93  * block's buffer, and then the buffer will be transmitted with
94  * one tx descriptor.
95  *
96  * A large fragment only binds with one tx control block's DMA
97  * handle, and it can span several tx descriptors for transmitting.
98  *
99  * So to transmit a packet (mblk), several tx control blocks can
100  * be used. After the processing, those tx control blocks will
101  * be put to the work list.
102  */
103 static boolean_t
igb_tx(igb_tx_ring_t * tx_ring,mblk_t * mp)104 igb_tx(igb_tx_ring_t *tx_ring, mblk_t *mp)
105 {
106 	igb_t *igb = tx_ring->igb;
107 	tx_type_t current_flag, next_flag;
108 	uint32_t current_len, next_len;
109 	uint32_t desc_total;
110 	size_t mbsize;
111 	int desc_num;
112 	boolean_t copy_done, eop;
113 	mblk_t *current_mp, *next_mp, *nmp;
114 	tx_control_block_t *tcb;
115 	tx_context_t tx_context, *ctx;
116 	link_list_t pending_list;
117 	mblk_t *hdr_new_mp = NULL;
118 	mblk_t *hdr_previous_mp = NULL;
119 	mblk_t *hdr_current_mp = NULL;
120 	uint32_t hdr_frag_len;
121 	uint32_t hdr_len, len;
122 	uint32_t copy_thresh;
123 
124 	copy_thresh = igb->tx_copy_thresh;
125 
126 	/* Get the mblk size */
127 	mbsize = 0;
128 	for (nmp = mp; nmp != NULL; nmp = nmp->b_cont) {
129 		mbsize += MBLKL(nmp);
130 	}
131 
132 	if (igb->tx_hcksum_enable) {
133 		ctx = &tx_context;
134 		/*
135 		 * Retrieve offloading context information from the mblk
136 		 * that will be used to decide whether/how to fill the
137 		 * context descriptor.
138 		 */
139 		if (igb_get_tx_context(mp, ctx) != TX_CXT_SUCCESS) {
140 			freemsg(mp);
141 			return (B_TRUE);
142 		}
143 
144 		if ((ctx->lso_flag &&
145 		    (mbsize > (ctx->mac_hdr_len + IGB_LSO_MAXLEN))) ||
146 		    (!ctx->lso_flag &&
147 		    (mbsize > (igb->max_frame_size - ETHERFCSL)))) {
148 			freemsg(mp);
149 			igb_log(igb, IGB_LOG_INFO, "igb_tx: packet oversize");
150 			return (B_TRUE);
151 		}
152 	} else {
153 		ctx = NULL;
154 		if (mbsize > (igb->max_frame_size - ETHERFCSL)) {
155 			freemsg(mp);
156 			igb_log(igb, IGB_LOG_INFO, "igb_tx: packet oversize");
157 			return (B_TRUE);
158 		}
159 	}
160 
161 	/*
162 	 * Check and recycle tx descriptors.
163 	 * The recycle threshold here should be selected carefully
164 	 */
165 	if (tx_ring->tbd_free < igb->tx_recycle_thresh)
166 		tx_ring->tx_recycle(tx_ring);
167 
168 	/*
169 	 * After the recycling, if the tbd_free is less than the
170 	 * tx_overload_threshold, assert overload, return B_FALSE;
171 	 * and we need to re-schedule the tx again.
172 	 */
173 	if (tx_ring->tbd_free < igb->tx_overload_thresh) {
174 		tx_ring->reschedule = B_TRUE;
175 		IGB_DEBUG_STAT(tx_ring->stat_overload);
176 		return (B_FALSE);
177 	}
178 
179 	/*
180 	 * The software should guarantee LSO packet header(MAC+IP+TCP)
181 	 * to be within one descriptor - this is required by h/w.
182 	 * Here will reallocate and refill the header if
183 	 * the headers(MAC+IP+TCP) is physical memory non-contiguous.
184 	 */
185 	if (ctx && ctx->lso_flag) {
186 		hdr_len = ctx->mac_hdr_len + ctx->ip_hdr_len + ctx->l4_hdr_len;
187 		len = MBLKL(mp);
188 		hdr_current_mp = mp;
189 		while (len < hdr_len) {
190 			hdr_previous_mp = hdr_current_mp;
191 			hdr_current_mp = hdr_current_mp->b_cont;
192 			len += MBLKL(hdr_current_mp);
193 		}
194 		/*
195 		 * If the header and the payload are in different mblks,
196 		 * we simply force the header to be copied into pre-allocated
197 		 * page-aligned buffer.
198 		 */
199 		if (len == hdr_len)
200 			goto adjust_threshold;
201 
202 		hdr_frag_len = hdr_len - (len - MBLKL(hdr_current_mp));
203 		/*
204 		 * There are two cases we will reallocate
205 		 * a mblk for the last header fragment.
206 		 * 1. the header is in multiple mblks and
207 		 *    the last fragment shares the same mblk
208 		 *    with the payload
209 		 * 2. the header is in a single mblk shared
210 		 *    with the payload but the header crosses
211 		 *    a page.
212 		 */
213 		if ((hdr_current_mp != mp) ||
214 		    (P2NPHASE((uintptr_t)hdr_current_mp->b_rptr, igb->page_size)
215 		    < hdr_len)) {
216 			/*
217 			 * reallocate the mblk for the last header fragment,
218 			 * expect it to be copied into pre-allocated
219 			 * page-aligned buffer
220 			 */
221 			hdr_new_mp = allocb(hdr_frag_len, 0);
222 			if (!hdr_new_mp) {
223 				return (B_FALSE);
224 			}
225 
226 			/* link the new header fragment with the other parts */
227 			bcopy(hdr_current_mp->b_rptr,
228 			    hdr_new_mp->b_rptr, hdr_frag_len);
229 			hdr_new_mp->b_wptr = hdr_new_mp->b_rptr + hdr_frag_len;
230 			hdr_new_mp->b_cont = hdr_current_mp;
231 			if (hdr_previous_mp)
232 				hdr_previous_mp->b_cont = hdr_new_mp;
233 			else
234 				mp = hdr_new_mp;
235 			hdr_current_mp->b_rptr += hdr_frag_len;
236 		}
237 adjust_threshold:
238 		/*
239 		 * adjust the bcopy threshhold to guarantee
240 		 * the header to use bcopy way
241 		 */
242 		if (copy_thresh < hdr_len)
243 			copy_thresh = hdr_len;
244 	}
245 
246 	/*
247 	 * The pending_list is a linked list that is used to save
248 	 * the tx control blocks that have packet data processed
249 	 * but have not put the data to the tx descriptor ring.
250 	 * It is used to reduce the lock contention of the tx_lock.
251 	 */
252 	LINK_LIST_INIT(&pending_list);
253 	desc_num = 0;
254 	desc_total = 0;
255 
256 	current_mp = mp;
257 	current_len = MBLKL(current_mp);
258 	/*
259 	 * Decide which method to use for the first fragment
260 	 */
261 	current_flag = (current_len <= copy_thresh) ?
262 	    USE_COPY : USE_DMA;
263 	/*
264 	 * If the mblk includes several contiguous small fragments,
265 	 * they may be copied into one buffer. This flag is used to
266 	 * indicate whether there are pending fragments that need to
267 	 * be copied to the current tx buffer.
268 	 *
269 	 * If this flag is B_TRUE, it indicates that a new tx control
270 	 * block is needed to process the next fragment using either
271 	 * copy or DMA binding.
272 	 *
273 	 * Otherwise, it indicates that the next fragment will be
274 	 * copied to the current tx buffer that is maintained by the
275 	 * current tx control block. No new tx control block is needed.
276 	 */
277 	copy_done = B_TRUE;
278 	while (current_mp) {
279 		next_mp = current_mp->b_cont;
280 		eop = (next_mp == NULL); /* Last fragment of the packet? */
281 		next_len = eop ? 0: MBLKL(next_mp);
282 
283 		/*
284 		 * When the current fragment is an empty fragment, if
285 		 * the next fragment will still be copied to the current
286 		 * tx buffer, we cannot skip this fragment here. Because
287 		 * the copy processing is pending for completion. We have
288 		 * to process this empty fragment in the tx_copy routine.
289 		 *
290 		 * If the copy processing is completed or a DMA binding
291 		 * processing is just completed, we can just skip this
292 		 * empty fragment.
293 		 */
294 		if ((current_len == 0) && (copy_done)) {
295 			current_mp = next_mp;
296 			current_len = next_len;
297 			current_flag = (current_len <= copy_thresh) ?
298 			    USE_COPY : USE_DMA;
299 			continue;
300 		}
301 
302 		if (copy_done) {
303 			/*
304 			 * Get a new tx control block from the free list
305 			 */
306 			tcb = igb_get_free_list(tx_ring);
307 
308 			if (tcb == NULL) {
309 				IGB_DEBUG_STAT(tx_ring->stat_fail_no_tcb);
310 				goto tx_failure;
311 			}
312 
313 			/*
314 			 * Push the tx control block to the pending list
315 			 * to avoid using lock too early
316 			 */
317 			LIST_PUSH_TAIL(&pending_list, &tcb->link);
318 		}
319 
320 		if (current_flag == USE_COPY) {
321 			/*
322 			 * Check whether to use bcopy or DMA binding to process
323 			 * the next fragment, and if using bcopy, whether we
324 			 * need to continue copying the next fragment into the
325 			 * current tx buffer.
326 			 */
327 			ASSERT((tcb->tx_buf.len + current_len) <=
328 			    tcb->tx_buf.size);
329 
330 			if (eop) {
331 				/*
332 				 * This is the last fragment of the packet, so
333 				 * the copy processing will be completed with
334 				 * this fragment.
335 				 */
336 				next_flag = USE_NONE;
337 				copy_done = B_TRUE;
338 			} else if ((tcb->tx_buf.len + current_len + next_len) >
339 			    tcb->tx_buf.size) {
340 				/*
341 				 * If the next fragment is too large to be
342 				 * copied to the current tx buffer, we need
343 				 * to complete the current copy processing.
344 				 */
345 				next_flag = (next_len > copy_thresh) ?
346 				    USE_DMA: USE_COPY;
347 				copy_done = B_TRUE;
348 			} else if (next_len > copy_thresh) {
349 				/*
350 				 * The next fragment needs to be processed with
351 				 * DMA binding. So the copy prcessing will be
352 				 * completed with the current fragment.
353 				 */
354 				next_flag = USE_DMA;
355 				copy_done = B_TRUE;
356 			} else {
357 				/*
358 				 * Continue to copy the next fragment to the
359 				 * current tx buffer.
360 				 */
361 				next_flag = USE_COPY;
362 				copy_done = B_FALSE;
363 			}
364 
365 			desc_num = igb_tx_copy(tx_ring, tcb, current_mp,
366 			    current_len, copy_done);
367 		} else {
368 			/*
369 			 * Check whether to use bcopy or DMA binding to process
370 			 * the next fragment.
371 			 */
372 			next_flag = (next_len > copy_thresh) ?
373 			    USE_DMA: USE_COPY;
374 			ASSERT(copy_done == B_TRUE);
375 
376 			desc_num = igb_tx_bind(tx_ring, tcb, current_mp,
377 			    current_len);
378 		}
379 
380 		if (desc_num > 0)
381 			desc_total += desc_num;
382 		else if (desc_num < 0)
383 			goto tx_failure;
384 
385 		current_mp = next_mp;
386 		current_len = next_len;
387 		current_flag = next_flag;
388 	}
389 
390 	/*
391 	 * Attach the mblk to the last tx control block
392 	 */
393 	ASSERT(tcb);
394 	ASSERT(tcb->mp == NULL);
395 	tcb->mp = mp;
396 
397 	/*
398 	 * Before fill the tx descriptor ring with the data, we need to
399 	 * ensure there are adequate free descriptors for transmit
400 	 * (including one context descriptor).
401 	 * Do not use up all the tx descriptors.
402 	 * Otherwise tx recycle will fail and cause false hang.
403 	 */
404 	if (tx_ring->tbd_free <= (desc_total + 1)) {
405 		tx_ring->tx_recycle(tx_ring);
406 	}
407 
408 	mutex_enter(&tx_ring->tx_lock);
409 
410 	/*
411 	 * If the number of free tx descriptors is not enough for transmit
412 	 * then return failure.
413 	 *
414 	 * Note: we must put this check under the mutex protection to
415 	 * ensure the correctness when multiple threads access it in
416 	 * parallel.
417 	 */
418 	if (tx_ring->tbd_free <= (desc_total + 1)) {
419 		IGB_DEBUG_STAT(tx_ring->stat_fail_no_tbd);
420 		mutex_exit(&tx_ring->tx_lock);
421 		goto tx_failure;
422 	}
423 
424 	desc_num = igb_tx_fill_ring(tx_ring, &pending_list, ctx, mbsize);
425 
426 	ASSERT((desc_num == desc_total) || (desc_num == (desc_total + 1)));
427 
428 	/* Update per-ring tx statistics */
429 	tx_ring->tx_pkts++;
430 	tx_ring->tx_bytes += mbsize;
431 
432 	mutex_exit(&tx_ring->tx_lock);
433 
434 	return (B_TRUE);
435 
436 tx_failure:
437 	/*
438 	 * If new mblk has been allocted for the last header
439 	 * fragment of a LSO packet, we should restore the
440 	 * modified mp.
441 	 */
442 	if (hdr_new_mp) {
443 		hdr_new_mp->b_cont = NULL;
444 		freeb(hdr_new_mp);
445 		hdr_current_mp->b_rptr -= hdr_frag_len;
446 		if (hdr_previous_mp)
447 			hdr_previous_mp->b_cont = hdr_current_mp;
448 		else
449 			mp = hdr_current_mp;
450 	}
451 
452 	/*
453 	 * Discard the mblk and free the used resources
454 	 */
455 	tcb = (tx_control_block_t *)LIST_GET_HEAD(&pending_list);
456 	while (tcb) {
457 		tcb->mp = NULL;
458 
459 		igb_free_tcb(tcb);
460 
461 		tcb = (tx_control_block_t *)
462 		    LIST_GET_NEXT(&pending_list, &tcb->link);
463 	}
464 
465 	/*
466 	 * Return the tx control blocks in the pending list to the free list.
467 	 */
468 	igb_put_free_list(tx_ring, &pending_list);
469 
470 	/* Transmit failed, do not drop the mblk, rechedule the transmit */
471 	tx_ring->reschedule = B_TRUE;
472 
473 	return (B_FALSE);
474 }
475 
476 /*
477  * igb_tx_copy
478  *
479  * Copy the mblk fragment to the pre-allocated tx buffer
480  */
481 static int
igb_tx_copy(igb_tx_ring_t * tx_ring,tx_control_block_t * tcb,mblk_t * mp,uint32_t len,boolean_t copy_done)482 igb_tx_copy(igb_tx_ring_t *tx_ring, tx_control_block_t *tcb, mblk_t *mp,
483     uint32_t len, boolean_t copy_done)
484 {
485 	dma_buffer_t *tx_buf;
486 	uint32_t desc_num;
487 	_NOTE(ARGUNUSED(tx_ring));
488 
489 	tx_buf = &tcb->tx_buf;
490 
491 	/*
492 	 * Copy the packet data of the mblk fragment into the
493 	 * pre-allocated tx buffer, which is maintained by the
494 	 * tx control block.
495 	 *
496 	 * Several mblk fragments can be copied into one tx buffer.
497 	 * The destination address of the current copied fragment in
498 	 * the tx buffer is next to the end of the previous copied
499 	 * fragment.
500 	 */
501 	if (len > 0) {
502 		bcopy(mp->b_rptr, tx_buf->address + tx_buf->len, len);
503 
504 		tx_buf->len += len;
505 		tcb->frag_num++;
506 	}
507 
508 	desc_num = 0;
509 
510 	/*
511 	 * If it is the last fragment copied to the current tx buffer,
512 	 * in other words, if there's no remaining fragment or the remaining
513 	 * fragment requires a new tx control block to process, we need to
514 	 * complete the current copy processing by syncing up the current
515 	 * DMA buffer and saving the descriptor data.
516 	 */
517 	if (copy_done) {
518 		/*
519 		 * Sync the DMA buffer of the packet data
520 		 */
521 		DMA_SYNC(tx_buf, DDI_DMA_SYNC_FORDEV);
522 
523 		tcb->tx_type = USE_COPY;
524 
525 		/*
526 		 * Save the address and length to the private data structure
527 		 * of the tx control block, which will be used to fill the
528 		 * tx descriptor ring after all the fragments are processed.
529 		 */
530 		igb_save_desc(tcb, tx_buf->dma_address, tx_buf->len);
531 		desc_num++;
532 	}
533 
534 	return (desc_num);
535 }
536 
537 /*
538  * igb_tx_bind
539  *
540  * Bind the mblk fragment with DMA
541  */
542 static int
igb_tx_bind(igb_tx_ring_t * tx_ring,tx_control_block_t * tcb,mblk_t * mp,uint32_t len)543 igb_tx_bind(igb_tx_ring_t *tx_ring, tx_control_block_t *tcb, mblk_t *mp,
544     uint32_t len)
545 {
546 	int status, i;
547 	ddi_dma_cookie_t dma_cookie;
548 	uint_t ncookies;
549 	int desc_num;
550 
551 	/*
552 	 * Use DMA binding to process the mblk fragment
553 	 */
554 	status = ddi_dma_addr_bind_handle(tcb->tx_dma_handle, NULL,
555 	    (caddr_t)mp->b_rptr, len,
556 	    DDI_DMA_WRITE | DDI_DMA_STREAMING, DDI_DMA_DONTWAIT,
557 	    0, &dma_cookie, &ncookies);
558 
559 	if (status != DDI_DMA_MAPPED) {
560 		IGB_DEBUG_STAT(tx_ring->stat_fail_dma_bind);
561 		return (-1);
562 	}
563 
564 	tcb->frag_num++;
565 	tcb->tx_type = USE_DMA;
566 	/*
567 	 * Each fragment can span several cookies. One cookie will have
568 	 * one tx descriptor to transmit.
569 	 */
570 	desc_num = 0;
571 	for (i = ncookies; i > 0; i--) {
572 		/*
573 		 * Save the address and length to the private data structure
574 		 * of the tx control block, which will be used to fill the
575 		 * tx descriptor ring after all the fragments are processed.
576 		 */
577 		igb_save_desc(tcb,
578 		    dma_cookie.dmac_laddress,
579 		    dma_cookie.dmac_size);
580 
581 		desc_num++;
582 
583 		if (i > 1)
584 			ddi_dma_nextcookie(tcb->tx_dma_handle, &dma_cookie);
585 	}
586 
587 	return (desc_num);
588 }
589 
590 /*
591  * igb_get_tx_context
592  *
593  * Get the tx context information from the mblk
594  */
595 static int
igb_get_tx_context(mblk_t * mp,tx_context_t * ctx)596 igb_get_tx_context(mblk_t *mp, tx_context_t *ctx)
597 {
598 	uint32_t start;
599 	uint32_t flags;
600 	uint32_t lso_flag;
601 	uint32_t lso_cksum;
602 	uint32_t mss;
603 	uint32_t len;
604 	uint32_t size;
605 	uint32_t offset;
606 	unsigned char *pos;
607 	ushort_t etype;
608 	uint32_t mac_hdr_len;
609 	uint32_t l4_proto;
610 	uint32_t l4_hdr_len;
611 
612 	ASSERT(mp != NULL);
613 
614 	mac_hcksum_get(mp, &start, NULL, NULL, NULL, &flags);
615 	bzero(ctx, sizeof (tx_context_t));
616 
617 	ctx->hcksum_flags = flags;
618 
619 	if (flags == 0)
620 		return (TX_CXT_SUCCESS);
621 
622 	mac_lso_get(mp, &mss, &lso_flag);
623 	ctx->mss = mss;
624 	ctx->lso_flag = (lso_flag == HW_LSO);
625 
626 	etype = 0;
627 	mac_hdr_len = 0;
628 	l4_proto = 0;
629 
630 	/*
631 	 * Firstly get the position of the ether_type/ether_tpid.
632 	 * Here we don't assume the ether (VLAN) header is fully included
633 	 * in one mblk fragment, so we go thourgh the fragments to parse
634 	 * the ether type.
635 	 */
636 	size = len = MBLKL(mp);
637 	offset = offsetof(struct ether_header, ether_type);
638 	while (size <= offset) {
639 		mp = mp->b_cont;
640 		ASSERT(mp != NULL);
641 		len = MBLKL(mp);
642 		size += len;
643 	}
644 	pos = mp->b_rptr + offset + len - size;
645 
646 	etype = ntohs(*(ushort_t *)(uintptr_t)pos);
647 	if (etype == ETHERTYPE_VLAN) {
648 		/*
649 		 * Get the position of the ether_type in VLAN header
650 		 */
651 		offset = offsetof(struct ether_vlan_header, ether_type);
652 		while (size <= offset) {
653 			mp = mp->b_cont;
654 			ASSERT(mp != NULL);
655 			len = MBLKL(mp);
656 			size += len;
657 		}
658 		pos = mp->b_rptr + offset + len - size;
659 
660 		etype = ntohs(*(ushort_t *)(uintptr_t)pos);
661 		mac_hdr_len = sizeof (struct ether_vlan_header);
662 	} else {
663 		mac_hdr_len = sizeof (struct ether_header);
664 	}
665 
666 	/*
667 	 * Here we assume the IP(V6) header is fully included in one
668 	 * mblk fragment.
669 	 */
670 	lso_cksum = HCK_PARTIALCKSUM;
671 	ctx->l3_proto = etype;
672 	switch (etype) {
673 	case ETHERTYPE_IP:
674 		offset = mac_hdr_len;
675 		while (size <= offset) {
676 			mp = mp->b_cont;
677 			ASSERT(mp != NULL);
678 			len = MBLKL(mp);
679 			size += len;
680 		}
681 		pos = mp->b_rptr + offset + len - size;
682 
683 		if (ctx->lso_flag) {
684 			*((uint16_t *)(uintptr_t)(pos + offsetof(ipha_t,
685 			    ipha_length))) = 0;
686 
687 			/*
688 			 * To utilize igb LSO, here need to fill
689 			 * the tcp checksum field of the packet with the
690 			 * following pseudo-header checksum:
691 			 * (ip_source_addr, ip_destination_addr, l4_proto)
692 			 * and also need to fill the ip header checksum
693 			 * with zero. Currently the tcp/ip stack has done
694 			 * these.
695 			 */
696 			lso_cksum |= HCK_IPV4_HDRCKSUM;
697 		}
698 
699 		l4_proto = *(uint8_t *)(pos + offsetof(ipha_t, ipha_protocol));
700 		break;
701 	case ETHERTYPE_IPV6:
702 		/*
703 		 * We need to zero out the length in the header.
704 		 */
705 		if (ctx->lso_flag) {
706 			offset = offsetof(ip6_t, ip6_plen) + mac_hdr_len;
707 			while (size <= offset) {
708 				mp = mp->b_cont;
709 				ASSERT(mp != NULL);
710 				len = MBLKL(mp);
711 				size += len;
712 			}
713 			pos = mp->b_rptr + offset + len - size;
714 			*((uint16_t *)(uintptr_t)(pos)) = 0;
715 		}
716 
717 		offset = offsetof(ip6_t, ip6_nxt) + mac_hdr_len;
718 		while (size <= offset) {
719 			mp = mp->b_cont;
720 			ASSERT(mp != NULL);
721 			len = MBLKL(mp);
722 			size += len;
723 		}
724 		pos = mp->b_rptr + offset + len - size;
725 
726 		l4_proto = *(uint8_t *)pos;
727 		break;
728 	default:
729 		/* Unrecoverable error */
730 		igb_log(NULL, IGB_LOG_INFO, "Ethernet type field error with "
731 		    "tx hcksum flag set");
732 		return (TX_CXT_E_ETHER_TYPE);
733 	}
734 
735 	if (ctx->lso_flag) {
736 		/*
737 		 * LSO relies on tx h/w checksum, so here the packet will be
738 		 * dropped if the h/w checksum flags are not set.
739 		 */
740 		if ((ctx->hcksum_flags & lso_cksum) != lso_cksum) {
741 			igb_log(NULL, IGB_LOG_INFO, "igb_tx: h/w "
742 			    "checksum flags are not set for LSO, found "
743 			    "0x%x, needed bits 0x%x", ctx->hcksum_flags,
744 			    lso_cksum);
745 			return (TX_CXT_E_LSO_CSUM);
746 		}
747 
748 		offset = mac_hdr_len + start;
749 		while (size <= offset) {
750 			mp = mp->b_cont;
751 			ASSERT(mp != NULL);
752 			len = MBLKL(mp);
753 			size += len;
754 		}
755 		pos = mp->b_rptr + offset + len - size;
756 
757 		l4_hdr_len = TCP_HDR_LENGTH((tcph_t *)pos);
758 	} else {
759 		/*
760 		 * l4 header length is only required for LSO
761 		 */
762 		l4_hdr_len = 0;
763 	}
764 
765 	ctx->mac_hdr_len = mac_hdr_len;
766 	ctx->ip_hdr_len = start;
767 	ctx->l4_proto = l4_proto;
768 	ctx->l4_hdr_len = l4_hdr_len;
769 
770 	return (TX_CXT_SUCCESS);
771 }
772 
773 /*
774  * igb_check_tx_context
775  *
776  * Check if a new context descriptor is needed
777  */
778 static boolean_t
igb_check_tx_context(igb_tx_ring_t * tx_ring,tx_context_t * ctx)779 igb_check_tx_context(igb_tx_ring_t *tx_ring, tx_context_t *ctx)
780 {
781 	tx_context_t *last;
782 
783 	if (ctx == NULL)
784 		return (B_FALSE);
785 
786 	/*
787 	 * Compare the context data retrieved from the mblk and the
788 	 * stored context data of the last context descriptor. The data
789 	 * need to be checked are:
790 	 *	hcksum_flags
791 	 *	l4_proto
792 	 *	l3_proto
793 	 *	mss (only check for LSO)
794 	 *	l4_hdr_len (only check for LSO)
795 	 *	ip_hdr_len
796 	 *	mac_hdr_len
797 	 * Either one of the above data is changed, a new context descriptor
798 	 * will be needed.
799 	 */
800 	last = &tx_ring->tx_context;
801 
802 	if (ctx->hcksum_flags != 0) {
803 		if ((ctx->hcksum_flags != last->hcksum_flags) ||
804 		    (ctx->l4_proto != last->l4_proto) ||
805 		    (ctx->l3_proto != last->l3_proto) ||
806 		    (ctx->lso_flag && ((ctx->mss != last->mss) ||
807 		    (ctx->l4_hdr_len != last->l4_hdr_len))) ||
808 		    (ctx->ip_hdr_len != last->ip_hdr_len) ||
809 		    (ctx->mac_hdr_len != last->mac_hdr_len)) {
810 			return (B_TRUE);
811 		}
812 	}
813 
814 	return (B_FALSE);
815 }
816 
817 /*
818  * igb_fill_tx_context
819  *
820  * Fill the context descriptor with hardware checksum informations
821  */
822 static void
igb_fill_tx_context(struct e1000_adv_tx_context_desc * ctx_tbd,tx_context_t * ctx,uint32_t ring_index)823 igb_fill_tx_context(struct e1000_adv_tx_context_desc *ctx_tbd,
824     tx_context_t *ctx, uint32_t ring_index)
825 {
826 	/*
827 	 * Fill the context descriptor with the checksum
828 	 * context information we've got
829 	 */
830 	ctx_tbd->vlan_macip_lens = ctx->ip_hdr_len;
831 	ctx_tbd->vlan_macip_lens |= ctx->mac_hdr_len <<
832 	    E1000_ADVTXD_MACLEN_SHIFT;
833 
834 	ctx_tbd->type_tucmd_mlhl =
835 	    E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
836 
837 	/*
838 	 * When we have a TX context set up, we enforce that the ethertype is
839 	 * either IPv4 or IPv6 in igb_get_tx_context().
840 	 */
841 	if (ctx->lso_flag || ctx->hcksum_flags & HCK_IPV4_HDRCKSUM) {
842 		if (ctx->l3_proto == ETHERTYPE_IP) {
843 			ctx_tbd->type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
844 		} else {
845 			ctx_tbd->type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6;
846 		}
847 	}
848 
849 	if (ctx->lso_flag || ctx->hcksum_flags & HCK_PARTIALCKSUM) {
850 		switch (ctx->l4_proto) {
851 		case IPPROTO_TCP:
852 			ctx_tbd->type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
853 			break;
854 		case IPPROTO_UDP:
855 			/*
856 			 * We don't have to explicitly set:
857 			 *	ctx_tbd->type_tucmd_mlhl |=
858 			 *	    E1000_ADVTXD_TUCMD_L4T_UDP;
859 			 * Because E1000_ADVTXD_TUCMD_L4T_UDP == 0b
860 			 */
861 			break;
862 		default:
863 			/* Unrecoverable error */
864 			igb_log(NULL, IGB_LOG_INFO,
865 			    "L4 type error with tx hcksum");
866 			break;
867 		}
868 	}
869 
870 	ctx_tbd->seqnum_seed = 0;
871 	ctx_tbd->mss_l4len_idx = ring_index << 4;
872 	if (ctx->lso_flag) {
873 		ctx_tbd->mss_l4len_idx |=
874 		    (ctx->l4_hdr_len << E1000_ADVTXD_L4LEN_SHIFT) |
875 		    (ctx->mss << E1000_ADVTXD_MSS_SHIFT);
876 	}
877 }
878 
879 /*
880  * igb_tx_fill_ring
881  *
882  * Fill the tx descriptor ring with the data
883  */
884 static int
igb_tx_fill_ring(igb_tx_ring_t * tx_ring,link_list_t * pending_list,tx_context_t * ctx,size_t mbsize)885 igb_tx_fill_ring(igb_tx_ring_t *tx_ring, link_list_t *pending_list,
886     tx_context_t *ctx, size_t mbsize)
887 {
888 	struct e1000_hw *hw = &tx_ring->igb->hw;
889 	boolean_t load_context;
890 	uint32_t index, tcb_index, desc_num;
891 	union e1000_adv_tx_desc *tbd, *first_tbd;
892 	tx_control_block_t *tcb, *first_tcb;
893 	uint32_t hcksum_flags;
894 	int i;
895 	igb_t *igb = tx_ring->igb;
896 
897 	ASSERT(mutex_owned(&tx_ring->tx_lock));
898 
899 	tbd = NULL;
900 	first_tbd = NULL;
901 	first_tcb = NULL;
902 	desc_num = 0;
903 	hcksum_flags = 0;
904 	load_context = B_FALSE;
905 
906 	/*
907 	 * Get the index of the first tx descriptor that will be filled,
908 	 * and the index of the first work list item that will be attached
909 	 * with the first used tx control block in the pending list.
910 	 * Note: the two indexes are the same.
911 	 */
912 	index = tx_ring->tbd_tail;
913 	tcb_index = tx_ring->tbd_tail;
914 
915 	if (ctx != NULL) {
916 		hcksum_flags = ctx->hcksum_flags;
917 
918 		/*
919 		 * Check if a new context descriptor is needed for this packet
920 		 */
921 		load_context = igb_check_tx_context(tx_ring, ctx);
922 		if (load_context) {
923 			tbd = &tx_ring->tbd_ring[index];
924 
925 			/*
926 			 * Fill the context descriptor with the
927 			 * hardware checksum offload informations.
928 			 */
929 			igb_fill_tx_context(
930 			    (struct e1000_adv_tx_context_desc *)tbd,
931 			    ctx, tx_ring->index);
932 
933 			index = NEXT_INDEX(index, 1, tx_ring->ring_size);
934 			desc_num++;
935 
936 			/*
937 			 * Store the checksum context data if
938 			 * a new context descriptor is added
939 			 */
940 			tx_ring->tx_context = *ctx;
941 		}
942 	}
943 
944 	first_tbd = &tx_ring->tbd_ring[index];
945 
946 	/*
947 	 * Fill tx data descriptors with the data saved in the pending list.
948 	 * The tx control blocks in the pending list are added to the work list
949 	 * at the same time.
950 	 *
951 	 * The work list is strictly 1:1 corresponding to the descriptor ring.
952 	 * One item of the work list corresponds to one tx descriptor. Because
953 	 * one tx control block can span multiple tx descriptors, the tx
954 	 * control block will be added to the first work list item that
955 	 * corresponds to the first tx descriptor generated from that tx
956 	 * control block.
957 	 */
958 	tcb = (tx_control_block_t *)LIST_POP_HEAD(pending_list);
959 	first_tcb = tcb;
960 	while (tcb != NULL) {
961 
962 		for (i = 0; i < tcb->desc_num; i++) {
963 			tbd = &tx_ring->tbd_ring[index];
964 
965 			tbd->read.buffer_addr = tcb->desc[i].address;
966 			tbd->read.cmd_type_len = tcb->desc[i].length;
967 
968 			tbd->read.cmd_type_len |= E1000_ADVTXD_DCMD_RS |
969 			    E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_DATA |
970 			    E1000_ADVTXD_DCMD_IFCS;
971 
972 			tbd->read.olinfo_status = 0;
973 
974 			index = NEXT_INDEX(index, 1, tx_ring->ring_size);
975 			desc_num++;
976 		}
977 
978 		/*
979 		 * Add the tx control block to the work list
980 		 */
981 		ASSERT(tx_ring->work_list[tcb_index] == NULL);
982 		tx_ring->work_list[tcb_index] = tcb;
983 
984 		tcb_index = index;
985 		tcb = (tx_control_block_t *)LIST_POP_HEAD(pending_list);
986 	}
987 
988 	if (load_context) {
989 		/*
990 		 * Count the checksum context descriptor for
991 		 * the first tx control block.
992 		 */
993 		first_tcb->desc_num++;
994 	}
995 	first_tcb->last_index = PREV_INDEX(index, 1, tx_ring->ring_size);
996 
997 	/*
998 	 * The Insert Ethernet CRC (IFCS) bit and the checksum fields are only
999 	 * valid in the first descriptor of the packet.
1000 	 * 82576 also requires the payload length setting even without LSO
1001 	 */
1002 	ASSERT(first_tbd != NULL);
1003 	first_tbd->read.cmd_type_len |= E1000_ADVTXD_DCMD_IFCS;
1004 	if (ctx != NULL && ctx->lso_flag) {
1005 		first_tbd->read.cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
1006 		first_tbd->read.olinfo_status |=
1007 		    (mbsize - ctx->mac_hdr_len - ctx->ip_hdr_len
1008 		    - ctx->l4_hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT;
1009 	} else {
1010 		if (hw->mac.type >= e1000_82576) {
1011 			first_tbd->read.olinfo_status |=
1012 			    (mbsize << E1000_ADVTXD_PAYLEN_SHIFT);
1013 		}
1014 	}
1015 
1016 	/* Set hardware checksum bits */
1017 	if (hcksum_flags != 0) {
1018 		if (hcksum_flags & HCK_IPV4_HDRCKSUM)
1019 			first_tbd->read.olinfo_status |=
1020 			    E1000_TXD_POPTS_IXSM << 8;
1021 		if (hcksum_flags & HCK_PARTIALCKSUM)
1022 			first_tbd->read.olinfo_status |=
1023 			    E1000_TXD_POPTS_TXSM << 8;
1024 		first_tbd->read.olinfo_status |= tx_ring->index << 4;
1025 	}
1026 
1027 	/*
1028 	 * The last descriptor of packet needs End Of Packet (EOP),
1029 	 * and Report Status (RS) bits set
1030 	 */
1031 	ASSERT(tbd != NULL);
1032 	tbd->read.cmd_type_len |=
1033 	    E1000_ADVTXD_DCMD_EOP | E1000_ADVTXD_DCMD_RS;
1034 
1035 	IGB_DEBUG_STAT(tx_ring->stat_pkt_cnt);
1036 
1037 	/*
1038 	 * Sync the DMA buffer of the tx descriptor ring
1039 	 */
1040 	DMA_SYNC(&tx_ring->tbd_area, DDI_DMA_SYNC_FORDEV);
1041 
1042 	/*
1043 	 * Update the number of the free tx descriptors.
1044 	 * The mutual exclusion between the transmission and the recycling
1045 	 * (for the tx descriptor ring and the work list) is implemented
1046 	 * with the atomic operation on the number of the free tx descriptors.
1047 	 *
1048 	 * Note: we should always decrement the counter tbd_free before
1049 	 * advancing the hardware TDT pointer to avoid the race condition -
1050 	 * before the counter tbd_free is decremented, the transmit of the
1051 	 * tx descriptors has done and the counter tbd_free is increased by
1052 	 * the tx recycling.
1053 	 */
1054 	i = igb_atomic_reserve(&tx_ring->tbd_free, desc_num);
1055 	ASSERT(i >= 0);
1056 
1057 	tx_ring->tbd_tail = index;
1058 
1059 	/*
1060 	 * Advance the hardware TDT pointer of the tx descriptor ring
1061 	 */
1062 	E1000_WRITE_REG(hw, E1000_TDT(tx_ring->index), index);
1063 
1064 	if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
1065 		ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
1066 		atomic_or_32(&igb->igb_state, IGB_ERROR);
1067 	}
1068 
1069 	return (desc_num);
1070 }
1071 
1072 /*
1073  * igb_save_desc
1074  *
1075  * Save the address/length pair to the private array
1076  * of the tx control block. The address/length pairs
1077  * will be filled into the tx descriptor ring later.
1078  */
1079 static void
igb_save_desc(tx_control_block_t * tcb,uint64_t address,size_t length)1080 igb_save_desc(tx_control_block_t *tcb, uint64_t address, size_t length)
1081 {
1082 	sw_desc_t *desc;
1083 
1084 	desc = &tcb->desc[tcb->desc_num];
1085 	desc->address = address;
1086 	desc->length = length;
1087 
1088 	tcb->desc_num++;
1089 }
1090 
1091 /*
1092  * igb_tx_recycle_legacy
1093  *
1094  * Recycle the tx descriptors and tx control blocks.
1095  *
1096  * The work list is traversed to check if the corresponding
1097  * tx descriptors have been transmitted. If so, the resources
1098  * bound to the tx control blocks will be freed, and those
1099  * tx control blocks will be returned to the free list.
1100  */
1101 uint32_t
igb_tx_recycle_legacy(igb_tx_ring_t * tx_ring)1102 igb_tx_recycle_legacy(igb_tx_ring_t *tx_ring)
1103 {
1104 	uint32_t index, last_index, next_index;
1105 	int desc_num;
1106 	boolean_t desc_done;
1107 	tx_control_block_t *tcb;
1108 	link_list_t pending_list;
1109 	igb_t *igb = tx_ring->igb;
1110 
1111 	/*
1112 	 * The mutex_tryenter() is used to avoid unnecessary
1113 	 * lock contention.
1114 	 */
1115 	if (mutex_tryenter(&tx_ring->recycle_lock) == 0)
1116 		return (0);
1117 
1118 	ASSERT(tx_ring->tbd_free <= tx_ring->ring_size);
1119 
1120 	if (tx_ring->tbd_free == tx_ring->ring_size) {
1121 		tx_ring->recycle_fail = 0;
1122 		tx_ring->stall_watchdog = 0;
1123 		mutex_exit(&tx_ring->recycle_lock);
1124 		return (0);
1125 	}
1126 
1127 	/*
1128 	 * Sync the DMA buffer of the tx descriptor ring
1129 	 */
1130 	DMA_SYNC(&tx_ring->tbd_area, DDI_DMA_SYNC_FORKERNEL);
1131 
1132 	if (igb_check_dma_handle(
1133 	    tx_ring->tbd_area.dma_handle) != DDI_FM_OK) {
1134 		mutex_exit(&tx_ring->recycle_lock);
1135 		ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
1136 		atomic_or_32(&igb->igb_state, IGB_ERROR);
1137 		return (0);
1138 	}
1139 
1140 	LINK_LIST_INIT(&pending_list);
1141 	desc_num = 0;
1142 	index = tx_ring->tbd_head;	/* Index of next tbd/tcb to recycle */
1143 
1144 	tcb = tx_ring->work_list[index];
1145 	ASSERT(tcb != NULL);
1146 
1147 	while (tcb != NULL) {
1148 
1149 		/*
1150 		 * Get the last tx descriptor of this packet.
1151 		 * If the last tx descriptor is done, then
1152 		 * we can recycle all descriptors of a packet
1153 		 * which usually includes several tx control blocks.
1154 		 * For some chips, LSO descriptors can not be recycled
1155 		 * unless the whole packet's transmission is done.
1156 		 * That's why packet level recycling is used here.
1157 		 */
1158 		last_index = tcb->last_index;
1159 		/*
1160 		 * MAX_TX_RING_SIZE is used to judge whether
1161 		 * the index is a valid value or not.
1162 		 */
1163 		if (last_index == MAX_TX_RING_SIZE)
1164 			break;
1165 
1166 		next_index = NEXT_INDEX(last_index, 1, tx_ring->ring_size);
1167 
1168 		/*
1169 		 * Check if the Descriptor Done bit is set
1170 		 */
1171 		desc_done = tx_ring->tbd_ring[last_index].wb.status &
1172 		    E1000_TXD_STAT_DD;
1173 		if (desc_done) {
1174 			while (tcb != NULL) {
1175 				/*
1176 				 * Strip off the tx control block from the work
1177 				 * list, and add it to the pending list.
1178 				 */
1179 				tx_ring->work_list[index] = NULL;
1180 				LIST_PUSH_TAIL(&pending_list, &tcb->link);
1181 
1182 				/*
1183 				 * Count the total number of the tx descriptors
1184 				 * recycled.
1185 				 */
1186 				desc_num += tcb->desc_num;
1187 
1188 				/*
1189 				 * Advance the index of the tx descriptor ring
1190 				 */
1191 				index = NEXT_INDEX(index, tcb->desc_num,
1192 				    tx_ring->ring_size);
1193 
1194 				tcb = tx_ring->work_list[index];
1195 				if (index == next_index)
1196 					break;
1197 			}
1198 		} else {
1199 			break;
1200 		}
1201 	}
1202 
1203 	/*
1204 	 * If no tx descriptors are recycled, no need to do more processing
1205 	 */
1206 	if (desc_num == 0) {
1207 		tx_ring->recycle_fail++;
1208 		mutex_exit(&tx_ring->recycle_lock);
1209 		return (0);
1210 	}
1211 
1212 	tx_ring->recycle_fail = 0;
1213 	tx_ring->stall_watchdog = 0;
1214 
1215 	/*
1216 	 * Update the head index of the tx descriptor ring
1217 	 */
1218 	tx_ring->tbd_head = index;
1219 
1220 	/*
1221 	 * Update the number of the free tx descriptors with atomic operations
1222 	 */
1223 	atomic_add_32(&tx_ring->tbd_free, desc_num);
1224 
1225 	mutex_exit(&tx_ring->recycle_lock);
1226 
1227 	/*
1228 	 * Free the resources used by the tx control blocks
1229 	 * in the pending list
1230 	 */
1231 	tcb = (tx_control_block_t *)LIST_GET_HEAD(&pending_list);
1232 	while (tcb != NULL) {
1233 		/*
1234 		 * Release the resources occupied by the tx control block
1235 		 */
1236 		igb_free_tcb(tcb);
1237 
1238 		tcb = (tx_control_block_t *)
1239 		    LIST_GET_NEXT(&pending_list, &tcb->link);
1240 	}
1241 
1242 	/*
1243 	 * Add the tx control blocks in the pending list to the free list.
1244 	 */
1245 	igb_put_free_list(tx_ring, &pending_list);
1246 
1247 	return (desc_num);
1248 }
1249 
1250 /*
1251  * igb_tx_recycle_head_wb
1252  *
1253  * Check the head write-back, and recycle all the transmitted
1254  * tx descriptors and tx control blocks.
1255  */
1256 uint32_t
igb_tx_recycle_head_wb(igb_tx_ring_t * tx_ring)1257 igb_tx_recycle_head_wb(igb_tx_ring_t *tx_ring)
1258 {
1259 	uint32_t index;
1260 	uint32_t head_wb;
1261 	int desc_num;
1262 	tx_control_block_t *tcb;
1263 	link_list_t pending_list;
1264 	igb_t *igb = tx_ring->igb;
1265 
1266 	/*
1267 	 * The mutex_tryenter() is used to avoid unnecessary
1268 	 * lock contention.
1269 	 */
1270 	if (mutex_tryenter(&tx_ring->recycle_lock) == 0)
1271 		return (0);
1272 
1273 	ASSERT(tx_ring->tbd_free <= tx_ring->ring_size);
1274 
1275 	if (tx_ring->tbd_free == tx_ring->ring_size) {
1276 		tx_ring->recycle_fail = 0;
1277 		tx_ring->stall_watchdog = 0;
1278 		mutex_exit(&tx_ring->recycle_lock);
1279 		return (0);
1280 	}
1281 
1282 	/*
1283 	 * Sync the DMA buffer of the tx descriptor ring
1284 	 *
1285 	 * Note: For head write-back mode, the tx descriptors will not
1286 	 * be written back, but the head write-back value is stored at
1287 	 * the last extra tbd at the end of the DMA area, we still need
1288 	 * to sync the head write-back value for kernel.
1289 	 *
1290 	 * DMA_SYNC(&tx_ring->tbd_area, DDI_DMA_SYNC_FORKERNEL);
1291 	 */
1292 	(void) ddi_dma_sync(tx_ring->tbd_area.dma_handle,
1293 	    sizeof (union e1000_adv_tx_desc) * tx_ring->ring_size,
1294 	    sizeof (uint32_t),
1295 	    DDI_DMA_SYNC_FORKERNEL);
1296 
1297 	if (igb_check_dma_handle(
1298 	    tx_ring->tbd_area.dma_handle) != DDI_FM_OK) {
1299 		mutex_exit(&tx_ring->recycle_lock);
1300 		ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
1301 		atomic_or_32(&igb->igb_state, IGB_ERROR);
1302 		return (0);
1303 	}
1304 
1305 	LINK_LIST_INIT(&pending_list);
1306 	desc_num = 0;
1307 	index = tx_ring->tbd_head;	/* Next index to clean */
1308 
1309 	/*
1310 	 * Get the value of head write-back
1311 	 */
1312 	head_wb = *tx_ring->tbd_head_wb;
1313 	while (index != head_wb) {
1314 		tcb = tx_ring->work_list[index];
1315 		ASSERT(tcb != NULL);
1316 
1317 		if (OFFSET(index, head_wb, tx_ring->ring_size) <
1318 		    tcb->desc_num) {
1319 			/*
1320 			 * The current tx control block is not
1321 			 * completely transmitted, stop recycling
1322 			 */
1323 			break;
1324 		}
1325 
1326 		/*
1327 		 * Strip off the tx control block from the work list,
1328 		 * and add it to the pending list.
1329 		 */
1330 		tx_ring->work_list[index] = NULL;
1331 		LIST_PUSH_TAIL(&pending_list, &tcb->link);
1332 
1333 		/*
1334 		 * Advance the index of the tx descriptor ring
1335 		 */
1336 		index = NEXT_INDEX(index, tcb->desc_num, tx_ring->ring_size);
1337 
1338 		/*
1339 		 * Count the total number of the tx descriptors recycled
1340 		 */
1341 		desc_num += tcb->desc_num;
1342 	}
1343 
1344 	/*
1345 	 * If no tx descriptors are recycled, no need to do more processing
1346 	 */
1347 	if (desc_num == 0) {
1348 		tx_ring->recycle_fail++;
1349 		mutex_exit(&tx_ring->recycle_lock);
1350 		return (0);
1351 	}
1352 
1353 	tx_ring->recycle_fail = 0;
1354 	tx_ring->stall_watchdog = 0;
1355 
1356 	/*
1357 	 * Update the head index of the tx descriptor ring
1358 	 */
1359 	tx_ring->tbd_head = index;
1360 
1361 	/*
1362 	 * Update the number of the free tx descriptors with atomic operations
1363 	 */
1364 	atomic_add_32(&tx_ring->tbd_free, desc_num);
1365 
1366 	mutex_exit(&tx_ring->recycle_lock);
1367 
1368 	/*
1369 	 * Free the resources used by the tx control blocks
1370 	 * in the pending list
1371 	 */
1372 	tcb = (tx_control_block_t *)LIST_GET_HEAD(&pending_list);
1373 	while (tcb) {
1374 		/*
1375 		 * Release the resources occupied by the tx control block
1376 		 */
1377 		igb_free_tcb(tcb);
1378 
1379 		tcb = (tx_control_block_t *)
1380 		    LIST_GET_NEXT(&pending_list, &tcb->link);
1381 	}
1382 
1383 	/*
1384 	 * Add the tx control blocks in the pending list to the free list.
1385 	 */
1386 	igb_put_free_list(tx_ring, &pending_list);
1387 
1388 	return (desc_num);
1389 }
1390 
1391 /*
1392  * igb_free_tcb - free up the tx control block
1393  *
1394  * Free the resources of the tx control block, including
1395  * unbind the previously bound DMA handle, and reset other
1396  * control fields.
1397  */
1398 void
igb_free_tcb(tx_control_block_t * tcb)1399 igb_free_tcb(tx_control_block_t *tcb)
1400 {
1401 	switch (tcb->tx_type) {
1402 	case USE_COPY:
1403 		/*
1404 		 * Reset the buffer length that is used for copy
1405 		 */
1406 		tcb->tx_buf.len = 0;
1407 		break;
1408 	case USE_DMA:
1409 		/*
1410 		 * Release the DMA resource that is used for
1411 		 * DMA binding.
1412 		 */
1413 		(void) ddi_dma_unbind_handle(tcb->tx_dma_handle);
1414 		break;
1415 	default:
1416 		break;
1417 	}
1418 
1419 	/*
1420 	 * Free the mblk
1421 	 */
1422 	if (tcb->mp != NULL) {
1423 		freemsg(tcb->mp);
1424 		tcb->mp = NULL;
1425 	}
1426 
1427 	tcb->tx_type = USE_NONE;
1428 	tcb->last_index = MAX_TX_RING_SIZE;
1429 	tcb->frag_num = 0;
1430 	tcb->desc_num = 0;
1431 }
1432 
1433 /*
1434  * igb_get_free_list - Get a free tx control block from the free list
1435  *
1436  * The atomic operation on the number of the available tx control block
1437  * in the free list is used to keep this routine mutual exclusive with
1438  * the routine igb_put_check_list.
1439  */
1440 static tx_control_block_t *
igb_get_free_list(igb_tx_ring_t * tx_ring)1441 igb_get_free_list(igb_tx_ring_t *tx_ring)
1442 {
1443 	tx_control_block_t *tcb;
1444 
1445 	/*
1446 	 * Check and update the number of the free tx control block
1447 	 * in the free list.
1448 	 */
1449 	if (igb_atomic_reserve(&tx_ring->tcb_free, 1) < 0)
1450 		return (NULL);
1451 
1452 	mutex_enter(&tx_ring->tcb_head_lock);
1453 
1454 	tcb = tx_ring->free_list[tx_ring->tcb_head];
1455 	ASSERT(tcb != NULL);
1456 	tx_ring->free_list[tx_ring->tcb_head] = NULL;
1457 	tx_ring->tcb_head = NEXT_INDEX(tx_ring->tcb_head, 1,
1458 	    tx_ring->free_list_size);
1459 
1460 	mutex_exit(&tx_ring->tcb_head_lock);
1461 
1462 	return (tcb);
1463 }
1464 
1465 /*
1466  * igb_put_free_list
1467  *
1468  * Put a list of used tx control blocks back to the free list
1469  *
1470  * A mutex is used here to ensure the serialization. The mutual exclusion
1471  * between igb_get_free_list and igb_put_free_list is implemented with
1472  * the atomic operation on the counter tcb_free.
1473  */
1474 void
igb_put_free_list(igb_tx_ring_t * tx_ring,link_list_t * pending_list)1475 igb_put_free_list(igb_tx_ring_t *tx_ring, link_list_t *pending_list)
1476 {
1477 	uint32_t index;
1478 	int tcb_num;
1479 	tx_control_block_t *tcb;
1480 
1481 	mutex_enter(&tx_ring->tcb_tail_lock);
1482 
1483 	index = tx_ring->tcb_tail;
1484 
1485 	tcb_num = 0;
1486 	tcb = (tx_control_block_t *)LIST_POP_HEAD(pending_list);
1487 	while (tcb != NULL) {
1488 		ASSERT(tx_ring->free_list[index] == NULL);
1489 		tx_ring->free_list[index] = tcb;
1490 
1491 		tcb_num++;
1492 
1493 		index = NEXT_INDEX(index, 1, tx_ring->free_list_size);
1494 
1495 		tcb = (tx_control_block_t *)LIST_POP_HEAD(pending_list);
1496 	}
1497 
1498 	tx_ring->tcb_tail = index;
1499 
1500 	/*
1501 	 * Update the number of the free tx control block
1502 	 * in the free list. This operation must be placed
1503 	 * under the protection of the lock.
1504 	 */
1505 	atomic_add_32(&tx_ring->tcb_free, tcb_num);
1506 
1507 	mutex_exit(&tx_ring->tcb_tail_lock);
1508 }
1509