xref: /titanic_51/usr/src/uts/common/io/igb/igb_tx.c (revision bc37da3aa8455efcf567c456746e3fb9d7f0a189)
1 /*
2  * CDDL HEADER START
3  *
4  * Copyright(c) 2007-2008 Intel Corporation. All rights reserved.
5  * The contents of this file are subject to the terms of the
6  * Common Development and Distribution License (the "License").
7  * You may not use this file except in compliance with the License.
8  *
9  * You can obtain a copy of the license at:
10  *	http://www.opensolaris.org/os/licensing.
11  * See the License for the specific language governing permissions
12  * and limitations under the License.
13  *
14  * When using or redistributing this file, you may do so under the
15  * License only. No other modification of this header is permitted.
16  *
17  * If applicable, add the following below this CDDL HEADER, with the
18  * fields enclosed by brackets "[]" replaced with your own identifying
19  * information: Portions Copyright [yyyy] [name of copyright owner]
20  *
21  * CDDL HEADER END
22  */
23 
24 /*
25  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
26  * Use is subject to license terms of the CDDL.
27  */
28 
29 #pragma ident	"%Z%%M%	%I%	%E% SMI"
30 
31 #include "igb_sw.h"
32 
33 static boolean_t igb_tx(igb_tx_ring_t *, mblk_t *);
34 static int igb_tx_copy(igb_tx_ring_t *, tx_control_block_t *, mblk_t *,
35     uint32_t, boolean_t, boolean_t);
36 static int igb_tx_bind(igb_tx_ring_t *, tx_control_block_t *, mblk_t *,
37     uint32_t);
38 static int igb_tx_fill_ring(igb_tx_ring_t *, link_list_t *, hcksum_context_t *);
39 static void igb_save_desc(tx_control_block_t *, uint64_t, size_t);
40 static tx_control_block_t *igb_get_free_list(igb_tx_ring_t *);
41 
42 static void igb_get_hcksum_context(mblk_t *, hcksum_context_t *);
43 static boolean_t igb_check_hcksum_context(igb_tx_ring_t *, hcksum_context_t *);
44 static void igb_fill_hcksum_context(struct e1000_adv_tx_context_desc *,
45     hcksum_context_t *);
46 
47 #ifndef IGB_DEBUG
48 #pragma inline(igb_save_desc)
49 #pragma inline(igb_get_hcksum_context)
50 #pragma inline(igb_check_hcksum_context)
51 #pragma inline(igb_fill_hcksum_context)
52 #endif
53 
54 /*
55  * igb_m_tx
56  *
57  * The GLDv3 interface to call driver's tx routine to transmit
58  * the mblks.
59  */
60 mblk_t *
61 igb_m_tx(void *arg, mblk_t *mp)
62 {
63 	igb_t *igb = (igb_t *)arg;
64 	mblk_t *next;
65 	igb_tx_ring_t *tx_ring;
66 
67 	/*
68 	 * If the adapter is suspended, or it is not started, or the link
69 	 * is not up, the mblks are simply dropped.
70 	 */
71 	if (((igb->igb_state & IGB_SUSPENDED) != 0) ||
72 	    ((igb->igb_state & IGB_STARTED) == 0) ||
73 	    (igb->link_state != LINK_STATE_UP)) {
74 		/* Free the mblk chain */
75 		while (mp != NULL) {
76 			next = mp->b_next;
77 			mp->b_next = NULL;
78 
79 			freemsg(mp);
80 			mp = next;
81 		}
82 
83 		return (NULL);
84 	}
85 
86 	/*
87 	 * Decide which tx ring is used to transmit the packets.
88 	 * This needs to be updated later to fit the new interface
89 	 * of the multiple rings support.
90 	 */
91 	tx_ring = &igb->tx_rings[0];
92 
93 	while (mp != NULL) {
94 		next = mp->b_next;
95 		mp->b_next = NULL;
96 
97 		if (!igb_tx(tx_ring, mp)) {
98 			mp->b_next = next;
99 			break;
100 		}
101 
102 		mp = next;
103 	}
104 
105 	return (mp);
106 }
107 
108 /*
109  * igb_tx - Main transmit processing
110  *
111  * Called from igb_m_tx with an mblk ready to transmit. this
112  * routine sets up the transmit descriptors and sends data to
113  * the wire.
114  *
115  * One mblk can consist of several fragments, each fragment
116  * will be processed with different methods based on the size.
117  * For the fragments with size less than the bcopy threshold,
118  * they will be processed by using bcopy; otherwise, they will
119  * be processed by using DMA binding.
120  *
121  * To process the mblk, a tx control block is got from the
122  * free list. One tx control block contains one tx buffer, which
123  * is used to copy mblk fragments' data; and one tx DMA handle,
124  * which is used to bind a mblk fragment with DMA resource.
125  *
126  * Several small mblk fragments can be copied into one tx control
127  * block's buffer, and then the buffer will be transmitted with
128  * one tx descriptor.
129  *
130  * A large fragment only binds with one tx control block's DMA
131  * handle, and it can span several tx descriptors for transmitting.
132  *
133  * So to transmit a packet (mblk), several tx control blocks can
134  * be used. After the processing, those tx control blocks will
135  * be put to the work list.
136  */
137 static boolean_t
138 igb_tx(igb_tx_ring_t *tx_ring, mblk_t *mp)
139 {
140 	igb_t *igb = tx_ring->igb;
141 	tx_type_t current_flag, next_flag;
142 	uint32_t current_len, next_len;
143 	uint32_t desc_total;
144 	size_t mbsize;
145 	int desc_num;
146 	boolean_t copy_done, eop;
147 	mblk_t *current_mp, *next_mp, *nmp;
148 	tx_control_block_t *tcb;
149 	hcksum_context_t hcksum_context, *hcksum;
150 	link_list_t pending_list;
151 
152 	/* Get the mblk size */
153 	mbsize = 0;
154 	for (nmp = mp; nmp != NULL; nmp = nmp->b_cont) {
155 		mbsize += MBLK_LEN(nmp);
156 	}
157 
158 	/*
159 	 * If the mblk size exceeds the max frame size,
160 	 * discard this mblk, and return B_TRUE
161 	 */
162 	if (mbsize > (igb->max_frame_size - ETHERFCSL)) {
163 		freemsg(mp);
164 		IGB_DEBUGLOG_0(igb, "igb_tx: packet oversize");
165 		return (B_TRUE);
166 	}
167 
168 	/*
169 	 * Check and recycle tx descriptors.
170 	 * The recycle threshold here should be selected carefully
171 	 */
172 	if (tx_ring->tbd_free < tx_ring->recycle_thresh)
173 		tx_ring->tx_recycle(tx_ring);
174 
175 	/*
176 	 * After the recycling, if the tbd_free is less than the
177 	 * overload_threshold, assert overload, return B_FALSE;
178 	 * and we need to re-schedule the tx again.
179 	 */
180 	if (tx_ring->tbd_free < tx_ring->overload_thresh) {
181 		tx_ring->reschedule = B_TRUE;
182 		IGB_DEBUG_STAT(tx_ring->stat_overload);
183 		return (B_FALSE);
184 	}
185 
186 	/*
187 	 * The pending_list is a linked list that is used to save
188 	 * the tx control blocks that have packet data processed
189 	 * but have not put the data to the tx descriptor ring.
190 	 * It is used to reduce the lock contention of the tx_lock.
191 	 */
192 	LINK_LIST_INIT(&pending_list);
193 	desc_num = 0;
194 	desc_total = 0;
195 
196 	current_mp = mp;
197 	current_len = MBLK_LEN(current_mp);
198 	/*
199 	 * Decide which method to use for the first fragment
200 	 */
201 	current_flag = (current_len <= tx_ring->copy_thresh) ?
202 	    USE_COPY : USE_DMA;
203 	/*
204 	 * If the mblk includes several contiguous small fragments,
205 	 * they may be copied into one buffer. This flag is used to
206 	 * indicate whether there are pending fragments that need to
207 	 * be copied to the current tx buffer.
208 	 *
209 	 * If this flag is B_TRUE, it indicates that a new tx control
210 	 * block is needed to process the next fragment using either
211 	 * copy or DMA binding.
212 	 *
213 	 * Otherwise, it indicates that the next fragment will be
214 	 * copied to the current tx buffer that is maintained by the
215 	 * current tx control block. No new tx control block is needed.
216 	 */
217 	copy_done = B_TRUE;
218 	while (current_mp) {
219 		next_mp = current_mp->b_cont;
220 		eop = (next_mp == NULL); /* Last fragment of the packet? */
221 		next_len = eop ? 0: MBLK_LEN(next_mp);
222 
223 		/*
224 		 * When the current fragment is an empty fragment, if
225 		 * the next fragment will still be copied to the current
226 		 * tx buffer, we cannot skip this fragment here. Because
227 		 * the copy processing is pending for completion. We have
228 		 * to process this empty fragment in the tx_copy routine.
229 		 *
230 		 * If the copy processing is completed or a DMA binding
231 		 * processing is just completed, we can just skip this
232 		 * empty fragment.
233 		 */
234 		if ((current_len == 0) && (copy_done)) {
235 			current_mp = next_mp;
236 			current_len = next_len;
237 			current_flag = (current_len <= tx_ring->copy_thresh) ?
238 			    USE_COPY : USE_DMA;
239 			continue;
240 		}
241 
242 		if (copy_done) {
243 			/*
244 			 * Get a new tx control block from the free list
245 			 */
246 			tcb = igb_get_free_list(tx_ring);
247 
248 			if (tcb == NULL) {
249 				IGB_DEBUG_STAT(tx_ring->stat_fail_no_tcb);
250 				goto tx_failure;
251 			}
252 
253 			/*
254 			 * Push the tx control block to the pending list
255 			 * to avoid using lock too early
256 			 */
257 			LIST_PUSH_TAIL(&pending_list, &tcb->link);
258 		}
259 
260 		if (current_flag == USE_COPY) {
261 			/*
262 			 * Check whether to use bcopy or DMA binding to process
263 			 * the next fragment, and if using bcopy, whether we
264 			 * need to continue copying the next fragment into the
265 			 * current tx buffer.
266 			 */
267 			ASSERT((tcb->tx_buf.len + current_len) <=
268 			    tcb->tx_buf.size);
269 
270 			if (eop) {
271 				/*
272 				 * This is the last fragment of the packet, so
273 				 * the copy processing will be completed with
274 				 * this fragment.
275 				 */
276 				next_flag = USE_NONE;
277 				copy_done = B_TRUE;
278 			} else if ((tcb->tx_buf.len + current_len + next_len) >
279 			    tcb->tx_buf.size) {
280 				/*
281 				 * If the next fragment is too large to be
282 				 * copied to the current tx buffer, we need
283 				 * to complete the current copy processing.
284 				 */
285 				next_flag = (next_len > tx_ring->copy_thresh) ?
286 				    USE_DMA: USE_COPY;
287 				copy_done = B_TRUE;
288 			} else if (next_len > tx_ring->copy_thresh) {
289 				/*
290 				 * The next fragment needs to be processed with
291 				 * DMA binding. So the copy prcessing will be
292 				 * completed with the current fragment.
293 				 */
294 				next_flag = USE_DMA;
295 				copy_done = B_TRUE;
296 			} else {
297 				/*
298 				 * Continue to copy the next fragment to the
299 				 * current tx buffer.
300 				 */
301 				next_flag = USE_COPY;
302 				copy_done = B_FALSE;
303 			}
304 
305 			desc_num = igb_tx_copy(tx_ring, tcb, current_mp,
306 			    current_len, copy_done, eop);
307 		} else {
308 			/*
309 			 * Check whether to use bcopy or DMA binding to process
310 			 * the next fragment.
311 			 */
312 			next_flag = (next_len > tx_ring->copy_thresh) ?
313 			    USE_DMA: USE_COPY;
314 			ASSERT(copy_done == B_TRUE);
315 
316 			desc_num = igb_tx_bind(tx_ring, tcb, current_mp,
317 			    current_len);
318 		}
319 
320 		if (desc_num > 0)
321 			desc_total += desc_num;
322 		else if (desc_num < 0)
323 			goto tx_failure;
324 
325 		current_mp = next_mp;
326 		current_len = next_len;
327 		current_flag = next_flag;
328 	}
329 
330 	/*
331 	 * Attach the mblk to the last tx control block
332 	 */
333 	ASSERT(tcb);
334 	ASSERT(tcb->mp == NULL);
335 	tcb->mp = mp;
336 
337 	if (igb->tx_hcksum_enable) {
338 		/*
339 		 * Retrieve checksum context information from the mblk that will
340 		 * be used to decide whether/how to fill the context descriptor.
341 		 */
342 		hcksum = &hcksum_context;
343 		igb_get_hcksum_context(mp, hcksum);
344 	} else {
345 		hcksum = NULL;
346 	}
347 
348 	/*
349 	 * Before fill the tx descriptor ring with the data, we need to
350 	 * ensure there are adequate free descriptors for transmit
351 	 * (including one context descriptor).
352 	 */
353 	if (tx_ring->tbd_free < (desc_total + 1)) {
354 		tx_ring->tx_recycle(tx_ring);
355 	}
356 
357 	mutex_enter(&tx_ring->tx_lock);
358 
359 	/*
360 	 * If the number of free tx descriptors is not enough for transmit
361 	 * then return failure.
362 	 *
363 	 * Note: we must put this check under the mutex protection to
364 	 * ensure the correctness when multiple threads access it in
365 	 * parallel.
366 	 */
367 	if (tx_ring->tbd_free < (desc_total + 1)) {
368 		IGB_DEBUG_STAT(tx_ring->stat_fail_no_tbd);
369 		mutex_exit(&tx_ring->tx_lock);
370 		goto tx_failure;
371 	}
372 
373 	desc_num = igb_tx_fill_ring(tx_ring, &pending_list, hcksum);
374 
375 	ASSERT((desc_num == desc_total) || (desc_num == (desc_total + 1)));
376 
377 	mutex_exit(&tx_ring->tx_lock);
378 
379 	return (B_TRUE);
380 
381 tx_failure:
382 	/*
383 	 * Discard the mblk and free the used resources
384 	 */
385 	tcb = (tx_control_block_t *)LIST_GET_HEAD(&pending_list);
386 	while (tcb) {
387 		tcb->mp = NULL;
388 
389 		igb_free_tcb(tcb);
390 
391 		tcb = (tx_control_block_t *)
392 		    LIST_GET_NEXT(&pending_list, &tcb->link);
393 	}
394 
395 	/*
396 	 * Return the tx control blocks in the pending list to the free list.
397 	 */
398 	igb_put_free_list(tx_ring, &pending_list);
399 
400 	/* Transmit failed, do not drop the mblk, rechedule the transmit */
401 	tx_ring->reschedule = B_TRUE;
402 
403 	return (B_FALSE);
404 }
405 
406 /*
407  * igb_tx_copy
408  *
409  * Copy the mblk fragment to the pre-allocated tx buffer
410  */
411 static int
412 igb_tx_copy(igb_tx_ring_t *tx_ring, tx_control_block_t *tcb, mblk_t *mp,
413     uint32_t len, boolean_t copy_done, boolean_t eop)
414 {
415 	dma_buffer_t *tx_buf;
416 	uint32_t desc_num;
417 	_NOTE(ARGUNUSED(tx_ring));
418 
419 	tx_buf = &tcb->tx_buf;
420 
421 	/*
422 	 * Copy the packet data of the mblk fragment into the
423 	 * pre-allocated tx buffer, which is maintained by the
424 	 * tx control block.
425 	 *
426 	 * Several mblk fragments can be copied into one tx buffer.
427 	 * The destination address of the current copied fragment in
428 	 * the tx buffer is next to the end of the previous copied
429 	 * fragment.
430 	 */
431 	if (len > 0) {
432 		bcopy(mp->b_rptr, tx_buf->address + tx_buf->len, len);
433 
434 		tx_buf->len += len;
435 		tcb->frag_num++;
436 	}
437 
438 	desc_num = 0;
439 
440 	/*
441 	 * If it is the last fragment copied to the current tx buffer,
442 	 * in other words, if there's no remaining fragment or the remaining
443 	 * fragment requires a new tx control block to process, we need to
444 	 * complete the current copy processing by syncing up the current
445 	 * DMA buffer and saving the descriptor data.
446 	 */
447 	if (copy_done) {
448 		/*
449 		 * For the packet smaller than 64 bytes, we need to
450 		 * pad it to 60 bytes. The NIC hardware will add 4
451 		 * bytes of CRC.
452 		 */
453 		if (eop && (tx_buf->len < ETHERMIN)) {
454 			bzero(tx_buf->address + tx_buf->len,
455 			    ETHERMIN - tx_buf->len);
456 			tx_buf->len = ETHERMIN;
457 		}
458 
459 		/*
460 		 * Sync the DMA buffer of the packet data
461 		 */
462 		DMA_SYNC(tx_buf, DDI_DMA_SYNC_FORDEV);
463 
464 		tcb->tx_type = USE_COPY;
465 
466 		/*
467 		 * Save the address and length to the private data structure
468 		 * of the tx control block, which will be used to fill the
469 		 * tx descriptor ring after all the fragments are processed.
470 		 */
471 		igb_save_desc(tcb, tx_buf->dma_address, tx_buf->len);
472 		desc_num++;
473 	}
474 
475 	return (desc_num);
476 }
477 
478 /*
479  * igb_tx_bind
480  *
481  * Bind the mblk fragment with DMA
482  */
483 static int
484 igb_tx_bind(igb_tx_ring_t *tx_ring, tx_control_block_t *tcb, mblk_t *mp,
485     uint32_t len)
486 {
487 	int status, i;
488 	ddi_dma_cookie_t dma_cookie;
489 	uint_t ncookies;
490 	int desc_num;
491 
492 	/*
493 	 * Use DMA binding to process the mblk fragment
494 	 */
495 	status = ddi_dma_addr_bind_handle(tcb->tx_dma_handle, NULL,
496 	    (caddr_t)mp->b_rptr, len,
497 	    DDI_DMA_WRITE | DDI_DMA_STREAMING, DDI_DMA_DONTWAIT,
498 	    0, &dma_cookie, &ncookies);
499 
500 	if (status != DDI_DMA_MAPPED) {
501 		IGB_DEBUG_STAT(tx_ring->stat_fail_dma_bind);
502 		return (-1);
503 	}
504 
505 	tcb->frag_num++;
506 	tcb->tx_type = USE_DMA;
507 	/*
508 	 * Each fragment can span several cookies. One cookie will have
509 	 * one tx descriptor to transmit.
510 	 */
511 	desc_num = 0;
512 	for (i = ncookies; i > 0; i--) {
513 		/*
514 		 * Save the address and length to the private data structure
515 		 * of the tx control block, which will be used to fill the
516 		 * tx descriptor ring after all the fragments are processed.
517 		 */
518 		igb_save_desc(tcb,
519 		    dma_cookie.dmac_laddress,
520 		    dma_cookie.dmac_size);
521 
522 		desc_num++;
523 
524 		if (i > 1)
525 			ddi_dma_nextcookie(tcb->tx_dma_handle, &dma_cookie);
526 	}
527 
528 	return (desc_num);
529 }
530 
531 /*
532  * igb_get_hcksum_context
533  *
534  * Get the hcksum context information from the mblk
535  */
536 static void
537 igb_get_hcksum_context(mblk_t *mp, hcksum_context_t *hcksum)
538 {
539 	uint32_t start;
540 	uint32_t flags;
541 	uint32_t len;
542 	uint32_t size;
543 	uint32_t offset;
544 	unsigned char *pos;
545 	ushort_t etype;
546 	uint32_t mac_hdr_len;
547 	uint32_t l4_proto;
548 
549 	ASSERT(mp != NULL);
550 
551 	hcksum_retrieve(mp, NULL, NULL, &start, NULL, NULL, NULL, &flags);
552 
553 	hcksum->hcksum_flags = flags;
554 
555 	if (flags == 0)
556 		return;
557 
558 	etype = 0;
559 	mac_hdr_len = 0;
560 	l4_proto = 0;
561 
562 	/*
563 	 * Firstly get the position of the ether_type/ether_tpid.
564 	 * Here we don't assume the ether (VLAN) header is fully included
565 	 * in one mblk fragment, so we go thourgh the fragments to parse
566 	 * the ether type.
567 	 */
568 	size = len = MBLK_LEN(mp);
569 	offset = offsetof(struct ether_header, ether_type);
570 	while (size <= offset) {
571 		mp = mp->b_cont;
572 		ASSERT(mp != NULL);
573 		len = MBLK_LEN(mp);
574 		size += len;
575 	}
576 	pos = mp->b_rptr + offset + len - size;
577 
578 	etype = ntohs(*(ushort_t *)(uintptr_t)pos);
579 	if (etype == ETHERTYPE_VLAN) {
580 		/*
581 		 * Get the position of the ether_type in VLAN header
582 		 */
583 		offset = offsetof(struct ether_vlan_header, ether_type);
584 		while (size <= offset) {
585 			mp = mp->b_cont;
586 			ASSERT(mp != NULL);
587 			len = MBLK_LEN(mp);
588 			size += len;
589 		}
590 		pos = mp->b_rptr + offset + len - size;
591 
592 		etype = ntohs(*(ushort_t *)(uintptr_t)pos);
593 		mac_hdr_len = sizeof (struct ether_vlan_header);
594 	} else {
595 		mac_hdr_len = sizeof (struct ether_header);
596 	}
597 
598 	/*
599 	 * Here we don't assume the IP(V6) header is fully included in
600 	 * one mblk fragment, so we go thourgh the fragments to parse
601 	 * the protocol type.
602 	 */
603 	switch (etype) {
604 	case ETHERTYPE_IP:
605 		offset = offsetof(ipha_t, ipha_protocol) + mac_hdr_len;
606 		while (size <= offset) {
607 			mp = mp->b_cont;
608 			ASSERT(mp != NULL);
609 			len = MBLK_LEN(mp);
610 			size += len;
611 		}
612 		pos = mp->b_rptr + offset + len - size;
613 
614 		l4_proto = *(uint8_t *)pos;
615 		break;
616 	case ETHERTYPE_IPV6:
617 		offset = offsetof(ip6_t, ip6_nxt) + mac_hdr_len;
618 		while (size <= offset) {
619 			mp = mp->b_cont;
620 			ASSERT(mp != NULL);
621 			len = MBLK_LEN(mp);
622 			size += len;
623 		}
624 		pos = mp->b_rptr + offset + len - size;
625 
626 		l4_proto = *(uint8_t *)pos;
627 		break;
628 	default:
629 		/* Unrecoverable error */
630 		IGB_DEBUGLOG_0(NULL, "Ether type error with tx hcksum");
631 		return;
632 	}
633 
634 	hcksum->mac_hdr_len = mac_hdr_len;
635 	hcksum->ip_hdr_len = start;
636 	hcksum->l4_proto = l4_proto;
637 }
638 
639 /*
640  * igb_check_hcksum_context
641  *
642  * Check if a new context descriptor is needed
643  */
644 static boolean_t
645 igb_check_hcksum_context(igb_tx_ring_t *tx_ring, hcksum_context_t *hcksum)
646 {
647 	hcksum_context_t *last;
648 
649 	if (hcksum == NULL)
650 		return (B_FALSE);
651 
652 	/*
653 	 * Compare the checksum data retrieved from the mblk and the
654 	 * stored checksum data of the last context descriptor. The data
655 	 * need to be checked are:
656 	 *	hcksum_flags
657 	 *	l4_proto
658 	 *	mac_hdr_len
659 	 *	ip_hdr_len
660 	 * Either one of the above data is changed, a new context descriptor
661 	 * will be needed.
662 	 */
663 	last = &tx_ring->hcksum_context;
664 
665 	if (hcksum->hcksum_flags != 0) {
666 		if ((hcksum->hcksum_flags != last->hcksum_flags) ||
667 		    (hcksum->l4_proto != last->l4_proto) ||
668 		    (hcksum->mac_hdr_len != last->mac_hdr_len) ||
669 		    (hcksum->ip_hdr_len != last->ip_hdr_len)) {
670 
671 			return (B_TRUE);
672 		}
673 	}
674 
675 	return (B_FALSE);
676 }
677 
678 /*
679  * igb_fill_hcksum_context
680  *
681  * Fill the context descriptor with hardware checksum informations
682  */
683 static void
684 igb_fill_hcksum_context(struct e1000_adv_tx_context_desc *ctx_tbd,
685     hcksum_context_t *hcksum)
686 {
687 	/*
688 	 * Fill the context descriptor with the checksum
689 	 * context information we've got
690 	 */
691 	ctx_tbd->vlan_macip_lens = hcksum->ip_hdr_len;
692 	ctx_tbd->vlan_macip_lens |= hcksum->mac_hdr_len <<
693 	    E1000_ADVTXD_MACLEN_SHIFT;
694 
695 	ctx_tbd->type_tucmd_mlhl =
696 	    E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
697 
698 	if (hcksum->hcksum_flags & HCK_IPV4_HDRCKSUM)
699 		ctx_tbd->type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
700 
701 	if (hcksum->hcksum_flags & HCK_PARTIALCKSUM) {
702 		switch (hcksum->l4_proto) {
703 		case IPPROTO_TCP:
704 			ctx_tbd->type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
705 			break;
706 		case IPPROTO_UDP:
707 			/*
708 			 * We don't have to explicitly set:
709 			 *	ctx_tbd->type_tucmd_mlhl |=
710 			 *	    E1000_ADVTXD_TUCMD_L4T_UDP;
711 			 * Because E1000_ADVTXD_TUCMD_L4T_UDP == 0b
712 			 */
713 			break;
714 		default:
715 			/* Unrecoverable error */
716 			IGB_DEBUGLOG_0(NULL, "L4 type error with tx hcksum");
717 			break;
718 		}
719 	}
720 
721 	ctx_tbd->seqnum_seed = 0;
722 	ctx_tbd->mss_l4len_idx = 0;
723 }
724 
725 /*
726  * igb_tx_fill_ring
727  *
728  * Fill the tx descriptor ring with the data
729  */
730 static int
731 igb_tx_fill_ring(igb_tx_ring_t *tx_ring, link_list_t *pending_list,
732     hcksum_context_t *hcksum)
733 {
734 	struct e1000_hw *hw = &tx_ring->igb->hw;
735 	boolean_t load_context;
736 	uint32_t index, tcb_index, desc_num;
737 	union e1000_adv_tx_desc *tbd, *first_tbd;
738 	tx_control_block_t *tcb, *first_tcb;
739 	uint32_t hcksum_flags;
740 	int i;
741 
742 	ASSERT(mutex_owned(&tx_ring->tx_lock));
743 
744 	tbd = NULL;
745 	first_tbd = NULL;
746 	first_tcb = NULL;
747 	desc_num = 0;
748 	hcksum_flags = 0;
749 	load_context = B_FALSE;
750 
751 	/*
752 	 * Get the index of the first tx descriptor that will be filled,
753 	 * and the index of the first work list item that will be attached
754 	 * with the first used tx control block in the pending list.
755 	 * Note: the two indexes are the same.
756 	 */
757 	index = tx_ring->tbd_tail;
758 	tcb_index = tx_ring->tbd_tail;
759 
760 	if (hcksum != NULL) {
761 		hcksum_flags = hcksum->hcksum_flags;
762 
763 		/*
764 		 * Check if a new context descriptor is needed for this packet
765 		 */
766 		load_context = igb_check_hcksum_context(tx_ring, hcksum);
767 		if (load_context) {
768 			first_tcb = (tx_control_block_t *)
769 			    LIST_GET_HEAD(pending_list);
770 			tbd = &tx_ring->tbd_ring[index];
771 
772 			/*
773 			 * Fill the context descriptor with the
774 			 * hardware checksum offload informations.
775 			 */
776 			igb_fill_hcksum_context(
777 			    (struct e1000_adv_tx_context_desc *)tbd, hcksum);
778 
779 			index = NEXT_INDEX(index, 1, tx_ring->ring_size);
780 			desc_num++;
781 
782 			/*
783 			 * Store the checksum context data if
784 			 * a new context descriptor is added
785 			 */
786 			tx_ring->hcksum_context = *hcksum;
787 		}
788 	}
789 
790 	first_tbd = &tx_ring->tbd_ring[index];
791 
792 	/*
793 	 * Fill tx data descriptors with the data saved in the pending list.
794 	 * The tx control blocks in the pending list are added to the work list
795 	 * at the same time.
796 	 *
797 	 * The work list is strictly 1:1 corresponding to the descriptor ring.
798 	 * One item of the work list corresponds to one tx descriptor. Because
799 	 * one tx control block can span multiple tx descriptors, the tx
800 	 * control block will be added to the first work list item that
801 	 * corresponds to the first tx descriptor generated from that tx
802 	 * control block.
803 	 */
804 	tcb = (tx_control_block_t *)LIST_POP_HEAD(pending_list);
805 	while (tcb != NULL) {
806 
807 		for (i = 0; i < tcb->desc_num; i++) {
808 			tbd = &tx_ring->tbd_ring[index];
809 
810 			tbd->read.buffer_addr = tcb->desc[i].address;
811 			tbd->read.cmd_type_len = tcb->desc[i].length;
812 
813 			tbd->read.cmd_type_len |= E1000_ADVTXD_DCMD_RS |
814 			    E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_DATA;
815 
816 			tbd->read.olinfo_status = 0;
817 
818 			index = NEXT_INDEX(index, 1, tx_ring->ring_size);
819 			desc_num++;
820 		}
821 
822 		if (first_tcb != NULL) {
823 			/*
824 			 * Count the checksum context descriptor for
825 			 * the first tx control block.
826 			 */
827 			first_tcb->desc_num++;
828 			first_tcb = NULL;
829 		}
830 
831 		/*
832 		 * Add the tx control block to the work list
833 		 */
834 		ASSERT(tx_ring->work_list[tcb_index] == NULL);
835 		tx_ring->work_list[tcb_index] = tcb;
836 
837 		tcb_index = index;
838 		tcb = (tx_control_block_t *)LIST_POP_HEAD(pending_list);
839 	}
840 
841 	/*
842 	 * The Insert Ethernet CRC (IFCS) bit and the checksum fields are only
843 	 * valid in the first descriptor of the packet.
844 	 */
845 	ASSERT(first_tbd != NULL);
846 	first_tbd->read.cmd_type_len |= E1000_ADVTXD_DCMD_IFCS;
847 
848 	/* Set hardware checksum bits */
849 	if (hcksum_flags != 0) {
850 		if (hcksum_flags & HCK_IPV4_HDRCKSUM)
851 			first_tbd->read.olinfo_status |=
852 			    E1000_TXD_POPTS_IXSM << 8;
853 		if (hcksum_flags & HCK_PARTIALCKSUM)
854 			first_tbd->read.olinfo_status |=
855 			    E1000_TXD_POPTS_TXSM << 8;
856 	}
857 
858 	/*
859 	 * The last descriptor of packet needs End Of Packet (EOP),
860 	 * and Report Status (RS) bits set
861 	 */
862 	ASSERT(tbd != NULL);
863 	tbd->read.cmd_type_len |=
864 	    E1000_ADVTXD_DCMD_EOP | E1000_ADVTXD_DCMD_RS;
865 
866 	/*
867 	 * Sync the DMA buffer of the tx descriptor ring
868 	 */
869 	DMA_SYNC(&tx_ring->tbd_area, DDI_DMA_SYNC_FORDEV);
870 
871 	/*
872 	 * Update the number of the free tx descriptors.
873 	 * The mutual exclusion between the transmission and the recycling
874 	 * (for the tx descriptor ring and the work list) is implemented
875 	 * with the atomic operation on the number of the free tx descriptors.
876 	 *
877 	 * Note: we should always decrement the counter tbd_free before
878 	 * advancing the hardware TDT pointer to avoid the race condition -
879 	 * before the counter tbd_free is decremented, the transmit of the
880 	 * tx descriptors has done and the counter tbd_free is increased by
881 	 * the tx recycling.
882 	 */
883 	i = igb_atomic_reserve(&tx_ring->tbd_free, desc_num);
884 	ASSERT(i >= 0);
885 
886 	tx_ring->tbd_tail = index;
887 
888 	/*
889 	 * Advance the hardware TDT pointer of the tx descriptor ring
890 	 */
891 	E1000_WRITE_REG(hw, E1000_TDT(tx_ring->index), index);
892 
893 	return (desc_num);
894 }
895 
896 /*
897  * igb_save_desc
898  *
899  * Save the address/length pair to the private array
900  * of the tx control block. The address/length pairs
901  * will be filled into the tx descriptor ring later.
902  */
903 static void
904 igb_save_desc(tx_control_block_t *tcb, uint64_t address, size_t length)
905 {
906 	sw_desc_t *desc;
907 
908 	desc = &tcb->desc[tcb->desc_num];
909 	desc->address = address;
910 	desc->length = length;
911 
912 	tcb->desc_num++;
913 }
914 
915 /*
916  * igb_tx_recycle_legacy
917  *
918  * Recycle the tx descriptors and tx control blocks.
919  *
920  * The work list is traversed to check if the corresponding
921  * tx descriptors have been transmitted. If so, the resources
922  * bound to the tx control blocks will be freed, and those
923  * tx control blocks will be returned to the free list.
924  */
925 uint32_t
926 igb_tx_recycle_legacy(igb_tx_ring_t *tx_ring)
927 {
928 	uint32_t index, last_index;
929 	int desc_num;
930 	boolean_t desc_done;
931 	tx_control_block_t *tcb;
932 	link_list_t pending_list;
933 
934 	/*
935 	 * The mutex_tryenter() is used to avoid unnecessary
936 	 * lock contention.
937 	 */
938 	if (mutex_tryenter(&tx_ring->recycle_lock) == 0)
939 		return (0);
940 
941 	ASSERT(tx_ring->tbd_free <= tx_ring->ring_size);
942 
943 	if (tx_ring->tbd_free == tx_ring->ring_size) {
944 		tx_ring->recycle_fail = 0;
945 		tx_ring->stall_watchdog = 0;
946 		mutex_exit(&tx_ring->recycle_lock);
947 		return (0);
948 	}
949 
950 	/*
951 	 * Sync the DMA buffer of the tx descriptor ring
952 	 */
953 	DMA_SYNC(&tx_ring->tbd_area, DDI_DMA_SYNC_FORKERNEL);
954 
955 	LINK_LIST_INIT(&pending_list);
956 	desc_num = 0;
957 	index = tx_ring->tbd_head;	/* Index of next tbd/tcb to recycle */
958 
959 	tcb = tx_ring->work_list[index];
960 	ASSERT(tcb != NULL);
961 
962 	desc_done = B_TRUE;
963 	while (desc_done && (tcb != NULL)) {
964 
965 		/*
966 		 * Get the last tx descriptor of the tx control block.
967 		 * If the last tx descriptor is done, it is done with
968 		 * all the tx descriptors of the tx control block.
969 		 * Then the tx control block and all the corresponding
970 		 * tx descriptors can be recycled.
971 		 */
972 		last_index = NEXT_INDEX(index, tcb->desc_num - 1,
973 		    tx_ring->ring_size);
974 
975 		/*
976 		 * Check if the Descriptor Done bit is set
977 		 */
978 		desc_done = tx_ring->tbd_ring[last_index].wb.status &
979 		    E1000_TXD_STAT_DD;
980 		if (desc_done) {
981 			/*
982 			 * Strip off the tx control block from the work list,
983 			 * and add it to the pending list.
984 			 */
985 			tx_ring->work_list[index] = NULL;
986 			LIST_PUSH_TAIL(&pending_list, &tcb->link);
987 
988 			/*
989 			 * Count the total number of the tx descriptors recycled
990 			 */
991 			desc_num += tcb->desc_num;
992 
993 			/*
994 			 * Advance the index of the tx descriptor ring
995 			 */
996 			index = NEXT_INDEX(last_index, 1, tx_ring->ring_size);
997 
998 			tcb = tx_ring->work_list[index];
999 		}
1000 	}
1001 
1002 	/*
1003 	 * If no tx descriptors are recycled, no need to do more processing
1004 	 */
1005 	if (desc_num == 0) {
1006 		tx_ring->recycle_fail++;
1007 		mutex_exit(&tx_ring->recycle_lock);
1008 		return (0);
1009 	}
1010 
1011 	tx_ring->recycle_fail = 0;
1012 	tx_ring->stall_watchdog = 0;
1013 
1014 	/*
1015 	 * Update the head index of the tx descriptor ring
1016 	 */
1017 	tx_ring->tbd_head = index;
1018 
1019 	/*
1020 	 * Update the number of the free tx descriptors with atomic operations
1021 	 */
1022 	atomic_add_32(&tx_ring->tbd_free, desc_num);
1023 
1024 	mutex_exit(&tx_ring->recycle_lock);
1025 
1026 	/*
1027 	 * Free the resources used by the tx control blocks
1028 	 * in the pending list
1029 	 */
1030 	tcb = (tx_control_block_t *)LIST_GET_HEAD(&pending_list);
1031 	while (tcb != NULL) {
1032 		/*
1033 		 * Release the resources occupied by the tx control block
1034 		 */
1035 		igb_free_tcb(tcb);
1036 
1037 		tcb = (tx_control_block_t *)
1038 		    LIST_GET_NEXT(&pending_list, &tcb->link);
1039 	}
1040 
1041 	/*
1042 	 * Add the tx control blocks in the pending list to the free list.
1043 	 */
1044 	igb_put_free_list(tx_ring, &pending_list);
1045 
1046 	return (desc_num);
1047 }
1048 
1049 /*
1050  * igb_tx_recycle_head_wb
1051  *
1052  * Check the head write-back, and recycle all the transmitted
1053  * tx descriptors and tx control blocks.
1054  */
1055 uint32_t
1056 igb_tx_recycle_head_wb(igb_tx_ring_t *tx_ring)
1057 {
1058 	uint32_t index;
1059 	uint32_t head_wb;
1060 	int desc_num;
1061 	tx_control_block_t *tcb;
1062 	link_list_t pending_list;
1063 
1064 	/*
1065 	 * The mutex_tryenter() is used to avoid unnecessary
1066 	 * lock contention.
1067 	 */
1068 	if (mutex_tryenter(&tx_ring->recycle_lock) == 0)
1069 		return (0);
1070 
1071 	ASSERT(tx_ring->tbd_free <= tx_ring->ring_size);
1072 
1073 	if (tx_ring->tbd_free == tx_ring->ring_size) {
1074 		tx_ring->recycle_fail = 0;
1075 		tx_ring->stall_watchdog = 0;
1076 		mutex_exit(&tx_ring->recycle_lock);
1077 		return (0);
1078 	}
1079 
1080 	/*
1081 	 * Sync the DMA buffer of the tx descriptor ring
1082 	 *
1083 	 * Note: For head write-back mode, the tx descriptors will not
1084 	 * be written back, but the head write-back value is stored at
1085 	 * the last extra tbd at the end of the DMA area, we still need
1086 	 * to sync the head write-back value for kernel.
1087 	 *
1088 	 * DMA_SYNC(&tx_ring->tbd_area, DDI_DMA_SYNC_FORKERNEL);
1089 	 */
1090 	(void) ddi_dma_sync(tx_ring->tbd_area.dma_handle,
1091 	    sizeof (union e1000_adv_tx_desc) * tx_ring->ring_size,
1092 	    sizeof (uint32_t),
1093 	    DDI_DMA_SYNC_FORKERNEL);
1094 
1095 	LINK_LIST_INIT(&pending_list);
1096 	desc_num = 0;
1097 	index = tx_ring->tbd_head;	/* Next index to clean */
1098 
1099 	/*
1100 	 * Get the value of head write-back
1101 	 */
1102 	head_wb = *tx_ring->tbd_head_wb;
1103 	while (index != head_wb) {
1104 		tcb = tx_ring->work_list[index];
1105 		ASSERT(tcb != NULL);
1106 
1107 		if (OFFSET(index, head_wb, tx_ring->ring_size) <
1108 		    tcb->desc_num) {
1109 			/*
1110 			 * The current tx control block is not
1111 			 * completely transmitted, stop recycling
1112 			 */
1113 			break;
1114 		}
1115 
1116 		/*
1117 		 * Strip off the tx control block from the work list,
1118 		 * and add it to the pending list.
1119 		 */
1120 		tx_ring->work_list[index] = NULL;
1121 		LIST_PUSH_TAIL(&pending_list, &tcb->link);
1122 
1123 		/*
1124 		 * Advance the index of the tx descriptor ring
1125 		 */
1126 		index = NEXT_INDEX(index, tcb->desc_num, tx_ring->ring_size);
1127 
1128 		/*
1129 		 * Count the total number of the tx descriptors recycled
1130 		 */
1131 		desc_num += tcb->desc_num;
1132 	}
1133 
1134 	/*
1135 	 * If no tx descriptors are recycled, no need to do more processing
1136 	 */
1137 	if (desc_num == 0) {
1138 		tx_ring->recycle_fail++;
1139 		mutex_exit(&tx_ring->recycle_lock);
1140 		return (0);
1141 	}
1142 
1143 	tx_ring->recycle_fail = 0;
1144 	tx_ring->stall_watchdog = 0;
1145 
1146 	/*
1147 	 * Update the head index of the tx descriptor ring
1148 	 */
1149 	tx_ring->tbd_head = index;
1150 
1151 	/*
1152 	 * Update the number of the free tx descriptors with atomic operations
1153 	 */
1154 	atomic_add_32(&tx_ring->tbd_free, desc_num);
1155 
1156 	mutex_exit(&tx_ring->recycle_lock);
1157 
1158 	/*
1159 	 * Free the resources used by the tx control blocks
1160 	 * in the pending list
1161 	 */
1162 	tcb = (tx_control_block_t *)LIST_GET_HEAD(&pending_list);
1163 	while (tcb) {
1164 		/*
1165 		 * Release the resources occupied by the tx control block
1166 		 */
1167 		igb_free_tcb(tcb);
1168 
1169 		tcb = (tx_control_block_t *)
1170 		    LIST_GET_NEXT(&pending_list, &tcb->link);
1171 	}
1172 
1173 	/*
1174 	 * Add the tx control blocks in the pending list to the free list.
1175 	 */
1176 	igb_put_free_list(tx_ring, &pending_list);
1177 
1178 	return (desc_num);
1179 }
1180 
1181 /*
1182  * igb_free_tcb - free up the tx control block
1183  *
1184  * Free the resources of the tx control block, including
1185  * unbind the previously bound DMA handle, and reset other
1186  * control fields.
1187  */
1188 void
1189 igb_free_tcb(tx_control_block_t *tcb)
1190 {
1191 	switch (tcb->tx_type) {
1192 	case USE_COPY:
1193 		/*
1194 		 * Reset the buffer length that is used for copy
1195 		 */
1196 		tcb->tx_buf.len = 0;
1197 		break;
1198 	case USE_DMA:
1199 		/*
1200 		 * Release the DMA resource that is used for
1201 		 * DMA binding.
1202 		 */
1203 		(void) ddi_dma_unbind_handle(tcb->tx_dma_handle);
1204 		break;
1205 	default:
1206 		break;
1207 	}
1208 
1209 	/*
1210 	 * Free the mblk
1211 	 */
1212 	if (tcb->mp != NULL) {
1213 		freemsg(tcb->mp);
1214 		tcb->mp = NULL;
1215 	}
1216 
1217 	tcb->tx_type = USE_NONE;
1218 	tcb->frag_num = 0;
1219 	tcb->desc_num = 0;
1220 }
1221 
1222 /*
1223  * igb_get_free_list - Get a free tx control block from the free list
1224  *
1225  * The atomic operation on the number of the available tx control block
1226  * in the free list is used to keep this routine mutual exclusive with
1227  * the routine igb_put_check_list.
1228  */
1229 static tx_control_block_t *
1230 igb_get_free_list(igb_tx_ring_t *tx_ring)
1231 {
1232 	tx_control_block_t *tcb;
1233 
1234 	/*
1235 	 * Check and update the number of the free tx control block
1236 	 * in the free list.
1237 	 */
1238 	if (igb_atomic_reserve(&tx_ring->tcb_free, 1) < 0)
1239 		return (NULL);
1240 
1241 	mutex_enter(&tx_ring->tcb_head_lock);
1242 
1243 	tcb = tx_ring->free_list[tx_ring->tcb_head];
1244 	ASSERT(tcb != NULL);
1245 	tx_ring->free_list[tx_ring->tcb_head] = NULL;
1246 	tx_ring->tcb_head = NEXT_INDEX(tx_ring->tcb_head, 1,
1247 	    tx_ring->free_list_size);
1248 
1249 	mutex_exit(&tx_ring->tcb_head_lock);
1250 
1251 	return (tcb);
1252 }
1253 
1254 /*
1255  * igb_put_free_list
1256  *
1257  * Put a list of used tx control blocks back to the free list
1258  *
1259  * A mutex is used here to ensure the serialization. The mutual exclusion
1260  * between igb_get_free_list and igb_put_free_list is implemented with
1261  * the atomic operation on the counter tcb_free.
1262  */
1263 void
1264 igb_put_free_list(igb_tx_ring_t *tx_ring, link_list_t *pending_list)
1265 {
1266 	uint32_t index;
1267 	int tcb_num;
1268 	tx_control_block_t *tcb;
1269 
1270 	mutex_enter(&tx_ring->tcb_tail_lock);
1271 
1272 	index = tx_ring->tcb_tail;
1273 
1274 	tcb_num = 0;
1275 	tcb = (tx_control_block_t *)LIST_POP_HEAD(pending_list);
1276 	while (tcb != NULL) {
1277 		ASSERT(tx_ring->free_list[index] == NULL);
1278 		tx_ring->free_list[index] = tcb;
1279 
1280 		tcb_num++;
1281 
1282 		index = NEXT_INDEX(index, 1, tx_ring->free_list_size);
1283 
1284 		tcb = (tx_control_block_t *)LIST_POP_HEAD(pending_list);
1285 	}
1286 
1287 	tx_ring->tcb_tail = index;
1288 
1289 	/*
1290 	 * Update the number of the free tx control block
1291 	 * in the free list. This operation must be placed
1292 	 * under the protection of the lock.
1293 	 */
1294 	atomic_add_32(&tx_ring->tcb_free, tcb_num);
1295 
1296 	mutex_exit(&tx_ring->tcb_tail_lock);
1297 }
1298