1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright(c) 2007-2010 Intel Corporation. All rights reserved.
24 */
25
26 /*
27 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
28 */
29
30 #include "igb_sw.h"
31
32 static boolean_t igb_tx(igb_tx_ring_t *, mblk_t *);
33 static int igb_tx_copy(igb_tx_ring_t *, tx_control_block_t *, mblk_t *,
34 uint32_t, boolean_t);
35 static int igb_tx_bind(igb_tx_ring_t *, tx_control_block_t *, mblk_t *,
36 uint32_t);
37 static int igb_tx_fill_ring(igb_tx_ring_t *, link_list_t *, tx_context_t *,
38 size_t);
39 static void igb_save_desc(tx_control_block_t *, uint64_t, size_t);
40 static tx_control_block_t *igb_get_free_list(igb_tx_ring_t *);
41 static int igb_get_tx_context(mblk_t *, tx_context_t *);
42 static boolean_t igb_check_tx_context(igb_tx_ring_t *, tx_context_t *);
43 static void igb_fill_tx_context(struct e1000_adv_tx_context_desc *,
44 tx_context_t *, uint32_t);
45
46 #ifndef IGB_DEBUG
47 #pragma inline(igb_save_desc)
48 #pragma inline(igb_get_tx_context)
49 #pragma inline(igb_check_tx_context)
50 #pragma inline(igb_fill_tx_context)
51 #endif
52
53 mblk_t *
igb_tx_ring_send(void * arg,mblk_t * mp)54 igb_tx_ring_send(void *arg, mblk_t *mp)
55 {
56 igb_tx_ring_t *tx_ring = (igb_tx_ring_t *)arg;
57 igb_t *igb;
58
59 ASSERT(tx_ring != NULL);
60
61 igb = tx_ring->igb;
62
63 if ((igb->igb_state & IGB_SUSPENDED) ||
64 (igb->igb_state & IGB_ERROR) ||
65 !(igb->igb_state & IGB_STARTED) ||
66 igb->link_state != LINK_STATE_UP) {
67 freemsg(mp);
68 return (NULL);
69 }
70
71 return ((igb_tx(tx_ring, mp)) ? NULL : mp);
72 }
73
74 /*
75 * igb_tx - Main transmit processing
76 *
77 * Called from igb_m_tx with an mblk ready to transmit. this
78 * routine sets up the transmit descriptors and sends data to
79 * the wire.
80 *
81 * One mblk can consist of several fragments, each fragment
82 * will be processed with different methods based on the size.
83 * For the fragments with size less than the bcopy threshold,
84 * they will be processed by using bcopy; otherwise, they will
85 * be processed by using DMA binding.
86 *
87 * To process the mblk, a tx control block is got from the
88 * free list. One tx control block contains one tx buffer, which
89 * is used to copy mblk fragments' data; and one tx DMA handle,
90 * which is used to bind a mblk fragment with DMA resource.
91 *
92 * Several small mblk fragments can be copied into one tx control
93 * block's buffer, and then the buffer will be transmitted with
94 * one tx descriptor.
95 *
96 * A large fragment only binds with one tx control block's DMA
97 * handle, and it can span several tx descriptors for transmitting.
98 *
99 * So to transmit a packet (mblk), several tx control blocks can
100 * be used. After the processing, those tx control blocks will
101 * be put to the work list.
102 */
103 static boolean_t
igb_tx(igb_tx_ring_t * tx_ring,mblk_t * mp)104 igb_tx(igb_tx_ring_t *tx_ring, mblk_t *mp)
105 {
106 igb_t *igb = tx_ring->igb;
107 tx_type_t current_flag, next_flag;
108 uint32_t current_len, next_len;
109 uint32_t desc_total;
110 size_t mbsize;
111 int desc_num;
112 boolean_t copy_done, eop;
113 mblk_t *current_mp, *next_mp, *nmp;
114 tx_control_block_t *tcb;
115 tx_context_t tx_context, *ctx;
116 link_list_t pending_list;
117 mblk_t *hdr_new_mp = NULL;
118 mblk_t *hdr_previous_mp = NULL;
119 mblk_t *hdr_current_mp = NULL;
120 uint32_t hdr_frag_len;
121 uint32_t hdr_len, len;
122 uint32_t copy_thresh;
123
124 copy_thresh = igb->tx_copy_thresh;
125
126 /* Get the mblk size */
127 mbsize = 0;
128 for (nmp = mp; nmp != NULL; nmp = nmp->b_cont) {
129 mbsize += MBLKL(nmp);
130 }
131
132 if (igb->tx_hcksum_enable) {
133 ctx = &tx_context;
134 /*
135 * Retrieve offloading context information from the mblk
136 * that will be used to decide whether/how to fill the
137 * context descriptor.
138 */
139 if (igb_get_tx_context(mp, ctx) != TX_CXT_SUCCESS) {
140 freemsg(mp);
141 return (B_TRUE);
142 }
143
144 if ((ctx->lso_flag &&
145 (mbsize > (ctx->mac_hdr_len + IGB_LSO_MAXLEN))) ||
146 (!ctx->lso_flag &&
147 (mbsize > (igb->max_frame_size - ETHERFCSL)))) {
148 freemsg(mp);
149 igb_log(igb, IGB_LOG_INFO, "igb_tx: packet oversize");
150 return (B_TRUE);
151 }
152 } else {
153 ctx = NULL;
154 if (mbsize > (igb->max_frame_size - ETHERFCSL)) {
155 freemsg(mp);
156 igb_log(igb, IGB_LOG_INFO, "igb_tx: packet oversize");
157 return (B_TRUE);
158 }
159 }
160
161 /*
162 * Check and recycle tx descriptors.
163 * The recycle threshold here should be selected carefully
164 */
165 if (tx_ring->tbd_free < igb->tx_recycle_thresh)
166 tx_ring->tx_recycle(tx_ring);
167
168 /*
169 * After the recycling, if the tbd_free is less than the
170 * tx_overload_threshold, assert overload, return B_FALSE;
171 * and we need to re-schedule the tx again.
172 */
173 if (tx_ring->tbd_free < igb->tx_overload_thresh) {
174 tx_ring->reschedule = B_TRUE;
175 IGB_DEBUG_STAT(tx_ring->stat_overload);
176 return (B_FALSE);
177 }
178
179 /*
180 * The software should guarantee LSO packet header(MAC+IP+TCP)
181 * to be within one descriptor - this is required by h/w.
182 * Here will reallocate and refill the header if
183 * the headers(MAC+IP+TCP) is physical memory non-contiguous.
184 */
185 if (ctx && ctx->lso_flag) {
186 hdr_len = ctx->mac_hdr_len + ctx->ip_hdr_len + ctx->l4_hdr_len;
187 len = MBLKL(mp);
188 hdr_current_mp = mp;
189 while (len < hdr_len) {
190 hdr_previous_mp = hdr_current_mp;
191 hdr_current_mp = hdr_current_mp->b_cont;
192 len += MBLKL(hdr_current_mp);
193 }
194 /*
195 * If the header and the payload are in different mblks,
196 * we simply force the header to be copied into pre-allocated
197 * page-aligned buffer.
198 */
199 if (len == hdr_len)
200 goto adjust_threshold;
201
202 hdr_frag_len = hdr_len - (len - MBLKL(hdr_current_mp));
203 /*
204 * There are two cases we will reallocate
205 * a mblk for the last header fragment.
206 * 1. the header is in multiple mblks and
207 * the last fragment shares the same mblk
208 * with the payload
209 * 2. the header is in a single mblk shared
210 * with the payload but the header crosses
211 * a page.
212 */
213 if ((hdr_current_mp != mp) ||
214 (P2NPHASE((uintptr_t)hdr_current_mp->b_rptr, igb->page_size)
215 < hdr_len)) {
216 /*
217 * reallocate the mblk for the last header fragment,
218 * expect it to be copied into pre-allocated
219 * page-aligned buffer
220 */
221 hdr_new_mp = allocb(hdr_frag_len, 0);
222 if (!hdr_new_mp) {
223 return (B_FALSE);
224 }
225
226 /* link the new header fragment with the other parts */
227 bcopy(hdr_current_mp->b_rptr,
228 hdr_new_mp->b_rptr, hdr_frag_len);
229 hdr_new_mp->b_wptr = hdr_new_mp->b_rptr + hdr_frag_len;
230 hdr_new_mp->b_cont = hdr_current_mp;
231 if (hdr_previous_mp)
232 hdr_previous_mp->b_cont = hdr_new_mp;
233 else
234 mp = hdr_new_mp;
235 hdr_current_mp->b_rptr += hdr_frag_len;
236 }
237 adjust_threshold:
238 /*
239 * adjust the bcopy threshhold to guarantee
240 * the header to use bcopy way
241 */
242 if (copy_thresh < hdr_len)
243 copy_thresh = hdr_len;
244 }
245
246 /*
247 * The pending_list is a linked list that is used to save
248 * the tx control blocks that have packet data processed
249 * but have not put the data to the tx descriptor ring.
250 * It is used to reduce the lock contention of the tx_lock.
251 */
252 LINK_LIST_INIT(&pending_list);
253 desc_num = 0;
254 desc_total = 0;
255
256 current_mp = mp;
257 current_len = MBLKL(current_mp);
258 /*
259 * Decide which method to use for the first fragment
260 */
261 current_flag = (current_len <= copy_thresh) ?
262 USE_COPY : USE_DMA;
263 /*
264 * If the mblk includes several contiguous small fragments,
265 * they may be copied into one buffer. This flag is used to
266 * indicate whether there are pending fragments that need to
267 * be copied to the current tx buffer.
268 *
269 * If this flag is B_TRUE, it indicates that a new tx control
270 * block is needed to process the next fragment using either
271 * copy or DMA binding.
272 *
273 * Otherwise, it indicates that the next fragment will be
274 * copied to the current tx buffer that is maintained by the
275 * current tx control block. No new tx control block is needed.
276 */
277 copy_done = B_TRUE;
278 while (current_mp) {
279 next_mp = current_mp->b_cont;
280 eop = (next_mp == NULL); /* Last fragment of the packet? */
281 next_len = eop ? 0: MBLKL(next_mp);
282
283 /*
284 * When the current fragment is an empty fragment, if
285 * the next fragment will still be copied to the current
286 * tx buffer, we cannot skip this fragment here. Because
287 * the copy processing is pending for completion. We have
288 * to process this empty fragment in the tx_copy routine.
289 *
290 * If the copy processing is completed or a DMA binding
291 * processing is just completed, we can just skip this
292 * empty fragment.
293 */
294 if ((current_len == 0) && (copy_done)) {
295 current_mp = next_mp;
296 current_len = next_len;
297 current_flag = (current_len <= copy_thresh) ?
298 USE_COPY : USE_DMA;
299 continue;
300 }
301
302 if (copy_done) {
303 /*
304 * Get a new tx control block from the free list
305 */
306 tcb = igb_get_free_list(tx_ring);
307
308 if (tcb == NULL) {
309 IGB_DEBUG_STAT(tx_ring->stat_fail_no_tcb);
310 goto tx_failure;
311 }
312
313 /*
314 * Push the tx control block to the pending list
315 * to avoid using lock too early
316 */
317 LIST_PUSH_TAIL(&pending_list, &tcb->link);
318 }
319
320 if (current_flag == USE_COPY) {
321 /*
322 * Check whether to use bcopy or DMA binding to process
323 * the next fragment, and if using bcopy, whether we
324 * need to continue copying the next fragment into the
325 * current tx buffer.
326 */
327 ASSERT((tcb->tx_buf.len + current_len) <=
328 tcb->tx_buf.size);
329
330 if (eop) {
331 /*
332 * This is the last fragment of the packet, so
333 * the copy processing will be completed with
334 * this fragment.
335 */
336 next_flag = USE_NONE;
337 copy_done = B_TRUE;
338 } else if ((tcb->tx_buf.len + current_len + next_len) >
339 tcb->tx_buf.size) {
340 /*
341 * If the next fragment is too large to be
342 * copied to the current tx buffer, we need
343 * to complete the current copy processing.
344 */
345 next_flag = (next_len > copy_thresh) ?
346 USE_DMA: USE_COPY;
347 copy_done = B_TRUE;
348 } else if (next_len > copy_thresh) {
349 /*
350 * The next fragment needs to be processed with
351 * DMA binding. So the copy prcessing will be
352 * completed with the current fragment.
353 */
354 next_flag = USE_DMA;
355 copy_done = B_TRUE;
356 } else {
357 /*
358 * Continue to copy the next fragment to the
359 * current tx buffer.
360 */
361 next_flag = USE_COPY;
362 copy_done = B_FALSE;
363 }
364
365 desc_num = igb_tx_copy(tx_ring, tcb, current_mp,
366 current_len, copy_done);
367 } else {
368 /*
369 * Check whether to use bcopy or DMA binding to process
370 * the next fragment.
371 */
372 next_flag = (next_len > copy_thresh) ?
373 USE_DMA: USE_COPY;
374 ASSERT(copy_done == B_TRUE);
375
376 desc_num = igb_tx_bind(tx_ring, tcb, current_mp,
377 current_len);
378 }
379
380 if (desc_num > 0)
381 desc_total += desc_num;
382 else if (desc_num < 0)
383 goto tx_failure;
384
385 current_mp = next_mp;
386 current_len = next_len;
387 current_flag = next_flag;
388 }
389
390 /*
391 * Attach the mblk to the last tx control block
392 */
393 ASSERT(tcb);
394 ASSERT(tcb->mp == NULL);
395 tcb->mp = mp;
396
397 /*
398 * Before fill the tx descriptor ring with the data, we need to
399 * ensure there are adequate free descriptors for transmit
400 * (including one context descriptor).
401 * Do not use up all the tx descriptors.
402 * Otherwise tx recycle will fail and cause false hang.
403 */
404 if (tx_ring->tbd_free <= (desc_total + 1)) {
405 tx_ring->tx_recycle(tx_ring);
406 }
407
408 mutex_enter(&tx_ring->tx_lock);
409
410 /*
411 * If the number of free tx descriptors is not enough for transmit
412 * then return failure.
413 *
414 * Note: we must put this check under the mutex protection to
415 * ensure the correctness when multiple threads access it in
416 * parallel.
417 */
418 if (tx_ring->tbd_free <= (desc_total + 1)) {
419 IGB_DEBUG_STAT(tx_ring->stat_fail_no_tbd);
420 mutex_exit(&tx_ring->tx_lock);
421 goto tx_failure;
422 }
423
424 desc_num = igb_tx_fill_ring(tx_ring, &pending_list, ctx, mbsize);
425
426 ASSERT((desc_num == desc_total) || (desc_num == (desc_total + 1)));
427
428 /* Update per-ring tx statistics */
429 tx_ring->tx_pkts++;
430 tx_ring->tx_bytes += mbsize;
431
432 mutex_exit(&tx_ring->tx_lock);
433
434 return (B_TRUE);
435
436 tx_failure:
437 /*
438 * If new mblk has been allocted for the last header
439 * fragment of a LSO packet, we should restore the
440 * modified mp.
441 */
442 if (hdr_new_mp) {
443 hdr_new_mp->b_cont = NULL;
444 freeb(hdr_new_mp);
445 hdr_current_mp->b_rptr -= hdr_frag_len;
446 if (hdr_previous_mp)
447 hdr_previous_mp->b_cont = hdr_current_mp;
448 else
449 mp = hdr_current_mp;
450 }
451
452 /*
453 * Discard the mblk and free the used resources
454 */
455 tcb = (tx_control_block_t *)LIST_GET_HEAD(&pending_list);
456 while (tcb) {
457 tcb->mp = NULL;
458
459 igb_free_tcb(tcb);
460
461 tcb = (tx_control_block_t *)
462 LIST_GET_NEXT(&pending_list, &tcb->link);
463 }
464
465 /*
466 * Return the tx control blocks in the pending list to the free list.
467 */
468 igb_put_free_list(tx_ring, &pending_list);
469
470 /* Transmit failed, do not drop the mblk, rechedule the transmit */
471 tx_ring->reschedule = B_TRUE;
472
473 return (B_FALSE);
474 }
475
476 /*
477 * igb_tx_copy
478 *
479 * Copy the mblk fragment to the pre-allocated tx buffer
480 */
481 static int
igb_tx_copy(igb_tx_ring_t * tx_ring,tx_control_block_t * tcb,mblk_t * mp,uint32_t len,boolean_t copy_done)482 igb_tx_copy(igb_tx_ring_t *tx_ring, tx_control_block_t *tcb, mblk_t *mp,
483 uint32_t len, boolean_t copy_done)
484 {
485 dma_buffer_t *tx_buf;
486 uint32_t desc_num;
487 _NOTE(ARGUNUSED(tx_ring));
488
489 tx_buf = &tcb->tx_buf;
490
491 /*
492 * Copy the packet data of the mblk fragment into the
493 * pre-allocated tx buffer, which is maintained by the
494 * tx control block.
495 *
496 * Several mblk fragments can be copied into one tx buffer.
497 * The destination address of the current copied fragment in
498 * the tx buffer is next to the end of the previous copied
499 * fragment.
500 */
501 if (len > 0) {
502 bcopy(mp->b_rptr, tx_buf->address + tx_buf->len, len);
503
504 tx_buf->len += len;
505 tcb->frag_num++;
506 }
507
508 desc_num = 0;
509
510 /*
511 * If it is the last fragment copied to the current tx buffer,
512 * in other words, if there's no remaining fragment or the remaining
513 * fragment requires a new tx control block to process, we need to
514 * complete the current copy processing by syncing up the current
515 * DMA buffer and saving the descriptor data.
516 */
517 if (copy_done) {
518 /*
519 * Sync the DMA buffer of the packet data
520 */
521 DMA_SYNC(tx_buf, DDI_DMA_SYNC_FORDEV);
522
523 tcb->tx_type = USE_COPY;
524
525 /*
526 * Save the address and length to the private data structure
527 * of the tx control block, which will be used to fill the
528 * tx descriptor ring after all the fragments are processed.
529 */
530 igb_save_desc(tcb, tx_buf->dma_address, tx_buf->len);
531 desc_num++;
532 }
533
534 return (desc_num);
535 }
536
537 /*
538 * igb_tx_bind
539 *
540 * Bind the mblk fragment with DMA
541 */
542 static int
igb_tx_bind(igb_tx_ring_t * tx_ring,tx_control_block_t * tcb,mblk_t * mp,uint32_t len)543 igb_tx_bind(igb_tx_ring_t *tx_ring, tx_control_block_t *tcb, mblk_t *mp,
544 uint32_t len)
545 {
546 int status, i;
547 ddi_dma_cookie_t dma_cookie;
548 uint_t ncookies;
549 int desc_num;
550
551 /*
552 * Use DMA binding to process the mblk fragment
553 */
554 status = ddi_dma_addr_bind_handle(tcb->tx_dma_handle, NULL,
555 (caddr_t)mp->b_rptr, len,
556 DDI_DMA_WRITE | DDI_DMA_STREAMING, DDI_DMA_DONTWAIT,
557 0, &dma_cookie, &ncookies);
558
559 if (status != DDI_DMA_MAPPED) {
560 IGB_DEBUG_STAT(tx_ring->stat_fail_dma_bind);
561 return (-1);
562 }
563
564 tcb->frag_num++;
565 tcb->tx_type = USE_DMA;
566 /*
567 * Each fragment can span several cookies. One cookie will have
568 * one tx descriptor to transmit.
569 */
570 desc_num = 0;
571 for (i = ncookies; i > 0; i--) {
572 /*
573 * Save the address and length to the private data structure
574 * of the tx control block, which will be used to fill the
575 * tx descriptor ring after all the fragments are processed.
576 */
577 igb_save_desc(tcb,
578 dma_cookie.dmac_laddress,
579 dma_cookie.dmac_size);
580
581 desc_num++;
582
583 if (i > 1)
584 ddi_dma_nextcookie(tcb->tx_dma_handle, &dma_cookie);
585 }
586
587 return (desc_num);
588 }
589
590 /*
591 * igb_get_tx_context
592 *
593 * Get the tx context information from the mblk
594 */
595 static int
igb_get_tx_context(mblk_t * mp,tx_context_t * ctx)596 igb_get_tx_context(mblk_t *mp, tx_context_t *ctx)
597 {
598 uint32_t start;
599 uint32_t flags;
600 uint32_t lso_flag;
601 uint32_t lso_cksum;
602 uint32_t mss;
603 uint32_t len;
604 uint32_t size;
605 uint32_t offset;
606 unsigned char *pos;
607 ushort_t etype;
608 uint32_t mac_hdr_len;
609 uint32_t l4_proto;
610 uint32_t l4_hdr_len;
611
612 ASSERT(mp != NULL);
613
614 mac_hcksum_get(mp, &start, NULL, NULL, NULL, &flags);
615 bzero(ctx, sizeof (tx_context_t));
616
617 ctx->hcksum_flags = flags;
618
619 if (flags == 0)
620 return (TX_CXT_SUCCESS);
621
622 mac_lso_get(mp, &mss, &lso_flag);
623 ctx->mss = mss;
624 ctx->lso_flag = (lso_flag == HW_LSO);
625
626 etype = 0;
627 mac_hdr_len = 0;
628 l4_proto = 0;
629
630 /*
631 * Firstly get the position of the ether_type/ether_tpid.
632 * Here we don't assume the ether (VLAN) header is fully included
633 * in one mblk fragment, so we go thourgh the fragments to parse
634 * the ether type.
635 */
636 size = len = MBLKL(mp);
637 offset = offsetof(struct ether_header, ether_type);
638 while (size <= offset) {
639 mp = mp->b_cont;
640 ASSERT(mp != NULL);
641 len = MBLKL(mp);
642 size += len;
643 }
644 pos = mp->b_rptr + offset + len - size;
645
646 etype = ntohs(*(ushort_t *)(uintptr_t)pos);
647 if (etype == ETHERTYPE_VLAN) {
648 /*
649 * Get the position of the ether_type in VLAN header
650 */
651 offset = offsetof(struct ether_vlan_header, ether_type);
652 while (size <= offset) {
653 mp = mp->b_cont;
654 ASSERT(mp != NULL);
655 len = MBLKL(mp);
656 size += len;
657 }
658 pos = mp->b_rptr + offset + len - size;
659
660 etype = ntohs(*(ushort_t *)(uintptr_t)pos);
661 mac_hdr_len = sizeof (struct ether_vlan_header);
662 } else {
663 mac_hdr_len = sizeof (struct ether_header);
664 }
665
666 /*
667 * Here we assume the IP(V6) header is fully included in one
668 * mblk fragment.
669 */
670 lso_cksum = HCK_PARTIALCKSUM;
671 ctx->l3_proto = etype;
672 switch (etype) {
673 case ETHERTYPE_IP:
674 offset = mac_hdr_len;
675 while (size <= offset) {
676 mp = mp->b_cont;
677 ASSERT(mp != NULL);
678 len = MBLKL(mp);
679 size += len;
680 }
681 pos = mp->b_rptr + offset + len - size;
682
683 if (ctx->lso_flag) {
684 *((uint16_t *)(uintptr_t)(pos + offsetof(ipha_t,
685 ipha_length))) = 0;
686
687 /*
688 * To utilize igb LSO, here need to fill
689 * the tcp checksum field of the packet with the
690 * following pseudo-header checksum:
691 * (ip_source_addr, ip_destination_addr, l4_proto)
692 * and also need to fill the ip header checksum
693 * with zero. Currently the tcp/ip stack has done
694 * these.
695 */
696 lso_cksum |= HCK_IPV4_HDRCKSUM;
697 }
698
699 l4_proto = *(uint8_t *)(pos + offsetof(ipha_t, ipha_protocol));
700 break;
701 case ETHERTYPE_IPV6:
702 /*
703 * We need to zero out the length in the header.
704 */
705 if (ctx->lso_flag) {
706 offset = offsetof(ip6_t, ip6_plen) + mac_hdr_len;
707 while (size <= offset) {
708 mp = mp->b_cont;
709 ASSERT(mp != NULL);
710 len = MBLKL(mp);
711 size += len;
712 }
713 pos = mp->b_rptr + offset + len - size;
714 *((uint16_t *)(uintptr_t)(pos)) = 0;
715 }
716
717 offset = offsetof(ip6_t, ip6_nxt) + mac_hdr_len;
718 while (size <= offset) {
719 mp = mp->b_cont;
720 ASSERT(mp != NULL);
721 len = MBLKL(mp);
722 size += len;
723 }
724 pos = mp->b_rptr + offset + len - size;
725
726 l4_proto = *(uint8_t *)pos;
727 break;
728 default:
729 /* Unrecoverable error */
730 igb_log(NULL, IGB_LOG_INFO, "Ethernet type field error with "
731 "tx hcksum flag set");
732 return (TX_CXT_E_ETHER_TYPE);
733 }
734
735 if (ctx->lso_flag) {
736 /*
737 * LSO relies on tx h/w checksum, so here the packet will be
738 * dropped if the h/w checksum flags are not set.
739 */
740 if ((ctx->hcksum_flags & lso_cksum) != lso_cksum) {
741 igb_log(NULL, IGB_LOG_INFO, "igb_tx: h/w "
742 "checksum flags are not set for LSO, found "
743 "0x%x, needed bits 0x%x", ctx->hcksum_flags,
744 lso_cksum);
745 return (TX_CXT_E_LSO_CSUM);
746 }
747
748 offset = mac_hdr_len + start;
749 while (size <= offset) {
750 mp = mp->b_cont;
751 ASSERT(mp != NULL);
752 len = MBLKL(mp);
753 size += len;
754 }
755 pos = mp->b_rptr + offset + len - size;
756
757 l4_hdr_len = TCP_HDR_LENGTH((tcph_t *)pos);
758 } else {
759 /*
760 * l4 header length is only required for LSO
761 */
762 l4_hdr_len = 0;
763 }
764
765 ctx->mac_hdr_len = mac_hdr_len;
766 ctx->ip_hdr_len = start;
767 ctx->l4_proto = l4_proto;
768 ctx->l4_hdr_len = l4_hdr_len;
769
770 return (TX_CXT_SUCCESS);
771 }
772
773 /*
774 * igb_check_tx_context
775 *
776 * Check if a new context descriptor is needed
777 */
778 static boolean_t
igb_check_tx_context(igb_tx_ring_t * tx_ring,tx_context_t * ctx)779 igb_check_tx_context(igb_tx_ring_t *tx_ring, tx_context_t *ctx)
780 {
781 tx_context_t *last;
782
783 if (ctx == NULL)
784 return (B_FALSE);
785
786 /*
787 * Compare the context data retrieved from the mblk and the
788 * stored context data of the last context descriptor. The data
789 * need to be checked are:
790 * hcksum_flags
791 * l4_proto
792 * l3_proto
793 * mss (only check for LSO)
794 * l4_hdr_len (only check for LSO)
795 * ip_hdr_len
796 * mac_hdr_len
797 * Either one of the above data is changed, a new context descriptor
798 * will be needed.
799 */
800 last = &tx_ring->tx_context;
801
802 if (ctx->hcksum_flags != 0) {
803 if ((ctx->hcksum_flags != last->hcksum_flags) ||
804 (ctx->l4_proto != last->l4_proto) ||
805 (ctx->l3_proto != last->l3_proto) ||
806 (ctx->lso_flag && ((ctx->mss != last->mss) ||
807 (ctx->l4_hdr_len != last->l4_hdr_len))) ||
808 (ctx->ip_hdr_len != last->ip_hdr_len) ||
809 (ctx->mac_hdr_len != last->mac_hdr_len)) {
810 return (B_TRUE);
811 }
812 }
813
814 return (B_FALSE);
815 }
816
817 /*
818 * igb_fill_tx_context
819 *
820 * Fill the context descriptor with hardware checksum informations
821 */
822 static void
igb_fill_tx_context(struct e1000_adv_tx_context_desc * ctx_tbd,tx_context_t * ctx,uint32_t ring_index)823 igb_fill_tx_context(struct e1000_adv_tx_context_desc *ctx_tbd,
824 tx_context_t *ctx, uint32_t ring_index)
825 {
826 /*
827 * Fill the context descriptor with the checksum
828 * context information we've got
829 */
830 ctx_tbd->vlan_macip_lens = ctx->ip_hdr_len;
831 ctx_tbd->vlan_macip_lens |= ctx->mac_hdr_len <<
832 E1000_ADVTXD_MACLEN_SHIFT;
833
834 ctx_tbd->type_tucmd_mlhl =
835 E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
836
837 /*
838 * When we have a TX context set up, we enforce that the ethertype is
839 * either IPv4 or IPv6 in igb_get_tx_context().
840 */
841 if (ctx->lso_flag || ctx->hcksum_flags & HCK_IPV4_HDRCKSUM) {
842 if (ctx->l3_proto == ETHERTYPE_IP) {
843 ctx_tbd->type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
844 } else {
845 ctx_tbd->type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6;
846 }
847 }
848
849 if (ctx->lso_flag || ctx->hcksum_flags & HCK_PARTIALCKSUM) {
850 switch (ctx->l4_proto) {
851 case IPPROTO_TCP:
852 ctx_tbd->type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
853 break;
854 case IPPROTO_UDP:
855 /*
856 * We don't have to explicitly set:
857 * ctx_tbd->type_tucmd_mlhl |=
858 * E1000_ADVTXD_TUCMD_L4T_UDP;
859 * Because E1000_ADVTXD_TUCMD_L4T_UDP == 0b
860 */
861 break;
862 default:
863 /* Unrecoverable error */
864 igb_log(NULL, IGB_LOG_INFO,
865 "L4 type error with tx hcksum");
866 break;
867 }
868 }
869
870 ctx_tbd->seqnum_seed = 0;
871 ctx_tbd->mss_l4len_idx = ring_index << 4;
872 if (ctx->lso_flag) {
873 ctx_tbd->mss_l4len_idx |=
874 (ctx->l4_hdr_len << E1000_ADVTXD_L4LEN_SHIFT) |
875 (ctx->mss << E1000_ADVTXD_MSS_SHIFT);
876 }
877 }
878
879 /*
880 * igb_tx_fill_ring
881 *
882 * Fill the tx descriptor ring with the data
883 */
884 static int
igb_tx_fill_ring(igb_tx_ring_t * tx_ring,link_list_t * pending_list,tx_context_t * ctx,size_t mbsize)885 igb_tx_fill_ring(igb_tx_ring_t *tx_ring, link_list_t *pending_list,
886 tx_context_t *ctx, size_t mbsize)
887 {
888 struct e1000_hw *hw = &tx_ring->igb->hw;
889 boolean_t load_context;
890 uint32_t index, tcb_index, desc_num;
891 union e1000_adv_tx_desc *tbd, *first_tbd;
892 tx_control_block_t *tcb, *first_tcb;
893 uint32_t hcksum_flags;
894 int i;
895 igb_t *igb = tx_ring->igb;
896
897 ASSERT(mutex_owned(&tx_ring->tx_lock));
898
899 tbd = NULL;
900 first_tbd = NULL;
901 first_tcb = NULL;
902 desc_num = 0;
903 hcksum_flags = 0;
904 load_context = B_FALSE;
905
906 /*
907 * Get the index of the first tx descriptor that will be filled,
908 * and the index of the first work list item that will be attached
909 * with the first used tx control block in the pending list.
910 * Note: the two indexes are the same.
911 */
912 index = tx_ring->tbd_tail;
913 tcb_index = tx_ring->tbd_tail;
914
915 if (ctx != NULL) {
916 hcksum_flags = ctx->hcksum_flags;
917
918 /*
919 * Check if a new context descriptor is needed for this packet
920 */
921 load_context = igb_check_tx_context(tx_ring, ctx);
922 if (load_context) {
923 tbd = &tx_ring->tbd_ring[index];
924
925 /*
926 * Fill the context descriptor with the
927 * hardware checksum offload informations.
928 */
929 igb_fill_tx_context(
930 (struct e1000_adv_tx_context_desc *)tbd,
931 ctx, tx_ring->index);
932
933 index = NEXT_INDEX(index, 1, tx_ring->ring_size);
934 desc_num++;
935
936 /*
937 * Store the checksum context data if
938 * a new context descriptor is added
939 */
940 tx_ring->tx_context = *ctx;
941 }
942 }
943
944 first_tbd = &tx_ring->tbd_ring[index];
945
946 /*
947 * Fill tx data descriptors with the data saved in the pending list.
948 * The tx control blocks in the pending list are added to the work list
949 * at the same time.
950 *
951 * The work list is strictly 1:1 corresponding to the descriptor ring.
952 * One item of the work list corresponds to one tx descriptor. Because
953 * one tx control block can span multiple tx descriptors, the tx
954 * control block will be added to the first work list item that
955 * corresponds to the first tx descriptor generated from that tx
956 * control block.
957 */
958 tcb = (tx_control_block_t *)LIST_POP_HEAD(pending_list);
959 first_tcb = tcb;
960 while (tcb != NULL) {
961
962 for (i = 0; i < tcb->desc_num; i++) {
963 tbd = &tx_ring->tbd_ring[index];
964
965 tbd->read.buffer_addr = tcb->desc[i].address;
966 tbd->read.cmd_type_len = tcb->desc[i].length;
967
968 tbd->read.cmd_type_len |= E1000_ADVTXD_DCMD_RS |
969 E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_DATA |
970 E1000_ADVTXD_DCMD_IFCS;
971
972 tbd->read.olinfo_status = 0;
973
974 index = NEXT_INDEX(index, 1, tx_ring->ring_size);
975 desc_num++;
976 }
977
978 /*
979 * Add the tx control block to the work list
980 */
981 ASSERT(tx_ring->work_list[tcb_index] == NULL);
982 tx_ring->work_list[tcb_index] = tcb;
983
984 tcb_index = index;
985 tcb = (tx_control_block_t *)LIST_POP_HEAD(pending_list);
986 }
987
988 if (load_context) {
989 /*
990 * Count the checksum context descriptor for
991 * the first tx control block.
992 */
993 first_tcb->desc_num++;
994 }
995 first_tcb->last_index = PREV_INDEX(index, 1, tx_ring->ring_size);
996
997 /*
998 * The Insert Ethernet CRC (IFCS) bit and the checksum fields are only
999 * valid in the first descriptor of the packet.
1000 * 82576 also requires the payload length setting even without LSO
1001 */
1002 ASSERT(first_tbd != NULL);
1003 first_tbd->read.cmd_type_len |= E1000_ADVTXD_DCMD_IFCS;
1004 if (ctx != NULL && ctx->lso_flag) {
1005 first_tbd->read.cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
1006 first_tbd->read.olinfo_status |=
1007 (mbsize - ctx->mac_hdr_len - ctx->ip_hdr_len
1008 - ctx->l4_hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT;
1009 } else {
1010 if (hw->mac.type >= e1000_82576) {
1011 first_tbd->read.olinfo_status |=
1012 (mbsize << E1000_ADVTXD_PAYLEN_SHIFT);
1013 }
1014 }
1015
1016 /* Set hardware checksum bits */
1017 if (hcksum_flags != 0) {
1018 if (hcksum_flags & HCK_IPV4_HDRCKSUM)
1019 first_tbd->read.olinfo_status |=
1020 E1000_TXD_POPTS_IXSM << 8;
1021 if (hcksum_flags & HCK_PARTIALCKSUM)
1022 first_tbd->read.olinfo_status |=
1023 E1000_TXD_POPTS_TXSM << 8;
1024 first_tbd->read.olinfo_status |= tx_ring->index << 4;
1025 }
1026
1027 /*
1028 * The last descriptor of packet needs End Of Packet (EOP),
1029 * and Report Status (RS) bits set
1030 */
1031 ASSERT(tbd != NULL);
1032 tbd->read.cmd_type_len |=
1033 E1000_ADVTXD_DCMD_EOP | E1000_ADVTXD_DCMD_RS;
1034
1035 IGB_DEBUG_STAT(tx_ring->stat_pkt_cnt);
1036
1037 /*
1038 * Sync the DMA buffer of the tx descriptor ring
1039 */
1040 DMA_SYNC(&tx_ring->tbd_area, DDI_DMA_SYNC_FORDEV);
1041
1042 /*
1043 * Update the number of the free tx descriptors.
1044 * The mutual exclusion between the transmission and the recycling
1045 * (for the tx descriptor ring and the work list) is implemented
1046 * with the atomic operation on the number of the free tx descriptors.
1047 *
1048 * Note: we should always decrement the counter tbd_free before
1049 * advancing the hardware TDT pointer to avoid the race condition -
1050 * before the counter tbd_free is decremented, the transmit of the
1051 * tx descriptors has done and the counter tbd_free is increased by
1052 * the tx recycling.
1053 */
1054 i = igb_atomic_reserve(&tx_ring->tbd_free, desc_num);
1055 ASSERT(i >= 0);
1056
1057 tx_ring->tbd_tail = index;
1058
1059 /*
1060 * Advance the hardware TDT pointer of the tx descriptor ring
1061 */
1062 E1000_WRITE_REG(hw, E1000_TDT(tx_ring->index), index);
1063
1064 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
1065 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
1066 atomic_or_32(&igb->igb_state, IGB_ERROR);
1067 }
1068
1069 return (desc_num);
1070 }
1071
1072 /*
1073 * igb_save_desc
1074 *
1075 * Save the address/length pair to the private array
1076 * of the tx control block. The address/length pairs
1077 * will be filled into the tx descriptor ring later.
1078 */
1079 static void
igb_save_desc(tx_control_block_t * tcb,uint64_t address,size_t length)1080 igb_save_desc(tx_control_block_t *tcb, uint64_t address, size_t length)
1081 {
1082 sw_desc_t *desc;
1083
1084 desc = &tcb->desc[tcb->desc_num];
1085 desc->address = address;
1086 desc->length = length;
1087
1088 tcb->desc_num++;
1089 }
1090
1091 /*
1092 * igb_tx_recycle_legacy
1093 *
1094 * Recycle the tx descriptors and tx control blocks.
1095 *
1096 * The work list is traversed to check if the corresponding
1097 * tx descriptors have been transmitted. If so, the resources
1098 * bound to the tx control blocks will be freed, and those
1099 * tx control blocks will be returned to the free list.
1100 */
1101 uint32_t
igb_tx_recycle_legacy(igb_tx_ring_t * tx_ring)1102 igb_tx_recycle_legacy(igb_tx_ring_t *tx_ring)
1103 {
1104 uint32_t index, last_index, next_index;
1105 int desc_num;
1106 boolean_t desc_done;
1107 tx_control_block_t *tcb;
1108 link_list_t pending_list;
1109 igb_t *igb = tx_ring->igb;
1110
1111 /*
1112 * The mutex_tryenter() is used to avoid unnecessary
1113 * lock contention.
1114 */
1115 if (mutex_tryenter(&tx_ring->recycle_lock) == 0)
1116 return (0);
1117
1118 ASSERT(tx_ring->tbd_free <= tx_ring->ring_size);
1119
1120 if (tx_ring->tbd_free == tx_ring->ring_size) {
1121 tx_ring->recycle_fail = 0;
1122 tx_ring->stall_watchdog = 0;
1123 mutex_exit(&tx_ring->recycle_lock);
1124 return (0);
1125 }
1126
1127 /*
1128 * Sync the DMA buffer of the tx descriptor ring
1129 */
1130 DMA_SYNC(&tx_ring->tbd_area, DDI_DMA_SYNC_FORKERNEL);
1131
1132 if (igb_check_dma_handle(
1133 tx_ring->tbd_area.dma_handle) != DDI_FM_OK) {
1134 mutex_exit(&tx_ring->recycle_lock);
1135 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
1136 atomic_or_32(&igb->igb_state, IGB_ERROR);
1137 return (0);
1138 }
1139
1140 LINK_LIST_INIT(&pending_list);
1141 desc_num = 0;
1142 index = tx_ring->tbd_head; /* Index of next tbd/tcb to recycle */
1143
1144 tcb = tx_ring->work_list[index];
1145 ASSERT(tcb != NULL);
1146
1147 while (tcb != NULL) {
1148
1149 /*
1150 * Get the last tx descriptor of this packet.
1151 * If the last tx descriptor is done, then
1152 * we can recycle all descriptors of a packet
1153 * which usually includes several tx control blocks.
1154 * For some chips, LSO descriptors can not be recycled
1155 * unless the whole packet's transmission is done.
1156 * That's why packet level recycling is used here.
1157 */
1158 last_index = tcb->last_index;
1159 /*
1160 * MAX_TX_RING_SIZE is used to judge whether
1161 * the index is a valid value or not.
1162 */
1163 if (last_index == MAX_TX_RING_SIZE)
1164 break;
1165
1166 next_index = NEXT_INDEX(last_index, 1, tx_ring->ring_size);
1167
1168 /*
1169 * Check if the Descriptor Done bit is set
1170 */
1171 desc_done = tx_ring->tbd_ring[last_index].wb.status &
1172 E1000_TXD_STAT_DD;
1173 if (desc_done) {
1174 while (tcb != NULL) {
1175 /*
1176 * Strip off the tx control block from the work
1177 * list, and add it to the pending list.
1178 */
1179 tx_ring->work_list[index] = NULL;
1180 LIST_PUSH_TAIL(&pending_list, &tcb->link);
1181
1182 /*
1183 * Count the total number of the tx descriptors
1184 * recycled.
1185 */
1186 desc_num += tcb->desc_num;
1187
1188 /*
1189 * Advance the index of the tx descriptor ring
1190 */
1191 index = NEXT_INDEX(index, tcb->desc_num,
1192 tx_ring->ring_size);
1193
1194 tcb = tx_ring->work_list[index];
1195 if (index == next_index)
1196 break;
1197 }
1198 } else {
1199 break;
1200 }
1201 }
1202
1203 /*
1204 * If no tx descriptors are recycled, no need to do more processing
1205 */
1206 if (desc_num == 0) {
1207 tx_ring->recycle_fail++;
1208 mutex_exit(&tx_ring->recycle_lock);
1209 return (0);
1210 }
1211
1212 tx_ring->recycle_fail = 0;
1213 tx_ring->stall_watchdog = 0;
1214
1215 /*
1216 * Update the head index of the tx descriptor ring
1217 */
1218 tx_ring->tbd_head = index;
1219
1220 /*
1221 * Update the number of the free tx descriptors with atomic operations
1222 */
1223 atomic_add_32(&tx_ring->tbd_free, desc_num);
1224
1225 mutex_exit(&tx_ring->recycle_lock);
1226
1227 /*
1228 * Free the resources used by the tx control blocks
1229 * in the pending list
1230 */
1231 tcb = (tx_control_block_t *)LIST_GET_HEAD(&pending_list);
1232 while (tcb != NULL) {
1233 /*
1234 * Release the resources occupied by the tx control block
1235 */
1236 igb_free_tcb(tcb);
1237
1238 tcb = (tx_control_block_t *)
1239 LIST_GET_NEXT(&pending_list, &tcb->link);
1240 }
1241
1242 /*
1243 * Add the tx control blocks in the pending list to the free list.
1244 */
1245 igb_put_free_list(tx_ring, &pending_list);
1246
1247 return (desc_num);
1248 }
1249
1250 /*
1251 * igb_tx_recycle_head_wb
1252 *
1253 * Check the head write-back, and recycle all the transmitted
1254 * tx descriptors and tx control blocks.
1255 */
1256 uint32_t
igb_tx_recycle_head_wb(igb_tx_ring_t * tx_ring)1257 igb_tx_recycle_head_wb(igb_tx_ring_t *tx_ring)
1258 {
1259 uint32_t index;
1260 uint32_t head_wb;
1261 int desc_num;
1262 tx_control_block_t *tcb;
1263 link_list_t pending_list;
1264 igb_t *igb = tx_ring->igb;
1265
1266 /*
1267 * The mutex_tryenter() is used to avoid unnecessary
1268 * lock contention.
1269 */
1270 if (mutex_tryenter(&tx_ring->recycle_lock) == 0)
1271 return (0);
1272
1273 ASSERT(tx_ring->tbd_free <= tx_ring->ring_size);
1274
1275 if (tx_ring->tbd_free == tx_ring->ring_size) {
1276 tx_ring->recycle_fail = 0;
1277 tx_ring->stall_watchdog = 0;
1278 mutex_exit(&tx_ring->recycle_lock);
1279 return (0);
1280 }
1281
1282 /*
1283 * Sync the DMA buffer of the tx descriptor ring
1284 *
1285 * Note: For head write-back mode, the tx descriptors will not
1286 * be written back, but the head write-back value is stored at
1287 * the last extra tbd at the end of the DMA area, we still need
1288 * to sync the head write-back value for kernel.
1289 *
1290 * DMA_SYNC(&tx_ring->tbd_area, DDI_DMA_SYNC_FORKERNEL);
1291 */
1292 (void) ddi_dma_sync(tx_ring->tbd_area.dma_handle,
1293 sizeof (union e1000_adv_tx_desc) * tx_ring->ring_size,
1294 sizeof (uint32_t),
1295 DDI_DMA_SYNC_FORKERNEL);
1296
1297 if (igb_check_dma_handle(
1298 tx_ring->tbd_area.dma_handle) != DDI_FM_OK) {
1299 mutex_exit(&tx_ring->recycle_lock);
1300 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
1301 atomic_or_32(&igb->igb_state, IGB_ERROR);
1302 return (0);
1303 }
1304
1305 LINK_LIST_INIT(&pending_list);
1306 desc_num = 0;
1307 index = tx_ring->tbd_head; /* Next index to clean */
1308
1309 /*
1310 * Get the value of head write-back
1311 */
1312 head_wb = *tx_ring->tbd_head_wb;
1313 while (index != head_wb) {
1314 tcb = tx_ring->work_list[index];
1315 ASSERT(tcb != NULL);
1316
1317 if (OFFSET(index, head_wb, tx_ring->ring_size) <
1318 tcb->desc_num) {
1319 /*
1320 * The current tx control block is not
1321 * completely transmitted, stop recycling
1322 */
1323 break;
1324 }
1325
1326 /*
1327 * Strip off the tx control block from the work list,
1328 * and add it to the pending list.
1329 */
1330 tx_ring->work_list[index] = NULL;
1331 LIST_PUSH_TAIL(&pending_list, &tcb->link);
1332
1333 /*
1334 * Advance the index of the tx descriptor ring
1335 */
1336 index = NEXT_INDEX(index, tcb->desc_num, tx_ring->ring_size);
1337
1338 /*
1339 * Count the total number of the tx descriptors recycled
1340 */
1341 desc_num += tcb->desc_num;
1342 }
1343
1344 /*
1345 * If no tx descriptors are recycled, no need to do more processing
1346 */
1347 if (desc_num == 0) {
1348 tx_ring->recycle_fail++;
1349 mutex_exit(&tx_ring->recycle_lock);
1350 return (0);
1351 }
1352
1353 tx_ring->recycle_fail = 0;
1354 tx_ring->stall_watchdog = 0;
1355
1356 /*
1357 * Update the head index of the tx descriptor ring
1358 */
1359 tx_ring->tbd_head = index;
1360
1361 /*
1362 * Update the number of the free tx descriptors with atomic operations
1363 */
1364 atomic_add_32(&tx_ring->tbd_free, desc_num);
1365
1366 mutex_exit(&tx_ring->recycle_lock);
1367
1368 /*
1369 * Free the resources used by the tx control blocks
1370 * in the pending list
1371 */
1372 tcb = (tx_control_block_t *)LIST_GET_HEAD(&pending_list);
1373 while (tcb) {
1374 /*
1375 * Release the resources occupied by the tx control block
1376 */
1377 igb_free_tcb(tcb);
1378
1379 tcb = (tx_control_block_t *)
1380 LIST_GET_NEXT(&pending_list, &tcb->link);
1381 }
1382
1383 /*
1384 * Add the tx control blocks in the pending list to the free list.
1385 */
1386 igb_put_free_list(tx_ring, &pending_list);
1387
1388 return (desc_num);
1389 }
1390
1391 /*
1392 * igb_free_tcb - free up the tx control block
1393 *
1394 * Free the resources of the tx control block, including
1395 * unbind the previously bound DMA handle, and reset other
1396 * control fields.
1397 */
1398 void
igb_free_tcb(tx_control_block_t * tcb)1399 igb_free_tcb(tx_control_block_t *tcb)
1400 {
1401 switch (tcb->tx_type) {
1402 case USE_COPY:
1403 /*
1404 * Reset the buffer length that is used for copy
1405 */
1406 tcb->tx_buf.len = 0;
1407 break;
1408 case USE_DMA:
1409 /*
1410 * Release the DMA resource that is used for
1411 * DMA binding.
1412 */
1413 (void) ddi_dma_unbind_handle(tcb->tx_dma_handle);
1414 break;
1415 default:
1416 break;
1417 }
1418
1419 /*
1420 * Free the mblk
1421 */
1422 if (tcb->mp != NULL) {
1423 freemsg(tcb->mp);
1424 tcb->mp = NULL;
1425 }
1426
1427 tcb->tx_type = USE_NONE;
1428 tcb->last_index = MAX_TX_RING_SIZE;
1429 tcb->frag_num = 0;
1430 tcb->desc_num = 0;
1431 }
1432
1433 /*
1434 * igb_get_free_list - Get a free tx control block from the free list
1435 *
1436 * The atomic operation on the number of the available tx control block
1437 * in the free list is used to keep this routine mutual exclusive with
1438 * the routine igb_put_check_list.
1439 */
1440 static tx_control_block_t *
igb_get_free_list(igb_tx_ring_t * tx_ring)1441 igb_get_free_list(igb_tx_ring_t *tx_ring)
1442 {
1443 tx_control_block_t *tcb;
1444
1445 /*
1446 * Check and update the number of the free tx control block
1447 * in the free list.
1448 */
1449 if (igb_atomic_reserve(&tx_ring->tcb_free, 1) < 0)
1450 return (NULL);
1451
1452 mutex_enter(&tx_ring->tcb_head_lock);
1453
1454 tcb = tx_ring->free_list[tx_ring->tcb_head];
1455 ASSERT(tcb != NULL);
1456 tx_ring->free_list[tx_ring->tcb_head] = NULL;
1457 tx_ring->tcb_head = NEXT_INDEX(tx_ring->tcb_head, 1,
1458 tx_ring->free_list_size);
1459
1460 mutex_exit(&tx_ring->tcb_head_lock);
1461
1462 return (tcb);
1463 }
1464
1465 /*
1466 * igb_put_free_list
1467 *
1468 * Put a list of used tx control blocks back to the free list
1469 *
1470 * A mutex is used here to ensure the serialization. The mutual exclusion
1471 * between igb_get_free_list and igb_put_free_list is implemented with
1472 * the atomic operation on the counter tcb_free.
1473 */
1474 void
igb_put_free_list(igb_tx_ring_t * tx_ring,link_list_t * pending_list)1475 igb_put_free_list(igb_tx_ring_t *tx_ring, link_list_t *pending_list)
1476 {
1477 uint32_t index;
1478 int tcb_num;
1479 tx_control_block_t *tcb;
1480
1481 mutex_enter(&tx_ring->tcb_tail_lock);
1482
1483 index = tx_ring->tcb_tail;
1484
1485 tcb_num = 0;
1486 tcb = (tx_control_block_t *)LIST_POP_HEAD(pending_list);
1487 while (tcb != NULL) {
1488 ASSERT(tx_ring->free_list[index] == NULL);
1489 tx_ring->free_list[index] = tcb;
1490
1491 tcb_num++;
1492
1493 index = NEXT_INDEX(index, 1, tx_ring->free_list_size);
1494
1495 tcb = (tx_control_block_t *)LIST_POP_HEAD(pending_list);
1496 }
1497
1498 tx_ring->tcb_tail = index;
1499
1500 /*
1501 * Update the number of the free tx control block
1502 * in the free list. This operation must be placed
1503 * under the protection of the lock.
1504 */
1505 atomic_add_32(&tx_ring->tcb_free, tcb_num);
1506
1507 mutex_exit(&tx_ring->tcb_tail_lock);
1508 }
1509